Merge pull request #185 from Tinsane/table_changes

Fixed bug with ghost file during increment

docker/pg/scripts/ghost_table_test.sh

@@ -0,0 +1,52 @@
+#!/bin/sh
+set -e -x
+
+export WALE_S3_PREFIX=s3://waldeltabucket
+export WALG_USE_WAL_DELTA=true
+
+/usr/lib/postgresql/10/bin/initdb ${PGDATA}
+
+echo "archive_mode = on" >> /var/lib/postgresql/10/main/postgresql.conf
+echo "archive_command = '/usr/bin/timeout 600 /usr/bin/wal-g wal-push %p && mkdir -p /tmp/deltas/$(basename %p)'" >> /var/lib/postgresql/10/main/postgresql.conf
+echo "archive_timeout = 600" >> /var/lib/postgresql/10/main/postgresql.conf
+
+/usr/lib/postgresql/10/bin/pg_ctl -D ${PGDATA} -w start
+
+pgbench -i -s 10 postgres
+psql -c "create table ghost (a int, b int);"
+wal-g backup-push ${PGDATA}
+pgbench -i -s 10 postgres
+psql -c "insert into ghost values (1, 2);"
+wal-g backup-push ${PGDATA}
+psql -c "drop table ghost;"
+pgbench -i -s 10 postgres
+/usr/lib/postgresql/10/bin/pg_ctl -D ${PGDATA} stop 
+/usr/lib/postgresql/10/bin/pg_ctl -D ${PGDATA} -w start
+wal-g backup-push ${PGDATA}
+
+pgbench -i -s 10 postgres
+psql -c "create table ghost (a int, b int);"
+psql -c "insert into ghost values (3, 4);"
+
+pg_dumpall -f /tmp/dump1
+sleep 1
+wal-g backup-push ${PGDATA}
+
+pkill -9 postgres
+
+rm -rf ${PGDATA}
+
+wal-g backup-fetch ${PGDATA} LATEST
+
+echo "restore_command = 'echo \"WAL file restoration: %f, %p\"&& /usr/bin/wal-g wal-fetch \"%f\" \"%p\"'" > ${PGDATA}/recovery.conf
+
+/usr/lib/postgresql/10/bin/pg_ctl -D ${PGDATA} -w start
+
+pg_dumpall -f /tmp/dump2
+
+diff /tmp/dump1 /tmp/dump2
+
+pkill -9 postgres
+rm -rf ${PGDATA}
+
+echo "Ghost table backup success!!!!!!"

docker/pg/scripts/run_integration_tests.sh

@@ -6,3 +6,4 @@ set -e -x
 ./tmp/delta_backup_wal_delta_test.sh
 ./tmp/wale_compatibility_test.sh
 ./tmp/several_delta_backups_test.sh
+./tmp/ghost_table_test.sh

internal/tar_interpreter.go

@@ -30,20 +30,20 @@ func NewFileTarInterpreter(dbDataDirectory string, sentinel BackupSentinelDto, f
 
 // TODO : unit tests
 func (tarInterpreter *FileTarInterpreter) unwrapRegularFile(fileReader io.Reader, fileInfo *tar.Header, targetPath string) error {
-	fileDescription, haveFileDescription := tarInterpreter.Sentinel.Files[fileInfo.Name]
-
-	// If this file is incremental we use it's base version from incremental path
-	if haveFileDescription && tarInterpreter.Sentinel.isIncremental() && fileDescription.IsIncremented {
-		err := ApplyFileIncrement(targetPath, fileReader)
-		return errors.Wrapf(err, "Interpret: failed to apply increment for '%s'", targetPath)
-	}
 	if tarInterpreter.FilesToUnwrap != nil {
 		if _, ok := tarInterpreter.FilesToUnwrap[fileInfo.Name]; !ok {
 			// don't have to unwrap it this time
 			tracelog.DebugLogger.Printf("Don't have to unwrap '%s' this time\n", fileInfo.Name)
 			return nil
 		}
 	}
+	fileDescription, haveFileDescription := tarInterpreter.Sentinel.Files[fileInfo.Name]
+
+	// If this file is incremental we use it's base version from incremental path
+	if haveFileDescription && tarInterpreter.Sentinel.isIncremental() && fileDescription.IsIncremented {
+		err := ApplyFileIncrement(targetPath, fileReader)
+		return errors.Wrapf(err, "Interpret: failed to apply increment for '%s'", targetPath)
+	}
 	err := prepareDirs(fileInfo.Name, targetPath)
 	if err != nil {
 		return errors.Wrap(err, "Interpret: failed to create all directories")