Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: retry restore if blocked by pending restore #119

Merged
merged 1 commit into from Mar 20, 2020
Merged

fix: retry restore if blocked by pending restore #119

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
40 changes: 34 additions & 6 deletions google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBackupTest.java
View file
Open in desktop
Expand Up @@ -21,6 +21,7 @@

import com.google.api.gax.longrunning.OperationFuture;
import com.google.api.gax.paging.Page;
import com.google.api.gax.rpc.FailedPreconditionException;
import com.google.cloud.Timestamp;
import com.google.cloud.spanner.Backup;
import com.google.cloud.spanner.BackupId;
Expand Down Expand Up @@ -475,13 +476,40 @@ private void testRestore(Backup backup, OperationFuture<Backup, CreateBackupMeta
throws InterruptedException, ExecutionException {
// Restore the backup to a new database.
String restoredDb = testHelper.getUniqueDatabaseId();
logger.info(
String.format(
"Restoring backup %s to database %s", backup.getId().getBackup(), restoredDb));
OperationFuture<Database, RestoreDatabaseMetadata> restoreOp =
backup.restore(DatabaseId.of(testHelper.getInstanceId(), restoredDb));
String restoreOperationName;
OperationFuture<Database, RestoreDatabaseMetadata> restoreOp;
int attempts = 0;
while (true) {
try {
logger.info(
String.format(
"Restoring backup %s to database %s", backup.getId().getBackup(), restoredDb));
restoreOp = backup.restore(DatabaseId.of(testHelper.getInstanceId(), restoredDb));
restoreOperationName = restoreOp.getName();
break;
} catch (ExecutionException e) {
if (e.getCause() instanceof FailedPreconditionException
&& e.getCause()
.getMessage()
.contains("Please retry the operation once the pending restores complete")) {
attempts++;
if (attempts == 10) {
logger.info(
"Restore operation failed 10 times because of other pending restores. Skipping restore test.");
return;
}
// wait and then retry.
logger.info(
String.format(
"Restoring backup %s to database %s must wait because of other pending restore operation",
backup.getId().getBackup(), restoredDb));
Thread.sleep(60_000L);
} else {
throw e;
}
}
}
databases.add(restoredDb);
final String restoreOperationName = restoreOp.getName();
logger.info(String.format("Restore operation %s running", restoreOperationName));
RestoreDatabaseMetadata metadata = restoreOp.getMetadata().get();
assertThat(metadata.getBackupInfo().getBackup()).isEqualTo(backup.getId().getName());
Expand Down