test: retry backup operations on failure because of pending operations

Retry backup operations during tests that fail because too many other backup
operations are pending at that moment.

Fixes #1019

samples/snippets/src/test/java/com/example/spanner/EncryptionKeyIT.java

@@ -17,13 +17,17 @@
 package com.example.spanner;
 
 import static com.google.common.truth.Truth.assertThat;
-
 import com.google.cloud.spanner.DatabaseAdminClient;
+import com.google.cloud.spanner.ErrorCode;
 import com.google.cloud.spanner.Spanner;
+import com.google.cloud.spanner.SpannerException;
 import com.google.cloud.spanner.SpannerOptions;
 import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.Uninterruptibles;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -104,32 +108,54 @@ public void testEncryptedDatabaseAndBackupAndRestore() throws Exception {
         "Database projects/" + projectId + "/instances/" + instanceId + "/databases/" + databaseId
             + " created with encryption key " + key);
 
-    out = SampleRunner.runSample(() ->
+    out = SampleRunner.runSampleWithRetry(() ->
         CreateBackupWithEncryptionKey.createBackupWithEncryptionKey(
             databaseAdminClient,
             projectId,
             instanceId,
             databaseId,
             backupId,
             key
-        ));
+        ), new ShouldRetryBackupOperation());
     assertThat(out).containsMatch(
         "Backup projects/" + projectId + "/instances/" + instanceId + "/backups/" + backupId
             + " of size \\d+ bytes was created at (.*) using encryption key " + key);
 
-    out = SampleRunner.runSample(() ->
+    out = SampleRunner.runSampleWithRetry(() ->
         RestoreBackupWithEncryptionKey.restoreBackupWithEncryptionKey(
             databaseAdminClient,
             projectId,
             instanceId,
             backupId,
             restoreId,
             key
-        ));
+        ), new ShouldRetryBackupOperation());
     assertThat(out).contains(
         "Database projects/" + projectId + "/instances/" + instanceId + "/databases/" + databaseId
             + " restored to projects/" + projectId + "/instances/" + instanceId + "/databases/"
             + restoreId + " from backup projects/" + projectId + "/instances/" + instanceId
             + "/backups/" + backupId + " using encryption key " + key);
   }
+
+  private static class ShouldRetryBackupOperation implements Predicate<SpannerException> {
+    private static final int MAX_ATTEMPTS = 10;
+    private int attempts = 0;
+
+    @Override
+    public boolean test(SpannerException e) {
+      if (e.getErrorCode() == ErrorCode.FAILED_PRECONDITION
+          && e.getMessage().contains("Please retry the operation once the pending")) {
+        attempts++;
+        if (attempts == MAX_ATTEMPTS) {
+          System.out.printf("Operation failed %d times because of other pending operations. "
+              + "Giving up operation.\n", attempts);
+          return false;
+        }
+        // Wait one minute before retrying.
+        Uninterruptibles.sleepUninterruptibly(60L, TimeUnit.SECONDS);
+        return true;
+      }
+      return false;
+    }
+  }
 }

samples/snippets/src/test/java/com/example/spanner/SampleRunner.java

@@ -16,20 +16,34 @@
 
 package com.example.spanner;
 
+import com.google.cloud.spanner.SpannerException;
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
 import java.util.concurrent.Callable;
+import java.util.function.Predicate;
 
-/**
- * Runs a sample and captures the output as a String.
- */
+/** Runs a sample and captures the output as a String. */
 public class SampleRunner {
   public static String runSample(Callable<Void> sample) throws Exception {
+    return runSampleWithRetry(sample, e -> false);
+  }
+
+  public static String runSampleWithRetry(Callable<Void> sample,
+      Predicate<SpannerException> shouldRetry) throws Exception {
     final PrintStream stdOut = System.out;
     final ByteArrayOutputStream bout = new ByteArrayOutputStream();
     final PrintStream out = new PrintStream(bout);
     System.setOut(out);
-    sample.call();
+    while (true) {
+      try {
+        sample.call();
+        break;
+      } catch (SpannerException e) {
+        if (!shouldRetry.test(e)) {
+          throw e;
+        }
+      }
+    }
     System.setOut(stdOut);
     return bout.toString();
   }