test: move encrypted tests to monolith tests

The encrypted database/backup/restore tests sometimes fail because of
other (parallel) backup operations. To reduce the probability of this,
we will try to move these integration tests into the monolitic test
run that tests all samples serially.

Updates #1019

samples/snippets/src/test/java/com/example/spanner/EncryptionKeyIT.java

@@ -32,6 +32,7 @@
 import java.util.function.Predicate;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.JUnit4;
@@ -41,6 +42,7 @@
  * CreateBackupWithEncryptionKey} and {@link RestoreBackupWithEncryptionKey}
  */
 @RunWith(JUnit4.class)
+@Ignore
 public class EncryptionKeyIT {
 
   private static String projectId;

samples/snippets/src/test/java/com/example/spanner/SpannerSampleIT.java

@@ -27,12 +27,15 @@
 import com.google.cloud.spanner.InstanceId;
 import com.google.cloud.spanner.Spanner;
 import com.google.cloud.spanner.SpannerException;
+import com.google.cloud.spanner.SpannerExceptionFactory;
 import com.google.cloud.spanner.SpannerOptions;
+import com.google.common.base.Preconditions;
 import com.google.common.util.concurrent.Uninterruptibles;
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
 import java.util.regex.Pattern;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -51,9 +54,13 @@ public class SpannerSampleIT {
   private static final String instanceId = System.getProperty("spanner.test.instance");
   private static final String baseDbId = System.getProperty("spanner.sample.database");
   private static final String databaseId = formatForTest(baseDbId);
+  private static final String encryptedDatabaseId = formatForTest(baseDbId);
+  private static final String encryptedBackupId = formatForTest(baseDbId);
+  private static final String encryptedRestoreId = formatForTest(baseDbId);
   static Spanner spanner;
   static DatabaseId dbId;
   static DatabaseAdminClient dbClient;
+  private static String key;
   private long lastUpdateDataTimeInMillis;
 
   private String runSample(String command) throws Exception {
@@ -75,6 +82,13 @@ public static void setUp() throws Exception {
     dbId = DatabaseId.of(options.getProjectId(), instanceId, databaseId);
     // Delete stale test databases that have been created earlier by this test, but not deleted.
     deleteStaleTestDatabases(instanceId, baseDbId);
+
+    String keyLocation = Preconditions
+        .checkNotNull(System.getProperty("spanner.test.key.location"));
+    String keyRing = Preconditions.checkNotNull(System.getProperty("spanner.test.key.ring"));
+    String keyName = Preconditions.checkNotNull(System.getProperty("spanner.test.key.name"));
+    key = "projects/" + options.getProjectId() + "/locations/" + keyLocation + "/keyRings/" + keyRing
+        + "/cryptoKeys/" + keyName;
   }
 
   static void deleteStaleTestDatabases(String instanceId, String baseDbId) {
@@ -102,6 +116,9 @@ public static void tearDown() throws Exception {
     dbClient.dropDatabase(dbId.getInstanceId().getInstance(), dbId.getDatabase());
     dbClient.dropDatabase(
         dbId.getInstanceId().getInstance(), SpannerSample.createRestoredSampleDbId(dbId));
+    dbClient.dropDatabase(instanceId, encryptedDatabaseId);
+    dbClient.dropDatabase(instanceId, encryptedRestoreId);
+    dbClient.deleteBackup(instanceId, encryptedBackupId);
     spanner.close();
   }
 
@@ -384,6 +401,39 @@ public void testSample() throws Exception {
 
     out = runSample("deletebackup");
     assertThat(out).contains("Deleted backup [" + backupId + "]");
+
+    String projectId = spanner.getOptions().getProjectId();
+    out = SampleRunner
+        .runSample(() -> CreateDatabaseWithEncryptionKey.createDatabaseWithEncryptionKey(
+            dbClient,
+            projectId,
+            instanceId,
+            encryptedDatabaseId,
+            key));
+    assertThat(out).contains(String.format(
+        "Database projects/%s/instances/%s/databases/%s created with encryption key %s",
+        projectId, instanceId, encryptedDatabaseId, key));
+
+    out = SampleRunner.runSampleWithRetry(
+        () -> CreateBackupWithEncryptionKey.createBackupWithEncryptionKey(dbClient, projectId,
+            instanceId, encryptedDatabaseId, encryptedBackupId, key),
+        new ShouldRetryBackupOperation());
+    assertThat(out).containsMatch(String.format(
+        "Backup projects/%s/instances/%s/backups/%s of size \\d+ bytes was created at (.*) using encryption key %s",
+        projectId, instanceId, encryptedBackupId, key));
+
+    out = SampleRunner.runSampleWithRetry(
+        () -> RestoreBackupWithEncryptionKey.restoreBackupWithEncryptionKey(dbClient, projectId,
+            instanceId, encryptedBackupId, encryptedRestoreId, key),
+        new ShouldRetryBackupOperation());
+    assertThat(out).contains(String.format(
+        "Database projects/%s/instances/%s/databases/%s"
+        + " restored to projects/%s/instances/%s/databases/%s" 
+        + " from backup projects/%s/instances/%s/backups/%s" 
+        + " using encryption key %s",
+        projectId, instanceId, encryptedDatabaseId,
+        projectId, instanceId, encryptedRestoreId,
+        projectId, instanceId, encryptedBackupId, key));
   }
 
   private String runSampleRunnable(Runnable sample) {
@@ -435,4 +485,28 @@ private static Pattern getTestDbIdPattern(String baseDbId) {
   static String formatForTest(String name) {
     return name + "-" + UUID.randomUUID().toString().substring(0, DBID_LENGTH);
   }
+
+  static class ShouldRetryBackupOperation implements Predicate<SpannerException> {
+    private static final int MAX_ATTEMPTS = 20;
+    private int attempts = 0;
+
+    @Override
+    public boolean test(SpannerException e) {
+      if (e.getErrorCode() == ErrorCode.FAILED_PRECONDITION
+          && e.getMessage().contains("Please retry the operation once the pending")) {
+        attempts++;
+        if (attempts == MAX_ATTEMPTS) {
+          // Throw custom exception so it is easier to locate in the log why it went wrong.
+          throw SpannerExceptionFactory.newSpannerException(ErrorCode.DEADLINE_EXCEEDED,
+              String.format("Operation failed %d times because of other pending operations. "
+                  + "Giving up operation.\n", attempts),
+              e);
+        }
+        // Wait one minute before retrying.
+        Uninterruptibles.sleepUninterruptibly(60L, TimeUnit.SECONDS);
+        return true;
+      }
+      return false;
+    }
+  }
 }