fix: clean up test instance if creation failed (#162)

googleapis · Apr 22, 2020 · ff571e1 · ff571e1
1 parent 73cd964
commit ff571e1
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 1 deletion.
diff --git a/google-cloud-spanner/clirr-ignored-differences.xml b/google-cloud-spanner/clirr-ignored-differences.xml
@@ -147,4 +147,16 @@
     <method>com.google.api.gax.paging.Page listDatabases()</method>
   </difference>
 
+  <!-- Adding operation RPCs to InstanceAdminClient. -->
+  <difference>
+    <differenceType>7012</differenceType>
+    <className>com/google/cloud/spanner/InstanceAdminClient</className>
+    <method>void cancelOperation(java.lang.String)</method>
+  </difference>
+  <difference>
+    <differenceType>7012</differenceType>
+    <className>com/google/cloud/spanner/InstanceAdminClient</className>
+    <method>com.google.longrunning.Operation getOperation(java.lang.String)</method>
+  </difference>
+
 </differences>
diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/InstanceAdminClient.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/InstanceAdminClient.java
@@ -20,6 +20,7 @@
 import com.google.api.gax.paging.Page;
 import com.google.cloud.Policy;
 import com.google.cloud.spanner.Options.ListOption;
+import com.google.longrunning.Operation;
 import com.google.spanner.admin.instance.v1.CreateInstanceMetadata;
 import com.google.spanner.admin.instance.v1.UpdateInstanceMetadata;
 
@@ -217,4 +218,10 @@ OperationFuture<Instance, UpdateInstanceMetadata> updateInstance(
 
   /** Returns a builder for {@code Instance} object with the given id. */
   Instance.Builder newInstanceBuilder(InstanceId id);
+
+  /** Cancels the specified long-running operation. */
+  void cancelOperation(String name);
+
+  /** Gets the specified long-running operation. */
+  Operation getOperation(String name);
 }
diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/InstanceAdminClientImpl.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/InstanceAdminClientImpl.java
@@ -30,6 +30,7 @@
 import com.google.cloud.spanner.spi.v1.SpannerRpc;
 import com.google.cloud.spanner.spi.v1.SpannerRpc.Paginated;
 import com.google.common.base.Preconditions;
+import com.google.longrunning.Operation;
 import com.google.protobuf.FieldMask;
 import com.google.spanner.admin.instance.v1.CreateInstanceMetadata;
 import com.google.spanner.admin.instance.v1.UpdateInstanceMetadata;
@@ -220,4 +221,14 @@ public Iterable<String> testInstanceIAMPermissions(
   public Instance.Builder newInstanceBuilder(InstanceId id) {
     return new Instance.Builder(this, dbClient, id);
   }
+
+  @Override
+  public void cancelOperation(String name) {
+    rpc.cancelOperation(Preconditions.checkNotNull(name));
+  }
+
+  @Override
+  public Operation getOperation(String name) {
+    return rpc.getOperation(Preconditions.checkNotNull(name));
+  }
 }
diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/IntegrationTestEnv.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/IntegrationTestEnv.java
@@ -22,6 +22,8 @@
 import com.google.cloud.spanner.testing.RemoteSpannerHelper;
 import com.google.common.collect.Iterators;
 import com.google.spanner.admin.instance.v1.CreateInstanceMetadata;
+import io.grpc.Status;
+import java.util.Random;
 import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -82,7 +84,10 @@ protected void before() throws Throwable {
       isOwnedInstance = false;
       logger.log(Level.INFO, "Using existing test instance: {0}", instanceId);
     } else {
-      instanceId = InstanceId.of(config.spannerOptions().getProjectId(), "test-instance");
+      instanceId =
+          InstanceId.of(
+              config.spannerOptions().getProjectId(),
+              String.format("test-instance-%08d", new Random().nextInt(100000000)));
       isOwnedInstance = true;
     }
     testHelper = createTestHelper(options, instanceId);
@@ -123,6 +128,34 @@ private void initializeInstance(InstanceId instanceId) {
     try {
       createdInstance = op.get(30000L, TimeUnit.MILLISECONDS);
     } catch (Exception e) {
+      boolean cancelled = false;
+      try {
+        // Try to cancel the createInstance operation.
+        instanceAdminClient.cancelOperation(op.getName());
+        com.google.longrunning.Operation createOperation =
+            instanceAdminClient.getOperation(op.getName());
+        cancelled =
+            createOperation.hasError()
+                && createOperation.getError().getCode() == Status.CANCELLED.getCode().value();
+        if (cancelled) {
+          logger.info("Cancelled the createInstance operation because the operation failed");
+        } else {
+          logger.info(
+              "Tried to cancel the createInstance operation because the operation failed, but the operation could not be cancelled. Current status: "
+                  + createOperation.getError().getCode());
+        }
+      } catch (Throwable t) {
+        logger.log(Level.WARNING, "Failed to cancel the createInstance operation", t);
+      }
+      if (!cancelled) {
+        try {
+          instanceAdminClient.deleteInstance(instanceId.getInstance());
+          logger.info(
+              "Deleted the test instance because the createInstance operation failed and cancelling the operation did not succeed");
+        } catch (Throwable t) {
+          logger.log(Level.WARNING, "Failed to delete the test instance", t);
+        }
+      }
       throw SpannerExceptionFactory.newSpannerException(e);
     }
     logger.log(Level.INFO, "Created test instance: {0}", createdInstance.getId());