Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: clean up test instance if creation failed #162

Merged
merged 1 commit into from Apr 22, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
12 changes: 12 additions & 0 deletions google-cloud-spanner/clirr-ignored-differences.xml
Expand Up @@ -147,4 +147,16 @@
<method>com.google.api.gax.paging.Page listDatabases()</method>
</difference>

<!-- Adding operation RPCs to InstanceAdminClient. -->
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/spanner/InstanceAdminClient</className>
<method>void cancelOperation(java.lang.String)</method>
</difference>
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/spanner/InstanceAdminClient</className>
<method>com.google.longrunning.Operation getOperation(java.lang.String)</method>
</difference>

</differences>
Expand Up @@ -20,6 +20,7 @@
import com.google.api.gax.paging.Page;
import com.google.cloud.Policy;
import com.google.cloud.spanner.Options.ListOption;
import com.google.longrunning.Operation;
import com.google.spanner.admin.instance.v1.CreateInstanceMetadata;
import com.google.spanner.admin.instance.v1.UpdateInstanceMetadata;

Expand Down Expand Up @@ -217,4 +218,10 @@ OperationFuture<Instance, UpdateInstanceMetadata> updateInstance(

/** Returns a builder for {@code Instance} object with the given id. */
Instance.Builder newInstanceBuilder(InstanceId id);

/** Cancels the specified long-running operation. */
void cancelOperation(String name);

/** Gets the specified long-running operation. */
Operation getOperation(String name);
}
Expand Up @@ -30,6 +30,7 @@
import com.google.cloud.spanner.spi.v1.SpannerRpc;
import com.google.cloud.spanner.spi.v1.SpannerRpc.Paginated;
import com.google.common.base.Preconditions;
import com.google.longrunning.Operation;
import com.google.protobuf.FieldMask;
import com.google.spanner.admin.instance.v1.CreateInstanceMetadata;
import com.google.spanner.admin.instance.v1.UpdateInstanceMetadata;
Expand Down Expand Up @@ -220,4 +221,14 @@ public Iterable<String> testInstanceIAMPermissions(
public Instance.Builder newInstanceBuilder(InstanceId id) {
return new Instance.Builder(this, dbClient, id);
}

@Override
public void cancelOperation(String name) {
rpc.cancelOperation(Preconditions.checkNotNull(name));
}

@Override
public Operation getOperation(String name) {
return rpc.getOperation(Preconditions.checkNotNull(name));
}
}
Expand Up @@ -22,6 +22,8 @@
import com.google.cloud.spanner.testing.RemoteSpannerHelper;
import com.google.common.collect.Iterators;
import com.google.spanner.admin.instance.v1.CreateInstanceMetadata;
import io.grpc.Status;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
Expand Down Expand Up @@ -82,7 +84,10 @@ protected void before() throws Throwable {
isOwnedInstance = false;
logger.log(Level.INFO, "Using existing test instance: {0}", instanceId);
} else {
instanceId = InstanceId.of(config.spannerOptions().getProjectId(), "test-instance");
instanceId =
InstanceId.of(
config.spannerOptions().getProjectId(),
String.format("test-instance-%08d", new Random().nextInt(100000000)));
isOwnedInstance = true;
}
testHelper = createTestHelper(options, instanceId);
Expand Down Expand Up @@ -123,6 +128,34 @@ private void initializeInstance(InstanceId instanceId) {
try {
createdInstance = op.get(30000L, TimeUnit.MILLISECONDS);
} catch (Exception e) {
boolean cancelled = false;
try {
// Try to cancel the createInstance operation.
instanceAdminClient.cancelOperation(op.getName());
com.google.longrunning.Operation createOperation =
instanceAdminClient.getOperation(op.getName());
cancelled =
createOperation.hasError()
&& createOperation.getError().getCode() == Status.CANCELLED.getCode().value();
if (cancelled) {
logger.info("Cancelled the createInstance operation because the operation failed");
} else {
logger.info(
"Tried to cancel the createInstance operation because the operation failed, but the operation could not be cancelled. Current status: "
+ createOperation.getError().getCode());
}
} catch (Throwable t) {
logger.log(Level.WARNING, "Failed to cancel the createInstance operation", t);
}
if (!cancelled) {
try {
instanceAdminClient.deleteInstance(instanceId.getInstance());
logger.info(
"Deleted the test instance because the createInstance operation failed and cancelling the operation did not succeed");
} catch (Throwable t) {
logger.log(Level.WARNING, "Failed to delete the test instance", t);
}
}
throw SpannerExceptionFactory.newSpannerException(e);
}
logger.log(Level.INFO, "Created test instance: {0}", createdInstance.getId());
Expand Down