From 3f9f74aed52bce681b4bfd10d1006e5fa05b7cc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20Olav=20L=C3=B8ite?= Date: Tue, 1 Dec 2020 00:04:21 +0100 Subject: [PATCH] feat: retry admin request limit exceeded error (#669) * feat: retry admin request limit exceeded error Automatically retry requests that fail because the admin requests per seconds limit has been exceeded using an exponential backoff. Fixes #655 and others * fix: remove unused variable * fix: extract strings to constants --- ...minRequestsPerMinuteExceededException.java | 36 ++ .../spanner/SpannerExceptionFactory.java | 23 ++ .../cloud/spanner/spi/v1/GapicSpannerRpc.java | 390 +++++++++++++----- .../spanner/spi/v1/GapicSpannerRpcTest.java | 38 ++ 4 files changed, 384 insertions(+), 103 deletions(-) create mode 100644 google-cloud-spanner/src/main/java/com/google/cloud/spanner/AdminRequestsPerMinuteExceededException.java diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/AdminRequestsPerMinuteExceededException.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/AdminRequestsPerMinuteExceededException.java new file mode 100644 index 0000000000..11870c94d0 --- /dev/null +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/AdminRequestsPerMinuteExceededException.java @@ -0,0 +1,36 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import javax.annotation.Nullable; + +/** + * Exception thrown by Cloud Spanner the number of administrative requests per minute has been + * exceeded. + */ +public class AdminRequestsPerMinuteExceededException extends SpannerException { + private static final long serialVersionUID = -6395746612598975751L; + + static final String ADMIN_REQUESTS_LIMIT_KEY = "quota_limit"; + static final String ADMIN_REQUESTS_LIMIT_VALUE = "AdminMethodQuotaPerMinutePerProject"; + + /** Private constructor. Use {@link SpannerExceptionFactory} to create instances. */ + AdminRequestsPerMinuteExceededException( + DoNotConstructDirectly token, @Nullable String message, @Nullable Throwable cause) { + super(token, ErrorCode.RESOURCE_EXHAUSTED, true, message, cause); + } +} diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerExceptionFactory.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerExceptionFactory.java index 774aaf472e..706ee87fd7 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerExceptionFactory.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerExceptionFactory.java @@ -22,6 +22,7 @@ import com.google.cloud.spanner.SpannerException.DoNotConstructDirectly; import com.google.common.base.MoreObjects; import com.google.common.base.Predicate; +import com.google.rpc.ErrorInfo; import com.google.rpc.ResourceInfo; import io.grpc.Context; import io.grpc.Metadata; @@ -46,6 +47,8 @@ public final class SpannerExceptionFactory { "type.googleapis.com/google.spanner.admin.instance.v1.Instance"; private static final Metadata.Key KEY_RESOURCE_INFO = ProtoUtils.keyForProto(ResourceInfo.getDefaultInstance()); + private static final Metadata.Key KEY_ERROR_INFO = + ProtoUtils.keyForProto(ErrorInfo.getDefaultInstance()); public static SpannerException newSpannerException(ErrorCode code, @Nullable String message) { return newSpannerException(code, message, null); @@ -213,6 +216,16 @@ private static ResourceInfo extractResourceInfo(Throwable cause) { return null; } + private static ErrorInfo extractErrorInfo(Throwable cause) { + if (cause != null) { + Metadata trailers = Status.trailersFromThrowable(cause); + if (trailers != null) { + return trailers.get(KEY_ERROR_INFO); + } + } + return null; + } + static SpannerException newSpannerExceptionPreformatted( ErrorCode code, @Nullable String message, @Nullable Throwable cause) { // This is the one place in the codebase that is allowed to call constructors directly. @@ -220,6 +233,16 @@ static SpannerException newSpannerExceptionPreformatted( switch (code) { case ABORTED: return new AbortedException(token, message, cause); + case RESOURCE_EXHAUSTED: + ErrorInfo info = extractErrorInfo(cause); + if (info != null + && info.getMetadataMap() + .containsKey(AdminRequestsPerMinuteExceededException.ADMIN_REQUESTS_LIMIT_KEY) + && AdminRequestsPerMinuteExceededException.ADMIN_REQUESTS_LIMIT_VALUE.equals( + info.getMetadataMap() + .get(AdminRequestsPerMinuteExceededException.ADMIN_REQUESTS_LIMIT_KEY))) { + return new AdminRequestsPerMinuteExceededException(token, message, cause); + } case NOT_FOUND: ResourceInfo resourceInfo = extractResourceInfo(cause); if (resourceInfo != null) { diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java index 48160736a3..4d7d836dea 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java @@ -47,7 +47,9 @@ import com.google.api.gax.rpc.WatchdogProvider; import com.google.api.pathtemplate.PathTemplate; import com.google.cloud.RetryHelper; +import com.google.cloud.RetryHelper.RetryHelperException; import com.google.cloud.grpc.GrpcTransportOptions; +import com.google.cloud.spanner.AdminRequestsPerMinuteExceededException; import com.google.cloud.spanner.ErrorCode; import com.google.cloud.spanner.SpannerException; import com.google.cloud.spanner.SpannerExceptionFactory; @@ -483,6 +485,43 @@ private static void checkEmulatorConnection( } } + private static final RetrySettings ADMIN_REQUESTS_LIMIT_EXCEEDED_RETRY_SETTINGS = + RetrySettings.newBuilder() + .setInitialRetryDelay(Duration.ofSeconds(2L)) + .setRetryDelayMultiplier(1.5) + .setMaxRetryDelay(Duration.ofSeconds(15L)) + .setMaxAttempts(10) + .build(); + + @VisibleForTesting + static final class AdminRequestsLimitExceededRetryAlgorithm + implements ResultRetryAlgorithm { + @Override + public TimedAttemptSettings createNextAttempt( + Throwable prevThrowable, T prevResponse, TimedAttemptSettings prevSettings) { + // Use default retry settings. + return null; + } + + @Override + public boolean shouldRetry(Throwable prevThrowable, T prevResponse) + throws CancellationException { + return prevThrowable instanceof AdminRequestsPerMinuteExceededException; + } + } + + private static T runWithRetryOnAdministrativeRequestsExceeded(Callable callable) { + try { + return RetryHelper.runWithRetries( + callable, + ADMIN_REQUESTS_LIMIT_EXCEEDED_RETRY_SETTINGS, + new AdminRequestsLimitExceededRetryAlgorithm<>(), + NanoClock.getDefaultClock()); + } catch (RetryHelperException e) { + throw SpannerExceptionFactory.asSpannerException(e.getCause()); + } + } + private static final class OperationFutureRetryAlgorithm implements ResultRetryAlgorithm> { @@ -554,30 +593,39 @@ private final class OperationFutureCallable call() throws Exception { acquireAdministrativeRequestsRateLimiter(); - String operationName = null; - if (isRetry) { - // Query the backend to see if the operation was actually created, and that the - // problem was caused by a network problem or other transient problem client side. - Operation operation = mostRecentOperation(lister, getStartTimeFunction, initialCallTime); - if (operation != null) { - // Operation found, resume tracking that operation. - operationName = operation.getName(); - } - } else { - initialCallTime = - Timestamp.newBuilder() - .setSeconds( - TimeUnit.SECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS)) - .build(); - } - isRetry = true; - - if (operationName == null) { - GrpcCallContext context = newCallContext(null, instanceName, initialRequest, method); - return operationCallable.futureCall(initialRequest, context); - } else { - return operationCallable.resumeFutureCall(operationName); - } + return runWithRetryOnAdministrativeRequestsExceeded( + new Callable>() { + @Override + public OperationFuture call() throws Exception { + String operationName = null; + if (isRetry) { + // Query the backend to see if the operation was actually created, and that the + // problem was caused by a network problem or other transient problem client side. + Operation operation = + mostRecentOperation(lister, getStartTimeFunction, initialCallTime); + if (operation != null) { + // Operation found, resume tracking that operation. + operationName = operation.getName(); + } + } else { + initialCallTime = + Timestamp.newBuilder() + .setSeconds( + TimeUnit.SECONDS.convert( + System.currentTimeMillis(), TimeUnit.MILLISECONDS)) + .build(); + } + isRetry = true; + + if (operationName == null) { + GrpcCallContext context = + newCallContext(null, instanceName, initialRequest, method); + return operationCallable.futureCall(initialRequest, context); + } else { + return operationCallable.resumeFutureCall(operationName); + } + } + }); } } @@ -757,13 +805,20 @@ public Paginated listBackupOperations( if (pageToken != null) { requestBuilder.setPageToken(pageToken); } - ListBackupOperationsRequest request = requestBuilder.build(); + final ListBackupOperationsRequest request = requestBuilder.build(); - GrpcCallContext context = + final GrpcCallContext context = newCallContext( null, instanceName, request, DatabaseAdminGrpc.getListBackupOperationsMethod()); ListBackupOperationsResponse response = - get(databaseAdminStub.listBackupOperationsCallable().futureCall(request, context)); + runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public ListBackupOperationsResponse call() throws Exception { + return get( + databaseAdminStub.listBackupOperationsCallable().futureCall(request, context)); + } + }); return new Paginated<>(response.getOperationsList(), response.getNextPageToken()); } @@ -780,13 +835,23 @@ public Paginated listDatabaseOperations( if (pageToken != null) { requestBuilder.setPageToken(pageToken); } - ListDatabaseOperationsRequest request = requestBuilder.build(); + final ListDatabaseOperationsRequest request = requestBuilder.build(); - GrpcCallContext context = + final GrpcCallContext context = newCallContext( null, instanceName, request, DatabaseAdminGrpc.getListDatabaseOperationsMethod()); ListDatabaseOperationsResponse response = - get(databaseAdminStub.listDatabaseOperationsCallable().futureCall(request, context)); + runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public ListDatabaseOperationsResponse call() throws Exception { + return get( + databaseAdminStub + .listDatabaseOperationsCallable() + .futureCall(request, context)); + } + }); + return new Paginated<>(response.getOperationsList(), response.getNextPageToken()); } @@ -803,12 +868,19 @@ public Paginated listBackups( if (pageToken != null) { requestBuilder.setPageToken(pageToken); } - ListBackupsRequest request = requestBuilder.build(); + final ListBackupsRequest request = requestBuilder.build(); - GrpcCallContext context = + final GrpcCallContext context = newCallContext(null, instanceName, request, DatabaseAdminGrpc.getListBackupsMethod()); ListBackupsResponse response = - get(databaseAdminStub.listBackupsCallable().futureCall(request, context)); + runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public ListBackupsResponse call() throws Exception { + return get(databaseAdminStub.listBackupsCallable().futureCall(request, context)); + } + }); + return new Paginated<>(response.getBackupsList(), response.getNextPageToken()); } @@ -821,12 +893,19 @@ public Paginated listDatabases( if (pageToken != null) { requestBuilder.setPageToken(pageToken); } - ListDatabasesRequest request = requestBuilder.build(); + final ListDatabasesRequest request = requestBuilder.build(); - GrpcCallContext context = + final GrpcCallContext context = newCallContext(null, instanceName, request, DatabaseAdminGrpc.getListDatabasesMethod()); ListDatabasesResponse response = - get(databaseAdminStub.listDatabasesCallable().futureCall(request, context)); + runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public ListDatabasesResponse call() throws Exception { + return get(databaseAdminStub.listDatabasesCallable().futureCall(request, context)); + } + }); + return new Paginated<>(response.getDatabasesList(), response.getNextPageToken()); } @@ -897,67 +976,97 @@ public Timestamp apply(Operation input) { @Override public OperationFuture updateDatabaseDdl( - String databaseName, Iterable updateDatabaseStatements, @Nullable String updateId) + final String databaseName, + final Iterable updateDatabaseStatements, + @Nullable final String updateId) throws SpannerException { acquireAdministrativeRequestsRateLimiter(); - UpdateDatabaseDdlRequest request = + final UpdateDatabaseDdlRequest request = UpdateDatabaseDdlRequest.newBuilder() .setDatabase(databaseName) .addAllStatements(updateDatabaseStatements) .setOperationId(MoreObjects.firstNonNull(updateId, "")) .build(); - GrpcCallContext context = + final GrpcCallContext context = newCallContext(null, databaseName, request, DatabaseAdminGrpc.getUpdateDatabaseDdlMethod()); - OperationCallable callable = + final OperationCallable callable = databaseAdminStub.updateDatabaseDdlOperationCallable(); - OperationFuture operationFuture = - callable.futureCall(request, context); - try { - operationFuture.getInitialFuture().get(); - } catch (InterruptedException e) { - throw newSpannerException(e); - } catch (ExecutionException e) { - Throwable t = e.getCause(); - if (t instanceof AlreadyExistsException) { - String operationName = - OPERATION_NAME_TEMPLATE.instantiate("database", databaseName, "operation", updateId); - return callable.resumeFutureCall(operationName, context); - } - } - return operationFuture; + + return runWithRetryOnAdministrativeRequestsExceeded( + new Callable>() { + @Override + public OperationFuture call() throws Exception { + OperationFuture operationFuture = + callable.futureCall(request, context); + try { + operationFuture.getInitialFuture().get(); + } catch (InterruptedException e) { + throw newSpannerException(e); + } catch (ExecutionException e) { + Throwable t = e.getCause(); + if (t instanceof AlreadyExistsException) { + String operationName = + OPERATION_NAME_TEMPLATE.instantiate( + "database", databaseName, "operation", updateId); + return callable.resumeFutureCall(operationName, context); + } + } + return operationFuture; + } + }); } @Override public void dropDatabase(String databaseName) throws SpannerException { acquireAdministrativeRequestsRateLimiter(); - DropDatabaseRequest request = + final DropDatabaseRequest request = DropDatabaseRequest.newBuilder().setDatabase(databaseName).build(); - GrpcCallContext context = + final GrpcCallContext context = newCallContext(null, databaseName, request, DatabaseAdminGrpc.getDropDatabaseMethod()); - get(databaseAdminStub.dropDatabaseCallable().futureCall(request, context)); + runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public Void call() throws Exception { + get(databaseAdminStub.dropDatabaseCallable().futureCall(request, context)); + return null; + } + }); } @Override public Database getDatabase(String databaseName) throws SpannerException { acquireAdministrativeRequestsRateLimiter(); - GetDatabaseRequest request = GetDatabaseRequest.newBuilder().setName(databaseName).build(); + final GetDatabaseRequest request = + GetDatabaseRequest.newBuilder().setName(databaseName).build(); - GrpcCallContext context = + final GrpcCallContext context = newCallContext(null, databaseName, request, DatabaseAdminGrpc.getGetDatabaseMethod()); - return get(databaseAdminStub.getDatabaseCallable().futureCall(request, context)); + return runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public Database call() throws Exception { + return get(databaseAdminStub.getDatabaseCallable().futureCall(request, context)); + } + }); } @Override public List getDatabaseDdl(String databaseName) throws SpannerException { acquireAdministrativeRequestsRateLimiter(); - GetDatabaseDdlRequest request = + final GetDatabaseDdlRequest request = GetDatabaseDdlRequest.newBuilder().setDatabase(databaseName).build(); - GrpcCallContext context = + final GrpcCallContext context = newCallContext(null, databaseName, request, DatabaseAdminGrpc.getGetDatabaseDdlMethod()); - return get(databaseAdminStub.getDatabaseDdlCallable().futureCall(request, context)) - .getStatementsList(); + return runWithRetryOnAdministrativeRequestsExceeded( + new Callable>() { + @Override + public List call() throws Exception { + return get(databaseAdminStub.getDatabaseDdlCallable().futureCall(request, context)) + .getStatementsList(); + } + }); } @Override @@ -1069,52 +1178,89 @@ public Timestamp apply(Operation input) { @Override public Backup updateBackup(Backup backup, FieldMask updateMask) { acquireAdministrativeRequestsRateLimiter(); - UpdateBackupRequest request = + final UpdateBackupRequest request = UpdateBackupRequest.newBuilder().setBackup(backup).setUpdateMask(updateMask).build(); - GrpcCallContext context = + final GrpcCallContext context = newCallContext(null, backup.getName(), request, DatabaseAdminGrpc.getUpdateBackupMethod()); - return databaseAdminStub.updateBackupCallable().call(request, context); + return runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public Backup call() throws Exception { + return databaseAdminStub.updateBackupCallable().call(request, context); + } + }); } @Override public void deleteBackup(String backupName) { acquireAdministrativeRequestsRateLimiter(); - DeleteBackupRequest request = DeleteBackupRequest.newBuilder().setName(backupName).build(); - GrpcCallContext context = + final DeleteBackupRequest request = + DeleteBackupRequest.newBuilder().setName(backupName).build(); + final GrpcCallContext context = newCallContext(null, backupName, request, DatabaseAdminGrpc.getDeleteBackupMethod()); - databaseAdminStub.deleteBackupCallable().call(request, context); + runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public Void call() throws Exception { + databaseAdminStub.deleteBackupCallable().call(request, context); + return null; + } + }); } @Override public Backup getBackup(String backupName) throws SpannerException { acquireAdministrativeRequestsRateLimiter(); - GetBackupRequest request = GetBackupRequest.newBuilder().setName(backupName).build(); - GrpcCallContext context = + final GetBackupRequest request = GetBackupRequest.newBuilder().setName(backupName).build(); + final GrpcCallContext context = newCallContext(null, backupName, request, DatabaseAdminGrpc.getGetBackupMethod()); - return get(databaseAdminStub.getBackupCallable().futureCall(request, context)); + return runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public Backup call() throws Exception { + return get(databaseAdminStub.getBackupCallable().futureCall(request, context)); + } + }); } @Override public Operation getOperation(String name) throws SpannerException { acquireAdministrativeRequestsRateLimiter(); - GetOperationRequest request = GetOperationRequest.newBuilder().setName(name).build(); - GrpcCallContext context = + final GetOperationRequest request = GetOperationRequest.newBuilder().setName(name).build(); + final GrpcCallContext context = newCallContext(null, name, request, OperationsGrpc.getGetOperationMethod()); - return get( - databaseAdminStub.getOperationsStub().getOperationCallable().futureCall(request, context)); + return runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public Operation call() throws Exception { + return get( + databaseAdminStub + .getOperationsStub() + .getOperationCallable() + .futureCall(request, context)); + } + }); } @Override public void cancelOperation(String name) throws SpannerException { acquireAdministrativeRequestsRateLimiter(); - CancelOperationRequest request = CancelOperationRequest.newBuilder().setName(name).build(); - GrpcCallContext context = + final CancelOperationRequest request = + CancelOperationRequest.newBuilder().setName(name).build(); + final GrpcCallContext context = newCallContext(null, name, request, OperationsGrpc.getCancelOperationMethod()); - get( - databaseAdminStub - .getOperationsStub() - .cancelOperationCallable() - .futureCall(request, context)); + runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public Void call() throws Exception { + get( + databaseAdminStub + .getOperationsStub() + .cancelOperationCallable() + .futureCall(request, context)); + return null; + } + }); } @Override @@ -1331,67 +1477,105 @@ public PartitionResponse partitionRead( @Override public Policy getDatabaseAdminIAMPolicy(String resource) { acquireAdministrativeRequestsRateLimiter(); - GetIamPolicyRequest request = GetIamPolicyRequest.newBuilder().setResource(resource).build(); - GrpcCallContext context = + final GetIamPolicyRequest request = + GetIamPolicyRequest.newBuilder().setResource(resource).build(); + final GrpcCallContext context = newCallContext(null, resource, request, DatabaseAdminGrpc.getGetIamPolicyMethod()); - return get(databaseAdminStub.getIamPolicyCallable().futureCall(request, context)); + return runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public Policy call() throws Exception { + return get(databaseAdminStub.getIamPolicyCallable().futureCall(request, context)); + } + }); } @Override public Policy setDatabaseAdminIAMPolicy(String resource, Policy policy) { acquireAdministrativeRequestsRateLimiter(); - SetIamPolicyRequest request = + final SetIamPolicyRequest request = SetIamPolicyRequest.newBuilder().setResource(resource).setPolicy(policy).build(); - GrpcCallContext context = + final GrpcCallContext context = newCallContext(null, resource, request, DatabaseAdminGrpc.getSetIamPolicyMethod()); - return get(databaseAdminStub.setIamPolicyCallable().futureCall(request, context)); + return runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public Policy call() throws Exception { + return get(databaseAdminStub.setIamPolicyCallable().futureCall(request, context)); + } + }); } @Override public TestIamPermissionsResponse testDatabaseAdminIAMPermissions( String resource, Iterable permissions) { acquireAdministrativeRequestsRateLimiter(); - TestIamPermissionsRequest request = + final TestIamPermissionsRequest request = TestIamPermissionsRequest.newBuilder() .setResource(resource) .addAllPermissions(permissions) .build(); - GrpcCallContext context = + final GrpcCallContext context = newCallContext(null, resource, request, DatabaseAdminGrpc.getTestIamPermissionsMethod()); - return get(databaseAdminStub.testIamPermissionsCallable().futureCall(request, context)); + return runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public TestIamPermissionsResponse call() throws Exception { + return get(databaseAdminStub.testIamPermissionsCallable().futureCall(request, context)); + } + }); } @Override public Policy getInstanceAdminIAMPolicy(String resource) { acquireAdministrativeRequestsRateLimiter(); - GetIamPolicyRequest request = GetIamPolicyRequest.newBuilder().setResource(resource).build(); - GrpcCallContext context = + final GetIamPolicyRequest request = + GetIamPolicyRequest.newBuilder().setResource(resource).build(); + final GrpcCallContext context = newCallContext(null, resource, request, InstanceAdminGrpc.getGetIamPolicyMethod()); - return get(instanceAdminStub.getIamPolicyCallable().futureCall(request, context)); + return runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public Policy call() throws Exception { + return get(instanceAdminStub.getIamPolicyCallable().futureCall(request, context)); + } + }); } @Override public Policy setInstanceAdminIAMPolicy(String resource, Policy policy) { acquireAdministrativeRequestsRateLimiter(); - SetIamPolicyRequest request = + final SetIamPolicyRequest request = SetIamPolicyRequest.newBuilder().setResource(resource).setPolicy(policy).build(); - GrpcCallContext context = + final GrpcCallContext context = newCallContext(null, resource, request, InstanceAdminGrpc.getSetIamPolicyMethod()); - return get(instanceAdminStub.setIamPolicyCallable().futureCall(request, context)); + return runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public Policy call() throws Exception { + return get(instanceAdminStub.setIamPolicyCallable().futureCall(request, context)); + } + }); } @Override public TestIamPermissionsResponse testInstanceAdminIAMPermissions( String resource, Iterable permissions) { acquireAdministrativeRequestsRateLimiter(); - TestIamPermissionsRequest request = + final TestIamPermissionsRequest request = TestIamPermissionsRequest.newBuilder() .setResource(resource) .addAllPermissions(permissions) .build(); - GrpcCallContext context = + final GrpcCallContext context = newCallContext(null, resource, request, InstanceAdminGrpc.getTestIamPermissionsMethod()); - return get(instanceAdminStub.testIamPermissionsCallable().futureCall(request, context)); + return runWithRetryOnAdministrativeRequestsExceeded( + new Callable() { + @Override + public TestIamPermissionsResponse call() throws Exception { + return get(instanceAdminStub.testIamPermissionsCallable().futureCall(request, context)); + } + }); } /** Gets the result of an async RPC call, handling any exceptions encountered. */ diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java index 3e1585a658..84aaa91bcf 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpcTest.java @@ -38,6 +38,7 @@ import com.google.cloud.spanner.ResultSet; import com.google.cloud.spanner.Spanner; import com.google.cloud.spanner.SpannerException; +import com.google.cloud.spanner.SpannerExceptionFactory; import com.google.cloud.spanner.SpannerOptions; import com.google.cloud.spanner.SpannerOptions.CallContextConfigurator; import com.google.cloud.spanner.SpannerOptions.CallCredentialsProvider; @@ -46,9 +47,11 @@ import com.google.cloud.spanner.TransactionRunner.TransactionCallable; import com.google.cloud.spanner.admin.database.v1.MockDatabaseAdminImpl; import com.google.cloud.spanner.admin.instance.v1.MockInstanceAdminImpl; +import com.google.cloud.spanner.spi.v1.GapicSpannerRpc.AdminRequestsLimitExceededRetryAlgorithm; import com.google.cloud.spanner.spi.v1.SpannerRpc.Option; import com.google.common.base.Stopwatch; import com.google.protobuf.ListValue; +import com.google.rpc.ErrorInfo; import com.google.spanner.admin.database.v1.Database; import com.google.spanner.admin.database.v1.DatabaseName; import com.google.spanner.admin.instance.v1.Instance; @@ -72,8 +75,10 @@ import io.grpc.ServerCall; import io.grpc.ServerCallHandler; import io.grpc.ServerInterceptor; +import io.grpc.Status; import io.grpc.auth.MoreCallCredentials; import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; +import io.grpc.protobuf.lite.ProtoLiteUtils; import java.io.IOException; import java.net.InetSocketAddress; import java.util.ArrayList; @@ -464,6 +469,39 @@ public void testNewCallContextWithNullRequestAndNullMethod() { rpc.shutdown(); } + @Test + public void testAdminRequestsLimitExceededRetryAlgorithm() { + AdminRequestsLimitExceededRetryAlgorithm alg = + new AdminRequestsLimitExceededRetryAlgorithm<>(); + + assertThat(alg.shouldRetry(null, 1L)).isFalse(); + + ErrorInfo info = + ErrorInfo.newBuilder() + .putMetadata("quota_limit", "AdminMethodQuotaPerMinutePerProject") + .build(); + Metadata.Key key = + Metadata.Key.of( + info.getDescriptorForType().getFullName() + Metadata.BINARY_HEADER_SUFFIX, + ProtoLiteUtils.metadataMarshaller(info)); + Metadata trailers = new Metadata(); + trailers.put(key, info); + + SpannerException adminRateExceeded = + SpannerExceptionFactory.newSpannerException( + Status.RESOURCE_EXHAUSTED.withDescription("foo").asRuntimeException(trailers)); + assertThat(alg.shouldRetry(adminRateExceeded, null)).isTrue(); + + SpannerException numDatabasesExceeded = + SpannerExceptionFactory.newSpannerException( + Status.RESOURCE_EXHAUSTED + .withDescription("Too many databases on instance") + .asRuntimeException()); + assertThat(alg.shouldRetry(numDatabasesExceeded, null)).isFalse(); + + assertThat(alg.shouldRetry(new Exception("random exception"), null)).isFalse(); + } + @SuppressWarnings("rawtypes") private SpannerOptions createSpannerOptions() { String endpoint = address.getHostString() + ":" + server.getPort();