feat: retry admin request limit exceeded error

Automatically retry requests that fail because the admin requests per seconds
limit has been exceeded using an exponential backoff.

Fixes #655 and others

google-cloud-spanner/src/main/java/com/google/cloud/spanner/AdminRequestsPerMinuteExceededException.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.spanner;
+
+import javax.annotation.Nullable;
+
+/**
+ * Exception thrown by Cloud Spanner the number of administrative requests per minute has been
+ * exceeded.
+ */
+public class AdminRequestsPerMinuteExceededException extends SpannerException {
+  private static final long serialVersionUID = -6395746612598975751L;
+
+  /** Private constructor. Use {@link SpannerExceptionFactory} to create instances. */
+  AdminRequestsPerMinuteExceededException(
+      DoNotConstructDirectly token, @Nullable String message, @Nullable Throwable cause) {
+    super(token, ErrorCode.RESOURCE_EXHAUSTED, true, message, cause);
+  }
+}

google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerExceptionFactory.java

@@ -22,6 +22,7 @@
 import com.google.cloud.spanner.SpannerException.DoNotConstructDirectly;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Predicate;
+import com.google.rpc.ErrorInfo;
 import com.google.rpc.ResourceInfo;
 import io.grpc.Context;
 import io.grpc.Metadata;
@@ -46,6 +47,8 @@ public final class SpannerExceptionFactory {
       "type.googleapis.com/google.spanner.admin.instance.v1.Instance";
   private static final Metadata.Key<ResourceInfo> KEY_RESOURCE_INFO =
       ProtoUtils.keyForProto(ResourceInfo.getDefaultInstance());
+  private static final Metadata.Key<ErrorInfo> KEY_ERROR_INFO =
+      ProtoUtils.keyForProto(ErrorInfo.getDefaultInstance());
 
   public static SpannerException newSpannerException(ErrorCode code, @Nullable String message) {
     return newSpannerException(code, message, null);
@@ -213,13 +216,31 @@ private static ResourceInfo extractResourceInfo(Throwable cause) {
     return null;
   }
 
+  private static ErrorInfo extractErrorInfo(Throwable cause) {
+    if (cause != null) {
+      Metadata trailers = Status.trailersFromThrowable(cause);
+      if (trailers != null) {
+        return trailers.get(KEY_ERROR_INFO);
+      }
+    }
+    return null;
+  }
+
   static SpannerException newSpannerExceptionPreformatted(
       ErrorCode code, @Nullable String message, @Nullable Throwable cause) {
     // This is the one place in the codebase that is allowed to call constructors directly.
     DoNotConstructDirectly token = DoNotConstructDirectly.ALLOWED;
     switch (code) {
       case ABORTED:
         return new AbortedException(token, message, cause);
+      case RESOURCE_EXHAUSTED:
+        ErrorInfo info = extractErrorInfo(cause);
+        if (info != null
+            && info.getMetadataMap().containsKey("quota_limit")
+            && "AdminMethodQuotaPerMinutePerProject"
+                .equals(info.getMetadataMap().get("quota_limit"))) {
+          return new AdminRequestsPerMinuteExceededException(token, message, cause);
+        }
       case NOT_FOUND:
         ResourceInfo resourceInfo = extractResourceInfo(cause);
         if (resourceInfo != null) {