feat: managed backups implementation (#2643)

* feat: managed backups

* Revert "chore(cleanup): Marking snapshot operations as unsupported (#2467)"

This reverts commit 4494e7a

for tests

* cherry pick remaining files in #2592

* fix deps build

* cleanup

* formatting

* fix test

* refactor and update tests

* cleanup unused api

* fix test

* formatting

* review feedback

* add create backup test

* clean up deps, remove client core dep in hbase-1x

* test update

* cleanup

* cleanup

Author: kolea2

bigtable-client-core-parent/bigtable-client-core/src/main/java/com/google/cloud/bigtable/grpc/BigtableClusterName.java

@@ -79,15 +79,15 @@ public String getClusterId() {
   }
 
   /**
-   * Create a fully qualified snapshot name based on the the clusterName and the snapshotId.
-   * Snapshot name will look like:
-   * "projects/{projectId}/instances/{instanceId}/clusters/{clusterId}/snapshots/{snapshotId}".
+   * Create a fully qualified backup name based on the the clusterName and the backupId. Backup name
+   * will look like:
+   * "projects/{projectId}/instances/{instanceId}/clusters/{clusterId}/backups/{backupId}".
    *
-   * @param snapshotId The id of the snapshot
-   * @return A fully qualified snapshot name that contains the fully qualified cluster name as the
-   *     parent and the snapshot name as the child.
+   * @param backupId The id of the backup
+   * @return A fully qualified backup name that contains the fully qualified cluster name as the
+   *     parent and the backup name as the child.
    */
-  public String toSnapshotName(String snapshotId) {
-    return clusterName + "/snapshots/" + snapshotId;
+  public String toBackupName(String backupId) {
+    return clusterName + "/backups/" + backupId;
   }
 }

bigtable-client-core-parent/bigtable-client-core/src/main/java/com/google/cloud/bigtable/grpc/BigtableInstanceGrpcClient.java

@@ -15,8 +15,6 @@
  */
 package com.google.cloud.bigtable.grpc;
 
-import com.google.api.client.util.BackOff;
-import com.google.api.client.util.ExponentialBackOff;
 import com.google.api.core.InternalApi;
 import com.google.bigtable.admin.v2.AppProfile;
 import com.google.bigtable.admin.v2.BigtableInstanceAdminGrpc;
@@ -38,7 +36,7 @@
 import com.google.bigtable.admin.v2.ListInstancesResponse;
 import com.google.bigtable.admin.v2.PartialUpdateInstanceRequest;
 import com.google.bigtable.admin.v2.UpdateAppProfileRequest;
-import com.google.common.primitives.Ints;
+import com.google.cloud.bigtable.util.OperationUtil;
 import com.google.iam.v1.GetIamPolicyRequest;
 import com.google.iam.v1.Policy;
 import com.google.iam.v1.SetIamPolicyRequest;
@@ -49,7 +47,6 @@
 import com.google.longrunning.OperationsGrpc;
 import com.google.protobuf.Empty;
 import io.grpc.Channel;
-import io.grpc.protobuf.StatusProto;
 import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
@@ -63,7 +60,7 @@
 public class BigtableInstanceGrpcClient implements BigtableInstanceClient {
 
   private final BigtableInstanceAdminGrpc.BigtableInstanceAdminBlockingStub instanceClient;
-  private final OperationsGrpc.OperationsBlockingStub operationsStub;
+  private final OperationUtil operationUtil;
 
   /**
    * Constructor for BigtableInstanceGrpcClient.
@@ -72,7 +69,7 @@ public class BigtableInstanceGrpcClient implements BigtableInstanceClient {
    */
   public BigtableInstanceGrpcClient(Channel channel) {
     this.instanceClient = BigtableInstanceAdminGrpc.newBlockingStub(channel);
-    operationsStub = OperationsGrpc.newBlockingStub(channel);
+    operationUtil = new OperationUtil(OperationsGrpc.newBlockingStub(channel));
   }
 
   /** {@inheritDoc} */
@@ -84,65 +81,20 @@ public Operation createInstance(CreateInstanceRequest request) {
   /** {@inheritDoc} */
   @Override
   public Operation getOperation(GetOperationRequest request) {
-    return operationsStub.getOperation(request);
+    return operationUtil.getOperation(request);
   }
 
   /** {@inheritDoc} */
   @Override
   public void waitForOperation(Operation operation) throws IOException, TimeoutException {
-    waitForOperation(operation, 10, TimeUnit.MINUTES);
+    operationUtil.waitForOperation(operation, 10, TimeUnit.MINUTES);
   }
 
   /** {@inheritDoc} */
   @Override
   public void waitForOperation(Operation operation, long timeout, TimeUnit timeUnit)
       throws TimeoutException, IOException {
-    GetOperationRequest request =
-        GetOperationRequest.newBuilder().setName(operation.getName()).build();
-
-    ExponentialBackOff backOff =
-        new ExponentialBackOff.Builder()
-            .setInitialIntervalMillis(100)
-            .setMultiplier(1.3)
-            .setMaxIntervalMillis(Ints.checkedCast(TimeUnit.SECONDS.toMillis(60)))
-            .setMaxElapsedTimeMillis(Ints.checkedCast(timeUnit.toMillis(timeout)))
-            .build();
-
-    Operation currentOperationState = operation;
-
-    while (true) {
-      if (currentOperationState.getDone()) {
-        switch (currentOperationState.getResultCase()) {
-          case RESPONSE:
-            return;
-          case ERROR:
-            throw StatusProto.toStatusRuntimeException(currentOperationState.getError());
-          case RESULT_NOT_SET:
-            throw new IllegalStateException(
-                "System returned invalid response for Operation check: " + currentOperationState);
-        }
-      }
-
-      final long backOffMillis;
-      try {
-        backOffMillis = backOff.nextBackOffMillis();
-      } catch (IOException e) {
-        // Should never happen.
-        throw new RuntimeException(e);
-      }
-      if (backOffMillis == BackOff.STOP) {
-        throw new TimeoutException("Operation did not complete in time");
-      } else {
-        try {
-          Thread.sleep(backOffMillis);
-        } catch (InterruptedException e) {
-          Thread.currentThread().interrupt();
-          throw new IOException("Interrupted while waiting for operation to finish");
-        }
-      }
-
-      currentOperationState = getOperation(request);
-    }
+    operationUtil.waitForOperation(operation, timeout, timeUnit);
   }
 
   /** {@inheritDoc} */

bigtable-client-core-parent/bigtable-client-core/src/main/java/com/google/cloud/bigtable/grpc/BigtableSessionSharedThreadPools.java

@@ -58,6 +58,9 @@ public static BigtableSessionSharedThreadPools getInstance() {
   /**
    * Getter for the field <code>batchThreadPool</code>.
    *
+   * <p>This needs to be a cached thread pool because long running operations could tie up the
+   * thread pool indefinitely
+   *
    * @return a {@link java.util.concurrent.ExecutorService} object.
    */
   public synchronized ExecutorService getBatchThreadPool() {

bigtable-client-core-parent/bigtable-client-core/src/main/java/com/google/cloud/bigtable/grpc/BigtableTableAdminClient.java

@@ -16,29 +16,31 @@
 package com.google.cloud.bigtable.grpc;
 
 import com.google.api.core.InternalExtensionOnly;
-import com.google.bigtable.admin.v2.CreateTableFromSnapshotRequest;
+import com.google.bigtable.admin.v2.CreateBackupRequest;
 import com.google.bigtable.admin.v2.CreateTableRequest;
-import com.google.bigtable.admin.v2.DeleteSnapshotRequest;
+import com.google.bigtable.admin.v2.DeleteBackupRequest;
 import com.google.bigtable.admin.v2.DeleteTableRequest;
 import com.google.bigtable.admin.v2.DropRowRangeRequest;
-import com.google.bigtable.admin.v2.GetSnapshotRequest;
+import com.google.bigtable.admin.v2.GetBackupRequest;
 import com.google.bigtable.admin.v2.GetTableRequest;
-import com.google.bigtable.admin.v2.ListSnapshotsRequest;
-import com.google.bigtable.admin.v2.ListSnapshotsResponse;
+import com.google.bigtable.admin.v2.ListBackupsRequest;
+import com.google.bigtable.admin.v2.ListBackupsResponse;
 import com.google.bigtable.admin.v2.ListTablesRequest;
 import com.google.bigtable.admin.v2.ListTablesResponse;
 import com.google.bigtable.admin.v2.ModifyColumnFamiliesRequest;
-import com.google.bigtable.admin.v2.Snapshot;
-import com.google.bigtable.admin.v2.SnapshotTableRequest;
+import com.google.bigtable.admin.v2.RestoreTableRequest;
 import com.google.bigtable.admin.v2.Table;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.iam.v1.GetIamPolicyRequest;
 import com.google.iam.v1.Policy;
 import com.google.iam.v1.SetIamPolicyRequest;
 import com.google.iam.v1.TestIamPermissionsRequest;
 import com.google.iam.v1.TestIamPermissionsResponse;
+import com.google.longrunning.GetOperationRequest;
 import com.google.longrunning.Operation;
 import com.google.protobuf.Empty;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
 /** A client for the Cloud Bigtable Table Admin API. */
@@ -183,24 +185,70 @@ void waitForReplication(BigtableTableName tableName, long timeout)
    */
   TestIamPermissionsResponse testIamPermissions(TestIamPermissionsRequest request);
 
-  // ////////////// SNAPSHOT methods /////////////
-  /** @deprecated Snapshots will be removed in the future */
-  @Deprecated
-  ListenableFuture<Operation> snapshotTableAsync(SnapshotTableRequest request);
+  /**
+   * Creates a new backup from a table in a specific cluster.
+   *
+   * @param request a {@link com.google.bigtable.admin.v2.CreateBackupRequest} object.
+   * @return The long running {@link Operation} for the request.
+   */
+  ListenableFuture<Operation> createBackupAsync(
+      com.google.bigtable.admin.v2.CreateBackupRequest request);
+
+  /**
+   * Lists all backups associated with the specified cluster.
+   *
+   * @param request a {@link GetBackupRequest} object.
+   * @return The {@link ListBackupsResponse} which has the list of the backups in the cluster.
+   */
+  ListenableFuture<ListBackupsResponse> listBackupsAsync(ListBackupsRequest request);
+
+  /**
+   * Permanently deletes the specified backup.
+   *
+   * @param request a {@link DeleteBackupRequest} object.
+   */
+  ListenableFuture<Empty> deleteBackupAsync(DeleteBackupRequest request);
 
-  /** @deprecated Snapshots will be removed in the future */
-  @Deprecated
-  ListenableFuture<Snapshot> getSnapshotAsync(GetSnapshotRequest request);
+  /**
+   * Creates a new table from a backup.
+   *
+   * @param request a {@link com.google.bigtable.admin.v2.RestoreTableRequest} object.
+   * @return The long running {@link Operation} for the request.
+   */
+  ListenableFuture<Operation> restoreTableAsync(
+      com.google.bigtable.admin.v2.RestoreTableRequest request);
 
-  /** @deprecated Snapshots will be removed in the future */
-  @Deprecated
-  ListenableFuture<ListSnapshotsResponse> listSnapshotsAsync(ListSnapshotsRequest request);
+  /**
+   * Gets the latest state of a long-running operation. Clients may use this method to poll the
+   * operation result at intervals as recommended by the API service.
+   *
+   * <p>{@link #createBackupAsync(CreateBackupRequest)} and {@link
+   * #restoreTableAsync(RestoreTableRequest)} will return a {@link
+   * com.google.longrunning.Operation}. Use this method and pass in the {@link
+   * com.google.longrunning.Operation}'s name in the request to see if the Operation is done via
+   * {@link com.google.longrunning.Operation#getDone()}. The backup will not be available until that
+   * happens.
+   *
+   * @param request a {@link com.google.longrunning.GetOperationRequest} object.
+   * @return a {@link com.google.longrunning.Operation} object.
+   */
+  Operation getOperation(GetOperationRequest request);
 
-  /** @deprecated Snapshots will be removed in the future */
-  @Deprecated
-  ListenableFuture<Empty> deleteSnapshotAsync(DeleteSnapshotRequest request);
+  /**
+   * Waits for the long running operation to complete by polling with exponential backoff. A default
+   * timeout of 10 minutes is used.
+   *
+   * @throws IOException
+   * @throws TimeoutException If the timeout is exceeded.
+   */
+  Operation waitForOperation(Operation operation) throws TimeoutException, IOException;
 
-  /** @deprecated Snapshots will be removed in the future */
-  @Deprecated
-  ListenableFuture<Operation> createTableFromSnapshotAsync(CreateTableFromSnapshotRequest request);
+  /**
+   * Waits for the long running operation to complete by polling with exponential backoff.
+   *
+   * @throws IOException
+   * @throws TimeoutException If the timeout is exceeded.
+   */
+  Operation waitForOperation(Operation operation, long timeout, TimeUnit timeUnit)
+      throws IOException, TimeoutException;
 }