Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: allow restore backup to different instance #515

Merged
merged 4 commits into from Apr 30, 2021
Merged

feat: allow restore backup to different instance #515

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
042ed76
feat: allow restore backup to different instance
Nov 5, 2020
ef66710
Address comments in the PR.
Apr 27, 2021
8495075
Merge branch 'master' of https://github.com/googleapis/java-bigtable
Apr 29, 2021
b168074
Improve unit test coverage for RestoreTableRequest.
Apr 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
55 changes: 42 additions & 13 deletions ...bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/RestoreTableRequest.java
View file
Open in desktop
Expand Up @@ -20,24 +20,46 @@
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;

/** Fluent wrapper for {@link com.google.bigtable.admin.v2.RestoreTableRequest} */
public final class RestoreTableRequest {
private final com.google.bigtable.admin.v2.RestoreTableRequest.Builder requestBuilder =
com.google.bigtable.admin.v2.RestoreTableRequest.newBuilder();
private final String backupId;
private final String clusterId;
private final String sourceBackupId;
private final String sourceClusterId;
private final String sourceInstanceId;

public static RestoreTableRequest of(String clusterId, String backupId) {
RestoreTableRequest request = new RestoreTableRequest(clusterId, backupId);
/**
* Create a {@link RestoreTableRequest} object. It assumes the source backup locates in the same
* instance as the destination table. To restore a table from a backup in another instance, use
* {@link #of(String, String, String) of} method.
*/
public static RestoreTableRequest of(String sourceClusterId, String sourceBackupId) {
RestoreTableRequest request = new RestoreTableRequest(null, sourceClusterId, sourceBackupId);
return request;
}

private RestoreTableRequest(String clusterId, String backupId) {
Preconditions.checkNotNull(clusterId);
Preconditions.checkNotNull(backupId);
this.backupId = backupId;
this.clusterId = clusterId;
/**
* Create a {@link RestoreTableRequest} object. The source backup could locate in a the same or a
* different instance.
*/
public static RestoreTableRequest of(
String sourceInstanceId, String sourceClusterId, String sourceBackupId) {
kolea2 marked this conversation as resolved.
Show resolved Hide resolved
liubonan marked this conversation as resolved.
Show resolved Hide resolved
RestoreTableRequest request =
new RestoreTableRequest(sourceInstanceId, sourceClusterId, sourceBackupId);
return request;
}

private RestoreTableRequest(
@Nullable String sourceInstanceId,
@Nonnull String sourceClusterId,
@Nonnull String sourceBackupId) {
Preconditions.checkNotNull(sourceClusterId);
Preconditions.checkNotNull(sourceBackupId);
this.sourceBackupId = sourceBackupId;
this.sourceInstanceId = sourceInstanceId;
this.sourceClusterId = sourceClusterId;
}

public RestoreTableRequest setTableId(String tableId) {
Expand All @@ -56,13 +78,15 @@ public boolean equals(Object o) {
}
RestoreTableRequest that = (RestoreTableRequest) o;
return Objects.equal(requestBuilder.getTableId(), that.requestBuilder.getTableId())
&& Objects.equal(clusterId, that.clusterId)
&& Objects.equal(backupId, that.backupId);
&& Objects.equal(sourceInstanceId, that.sourceInstanceId)
&& Objects.equal(sourceClusterId, that.sourceClusterId)
&& Objects.equal(sourceBackupId, that.sourceBackupId);
}

@Override
public int hashCode() {
return Objects.hashCode(requestBuilder.getTableId(), clusterId, backupId);
return Objects.hashCode(
requestBuilder.getTableId(), sourceInstanceId, sourceClusterId, sourceBackupId);
}

@InternalApi
Expand All @@ -73,7 +97,12 @@ public com.google.bigtable.admin.v2.RestoreTableRequest toProto(

return requestBuilder
.setParent(NameUtil.formatInstanceName(projectId, instanceId))
.setBackup(NameUtil.formatBackupName(projectId, instanceId, clusterId, backupId))
.setBackup(
NameUtil.formatBackupName(
projectId,
sourceInstanceId == null ? instanceId : sourceInstanceId,
sourceClusterId,
sourceBackupId))
.build();
}
}
116 changes: 90 additions & 26 deletions ...-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableBackupIT.java
View file
Open in desktop
Expand Up @@ -103,26 +103,9 @@ public static void createClient()
prefix = String.format("020%d", System.currentTimeMillis());

tableAdmin = testEnvRule.env().getTableAdminClientForInstance(targetInstance);

testTable =
tableAdmin.createTable(
CreateTableRequest.of(generateId(TEST_TABLE_SUFFIX)).addFamily("cf1"));

// Populate test data.
dataClient = testEnvRule.env().getDataClientForInstance(targetInstance);
byte[] rowBytes = new byte[1024];
Random random = new Random();
random.nextBytes(rowBytes);

List<ApiFuture<?>> futures = Lists.newArrayList();
for (int i = 0; i < 10; i++) {
ApiFuture<Void> future =
dataClient.mutateRowAsync(
RowMutation.create(testTable.getId(), "test-row-" + i)
.setCell("cf1", "", rowBytes.toString()));
futures.add(future);
}
ApiFutures.allAsList(futures).get(3, TimeUnit.MINUTES);
testTable = createAndPopulateTestTable(tableAdmin, dataClient);
}

@AfterClass
Expand Down Expand Up @@ -266,32 +249,89 @@ public void deleteBackupTest() throws InterruptedException {
@Test
public void restoreTableTest() throws InterruptedException, ExecutionException {
String backupId = generateId("restore-" + TEST_BACKUP_SUFFIX);
String tableId = generateId("restored-table");
String restoredTableId = generateId("restored-table");
tableAdmin.createBackup(createBackupRequest(backupId));

// Wait 2 minutes so that the RestoreTable API will trigger an optimize restored
// table operation.
Thread.sleep(120 * 1000);

try {
RestoreTableRequest req = RestoreTableRequest.of(targetCluster, backupId).setTableId(tableId);
RestoreTableRequest req =
RestoreTableRequest.of(targetCluster, backupId).setTableId(restoredTableId);
RestoredTableResult result = tableAdmin.restoreTable(req);
assertWithMessage("Incorrect restored table id")
.that(result.getTable().getId())
.isEqualTo(tableId);
.isEqualTo(restoredTableId);

if (result.getOptimizeRestoredTableOperationToken() != null) {
// The assertion might be missing if the test is running against a HDD cluster or an
// optimization is not necessary.
tableAdmin.awaitOptimizeRestoredTable(result.getOptimizeRestoredTableOperationToken());
Table restoredTable = tableAdmin.getTable(tableId);
Table restoredTable = tableAdmin.getTable(restoredTableId);
assertWithMessage("Incorrect restored table id")
.that(restoredTable.getId())
.isEqualTo(tableId);
.isEqualTo(restoredTableId);
}
} finally {
tableAdmin.deleteBackup(targetCluster, backupId);
tableAdmin.deleteTable(tableId);
tableAdmin.deleteTable(restoredTableId);
}
}

@Test
liubonan marked this conversation as resolved.
Show resolved Hide resolved
public void crossInstanceRestoreTest()
throws InterruptedException, IOException, ExecutionException, TimeoutException {
String backupId = generateId("cross-" + TEST_BACKUP_SUFFIX);
String restoredTableId = generateId("restored-table");

// Set up a new instance to test cross-instance restore. The source backup is stored in this
// instance.
String sourceInstance =
AbstractTestEnv.TEST_INSTANCE_PREFIX + "backup-" + Instant.now().getEpochSecond();
String sourceCluster = AbstractTestEnv.TEST_CLUSTER_PREFIX + Instant.now().getEpochSecond();
instanceAdmin.createInstance(
CreateInstanceRequest.of(sourceInstance)
.addCluster(sourceCluster, testEnvRule.env().getSecondaryZone(), 3, StorageType.SSD)
.setDisplayName("backups-source-test-instance")
.addLabel("state", "readytodelete")
.setType(Type.PRODUCTION));
BigtableTableAdminClient sourceTableAdmin =
testEnvRule.env().getTableAdminClientForInstance(sourceInstance);
Table sourceTable =
createAndPopulateTestTable(
sourceTableAdmin, testEnvRule.env().getDataClientForInstance(sourceInstance));
sourceTableAdmin.createBackup(
CreateBackupRequest.of(sourceCluster, backupId)
.setSourceTableId(sourceTable.getId())
.setExpireTime(Instant.now().plus(Duration.ofHours(6))));

// Wait 2 minutes so that the RestoreTable API will trigger an optimize restored
// table operation.
Thread.sleep(120 * 1000);

try {
RestoreTableRequest req =
RestoreTableRequest.of(sourceInstance, sourceCluster, backupId)
.setTableId(restoredTableId);
RestoredTableResult result = tableAdmin.restoreTable(req);
assertWithMessage("Incorrect restored table id")
.that(result.getTable().getId())
.isEqualTo(restoredTableId);
assertWithMessage("Incorrect instance id")
.that(result.getTable().getInstanceId())
.isEqualTo(targetInstance);

// The assertion might be missing if the test is running against a HDD cluster or an
// optimization is not necessary.
assertWithMessage("Empty OptimizeRestoredTable token")
.that(result.getOptimizeRestoredTableOperationToken())
.isNotNull();
tableAdmin.awaitOptimizeRestoredTable(result.getOptimizeRestoredTableOperationToken());
tableAdmin.getTable(restoredTableId);
} finally {
sourceTableAdmin.deleteBackup(sourceCluster, backupId);
instanceAdmin.deleteInstance(sourceInstance);
}
}

Expand Down Expand Up @@ -327,13 +367,37 @@ public void backupIamTest() throws InterruptedException {
}
}

private CreateBackupRequest createBackupRequest(String backupName) {
return CreateBackupRequest.of(targetCluster, backupName)
private CreateBackupRequest createBackupRequest(String backupId) {
return CreateBackupRequest.of(targetCluster, backupId)
.setSourceTableId(testTable.getId())
.setExpireTime(Instant.now().plus(Duration.ofDays(15)));
}

private static String generateId(String name) {
return prefix + "-" + name;
}

private static Table createAndPopulateTestTable(
BigtableTableAdminClient tableAdmin, BigtableDataClient dataClient)
throws InterruptedException, ExecutionException, TimeoutException {
Table testTable =
tableAdmin.createTable(
CreateTableRequest.of(generateId(TEST_TABLE_SUFFIX)).addFamily("cf1"));

// Populate test data.
byte[] rowBytes = new byte[1024];
Random random = new Random();
random.nextBytes(rowBytes);

List<ApiFuture<?>> futures = Lists.newArrayList();
for (int i = 0; i < 10; i++) {
ApiFuture<Void> future =
dataClient.mutateRowAsync(
RowMutation.create(testTable.getId(), "test-row-" + i)
.setCell("cf1", "", rowBytes.toString()));
futures.add(future);
}
ApiFutures.allAsList(futures).get(3, TimeUnit.MINUTES);
return testTable;
}
}