From ab14a5ec2dc2b7e2141305b5326f436eb6eee76f Mon Sep 17 00:00:00 2001 From: Thiago Nunes Date: Wed, 17 Feb 2021 15:32:33 +1100 Subject: [PATCH] feat!: Point In Time Recovery (PITR) (#452) * feat: exposes new pitr-lite database fields Exposes earliest version time and version retention period fields in the database class. * feat: adds it tests for updating version retention Adds integration tests for updating the version retention period. * feat: adds new create database tests for pitr Adds create database tests for PITR and refactors the integration test class. * chore: refactors tests Separates PITR database tests into 2 files for clarity. * fix: disables pitr-lite tests in emulator The feature is not supported in the emulator currently. * fix: closes result set in test Addresses PR comment. * fix: updates DatabaseInfo equals/hashcode To compare version retention period and earliest version time. * fix: formatting Fixes formatting of the DatabaseInfo * feature: adds test for throttled pitr field Adds test to check for the throttled field in the update database ddl metadata. * fix: explain further the pitr-lite params in docs Adds more explanations to the purpose of the added params for pitr-lite: version_retention_period and earliest_version_time. * feat: adds version time to backups Adds PITR-lite version time to backups. This should make it possible to specify the consistent time for copying the database. * test: adds integration tests for pitr backups * test: adds tests for pitr restore * test: fixes integration test for pitr restore * test: fixes backup unit test * test: fixes npe on pitr backup test * chore: fixes clirr errors * chore: refactors / addresses pr comments * test: fixes the it pitr sad cases tests * test: fixes pitr backup and restore tests * test: skips pitr backup and restore tests This is because the backend for these features is not ready yet. --- .../clirr-ignored-differences.xml | 15 +- .../java/com/google/cloud/spanner/Backup.java | 3 +- .../com/google/cloud/spanner/BackupInfo.java | 33 ++- .../com/google/cloud/spanner/Database.java | 10 +- .../cloud/spanner/DatabaseAdminClient.java | 26 ++ .../spanner/DatabaseAdminClientImpl.java | 30 ++- .../google/cloud/spanner/DatabaseInfo.java | 54 +++- .../spanner/testing/RemoteSpannerHelper.java | 10 + .../spanner/testing/TimestampHelper.java | 41 +++ .../com/google/cloud/spanner/BackupTest.java | 24 +- .../spanner/DatabaseAdminClientImplTest.java | 48 +++- .../spanner/DatabaseAdminClientTest.java | 69 +++-- .../google/cloud/spanner/DatabaseTest.java | 14 +- .../google/cloud/spanner/it/ITBackupTest.java | 51 +--- .../spanner/it/ITPitrBackupAndRestore.java | 244 ++++++++++++++++++ .../spanner/it/ITPitrCreateDatabaseTest.java | 144 +++++++++++ .../spanner/it/ITPitrUpdateDatabaseTest.java | 207 +++++++++++++++ 17 files changed, 935 insertions(+), 88 deletions(-) create mode 100644 google-cloud-spanner/src/main/java/com/google/cloud/spanner/testing/TimestampHelper.java create mode 100644 google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITPitrBackupAndRestore.java create mode 100644 google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITPitrCreateDatabaseTest.java create mode 100644 google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITPitrUpdateDatabaseTest.java diff --git a/google-cloud-spanner/clirr-ignored-differences.xml b/google-cloud-spanner/clirr-ignored-differences.xml index fcafcb7c3a..c72dc8c669 100644 --- a/google-cloud-spanner/clirr-ignored-differences.xml +++ b/google-cloud-spanner/clirr-ignored-differences.xml @@ -453,7 +453,7 @@ com/google/cloud/spanner/TransactionContext com.google.api.core.ApiFuture executeUpdateAsync(com.google.cloud.spanner.Statement) - + 7012 @@ -475,5 +475,16 @@ com/google/cloud/spanner/TransactionRunner com.google.cloud.spanner.CommitResponse getCommitResponse() - + + + + 7013 + com/google/cloud/spanner/BackupInfo$Builder + com.google.cloud.spanner.BackupInfo$Builder setVersionTime(com.google.cloud.Timestamp) + + + 7012 + com/google/cloud/spanner/DatabaseAdminClient + com.google.api.gax.longrunning.OperationFuture createBackup(com.google.cloud.spanner.Backup) + diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/Backup.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/Backup.java index d49e9f7f50..a0052cd381 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/Backup.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/Backup.java @@ -65,7 +65,7 @@ public OperationFuture create() { getExpireTime() != null, "Cannot create a backup without an expire time"); Preconditions.checkState( getDatabase() != null, "Cannot create a backup without a source database"); - return dbClient.createBackup(instance(), backup(), sourceDatabase(), getExpireTime()); + return dbClient.createBackup(this); } /** @@ -182,6 +182,7 @@ static Backup fromProto( .setState(fromProtoState(proto.getState())) .setSize(proto.getSizeBytes()) .setExpireTime(Timestamp.fromProto(proto.getExpireTime())) + .setVersionTime(Timestamp.fromProto(proto.getVersionTime())) .setDatabase(DatabaseId.of(proto.getDatabase())) .setProto(proto) .build(); diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BackupInfo.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BackupInfo.java index 5968bcdc21..199e6ae2ae 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BackupInfo.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BackupInfo.java @@ -18,6 +18,7 @@ import com.google.api.client.util.Preconditions; import com.google.cloud.Timestamp; +import com.google.spanner.admin.database.v1.Database; import java.util.Objects; import javax.annotation.Nullable; @@ -40,6 +41,17 @@ public abstract static class Builder { */ public abstract Builder setExpireTime(Timestamp expireTime); + /** + * Optional for creating a new backup. + * + *

Specifies the timestamp to have an externally consistent copy of the database. If no + * version time is specified, it will be automatically set to the backup create time. + * + *

The version time can be as far in the past as specified by the database earliest version + * time (see {@link Database#getEarliestVersionTime()}). + */ + public abstract Builder setVersionTime(Timestamp versionTime); + /** * Required for creating a new backup. * @@ -55,6 +67,7 @@ abstract static class BuilderImpl extends Builder { protected final BackupId id; private State state = State.UNSPECIFIED; private Timestamp expireTime; + private Timestamp versionTime; private DatabaseId database; private long size; private com.google.spanner.admin.database.v1.Backup proto; @@ -67,6 +80,7 @@ abstract static class BuilderImpl extends Builder { this.id = other.id; this.state = other.state; this.expireTime = other.expireTime; + this.versionTime = other.versionTime; this.database = other.database; this.size = other.size; this.proto = other.proto; @@ -84,6 +98,12 @@ public Builder setExpireTime(Timestamp expireTime) { return this; } + @Override + public Builder setVersionTime(Timestamp versionTime) { + this.versionTime = versionTime; + return this; + } + @Override public Builder setDatabase(DatabaseId database) { Preconditions.checkArgument( @@ -119,6 +139,7 @@ public enum State { private final BackupId id; private final State state; private final Timestamp expireTime; + private final Timestamp versionTime; private final DatabaseId database; private final long size; private final com.google.spanner.admin.database.v1.Backup proto; @@ -128,6 +149,7 @@ public enum State { this.state = builder.state; this.size = builder.size; this.expireTime = builder.expireTime; + this.versionTime = builder.versionTime; this.database = builder.database; this.proto = builder.proto; } @@ -157,6 +179,11 @@ public Timestamp getExpireTime() { return expireTime; } + /** Returns the version time of the backup. */ + public Timestamp getVersionTime() { + return versionTime; + } + /** Returns the id of the database that was used to create the backup. */ public DatabaseId getDatabase() { return database; @@ -180,17 +207,19 @@ public boolean equals(Object o) { && state == that.state && size == that.size && Objects.equals(expireTime, that.expireTime) + && Objects.equals(versionTime, that.versionTime) && Objects.equals(database, that.database); } @Override public int hashCode() { - return Objects.hash(id, state, size, expireTime, database); + return Objects.hash(id, state, size, expireTime, versionTime, database); } @Override public String toString() { return String.format( - "Backup[%s, %s, %d, %s, %s]", id.getName(), state, size, expireTime, database); + "Backup[%s, %s, %d, %s, %s, %s]", + id.getName(), state, size, expireTime, versionTime, database); } } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/Database.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/Database.java index 89ee6e875a..a442ad2399 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/Database.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/Database.java @@ -118,8 +118,14 @@ public OperationFuture backup(Backup backup) { Preconditions.checkArgument( backup.getInstanceId().equals(getId().getInstanceId()), "The instance of the backup must be equal to the instance of this database."); + return dbClient.createBackup( - instance(), backup.getId().getBackup(), database(), backup.getExpireTime()); + dbClient + .newBackupBuilder(backup.getId()) + .setDatabase(getId()) + .setExpireTime(backup.getExpireTime()) + .setVersionTime(backup.getVersionTime()) + .build()); } /** @@ -177,6 +183,8 @@ static Database fromProto( .setState(fromProtoState(proto.getState())) .setCreateTime(Timestamp.fromProto(proto.getCreateTime())) .setRestoreInfo(RestoreInfo.fromProtoOrNullIfDefaultInstance(proto.getRestoreInfo())) + .setVersionRetentionPeriod(proto.getVersionRetentionPeriod()) + .setEarliestVersionTime(Timestamp.fromProto(proto.getEarliestVersionTime())) .setProto(proto) .build(); } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseAdminClient.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseAdminClient.java index e7a8ed6787..eae1a3cdf4 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseAdminClient.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseAdminClient.java @@ -101,6 +101,32 @@ OperationFuture createBackup( String sourceInstanceId, String backupId, String databaseId, Timestamp expireTime) throws SpannerException; + /** + * Creates a new backup from a database in a Cloud Spanner instance. + * + *

Example to create a backup. + * + * 

{@code
+   * BackupId backupId     = BackupId.of("project", "instance", "backup-id");
+   * DatabaseId databaseId = DatabaseId.of("project", "instance", "database-id");
+   * Timestamp expireTime  = Timestamp.ofTimeMicroseconds(expireTimeMicros);
+   * Timestamp versionTime = Timestamp.ofTimeMicroseconds(versionTimeMicros);
+   *
+   * Backup backupToCreate = dbAdminClient
+   *     .newBackupBuilder(backupId)
+   *     .setDatabase(databaseId)
+   *     .setExpireTime(expireTime)
+   *     .setVersionTime(versionTime)
+   *     .build();
+   *
+   * OperationFuture op = dbAdminClient.createBackup(backupToCreate);
+   * Backup createdBackup = op.get();
+   * }
+ * + * @param backup the backup to be created + */ + OperationFuture createBackup(Backup backup) throws SpannerException; + /** * Restore a database from a backup. The database that is restored will be created and may not * already exist. diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseAdminClientImpl.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseAdminClientImpl.java index 28de150cae..a5eed214a4 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseAdminClientImpl.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseAdminClientImpl.java @@ -114,16 +114,31 @@ public Database apply(Exception e) { public OperationFuture createBackup( String instanceId, String backupId, String databaseId, Timestamp expireTime) throws SpannerException { - com.google.spanner.admin.database.v1.Backup backup = + final Backup backup = + newBackupBuilder(BackupId.of(projectId, instanceId, backupId)) + .setDatabase(DatabaseId.of(projectId, instanceId, databaseId)) + .setExpireTime(expireTime) + .build(); + return createBackup(backup); + } + + @Override + public OperationFuture createBackup(final Backup backup) { + final String instanceId = backup.getInstanceId().getInstance(); + final String databaseId = backup.getDatabase().getDatabase(); + final String backupId = backup.getId().getBackup(); + final com.google.spanner.admin.database.v1.Backup.Builder backupBuilder = com.google.spanner.admin.database.v1.Backup.newBuilder() .setDatabase(getDatabaseName(instanceId, databaseId)) - .setExpireTime(expireTime.toProto()) - .build(); - String instanceName = getInstanceName(instanceId); - OperationFuture - rawOperationFuture = rpc.createBackup(instanceName, backupId, backup); + .setExpireTime(backup.getExpireTime().toProto()); + if (backup.getVersionTime() != null) { + backupBuilder.setVersionTime(backup.getVersionTime().toProto()); + } + final String instanceName = getInstanceName(instanceId); + final OperationFuture + rawOperationFuture = rpc.createBackup(instanceName, backupId, backupBuilder.build()); - return new OperationFutureImpl( + return new OperationFutureImpl<>( rawOperationFuture.getPollingFuture(), rawOperationFuture.getInitialFuture(), new ApiFunction() { @@ -137,6 +152,7 @@ public Backup apply(OperationSnapshot snapshot) { com.google.spanner.admin.database.v1.Backup.newBuilder(proto) .setName(proto.getName()) .setExpireTime(proto.getExpireTime()) + .setVersionTime(proto.getVersionTime()) .setState(proto.getState()) .build(), DatabaseAdminClientImpl.this); diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseInfo.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseInfo.java index 5b7faa324e..5ba9f0aa76 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseInfo.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/DatabaseInfo.java @@ -30,6 +30,10 @@ public abstract static class Builder { abstract Builder setRestoreInfo(RestoreInfo restoreInfo); + abstract Builder setVersionRetentionPeriod(String versionRetentionPeriod); + + abstract Builder setEarliestVersionTime(Timestamp earliestVersionTime); + abstract Builder setProto(com.google.spanner.admin.database.v1.Database proto); /** Builds the database from this builder. */ @@ -41,6 +45,8 @@ abstract static class BuilderImpl extends Builder { private State state = State.UNSPECIFIED; private Timestamp createTime; private RestoreInfo restoreInfo; + private String versionRetentionPeriod; + private Timestamp earliestVersionTime; private com.google.spanner.admin.database.v1.Database proto; BuilderImpl(DatabaseId id) { @@ -52,6 +58,8 @@ abstract static class BuilderImpl extends Builder { this.state = other.state; this.createTime = other.createTime; this.restoreInfo = other.restoreInfo; + this.versionRetentionPeriod = other.versionRetentionPeriod; + this.earliestVersionTime = other.earliestVersionTime; this.proto = other.proto; } @@ -73,6 +81,18 @@ Builder setRestoreInfo(@Nullable RestoreInfo restoreInfo) { return this; } + @Override + Builder setVersionRetentionPeriod(String versionRetentionPeriod) { + this.versionRetentionPeriod = versionRetentionPeriod; + return this; + } + + @Override + Builder setEarliestVersionTime(Timestamp earliestVersionTime) { + this.earliestVersionTime = earliestVersionTime; + return this; + } + @Override Builder setProto(@Nullable com.google.spanner.admin.database.v1.Database proto) { this.proto = proto; @@ -96,6 +116,8 @@ public enum State { private final State state; private final Timestamp createTime; private final RestoreInfo restoreInfo; + private final String versionRetentionPeriod; + private final Timestamp earliestVersionTime; private final com.google.spanner.admin.database.v1.Database proto; public DatabaseInfo(DatabaseId id, State state) { @@ -103,6 +125,8 @@ public DatabaseInfo(DatabaseId id, State state) { this.state = state; this.createTime = null; this.restoreInfo = null; + this.versionRetentionPeriod = null; + this.earliestVersionTime = null; this.proto = null; } @@ -111,6 +135,8 @@ public DatabaseInfo(DatabaseId id, State state) { this.state = builder.state; this.createTime = builder.createTime; this.restoreInfo = builder.restoreInfo; + this.versionRetentionPeriod = builder.versionRetentionPeriod; + this.earliestVersionTime = builder.earliestVersionTime; this.proto = builder.proto; } @@ -129,6 +155,23 @@ public Timestamp getCreateTime() { return createTime; } + /** + * Returns the version retention period of the database. This is the period for which Cloud + * Spanner retains all versions of data for the database. For instance, if set to 3 days, Cloud + * Spanner will retain data versions that are up to 3 days old. + */ + public String getVersionRetentionPeriod() { + return versionRetentionPeriod; + } + + /** + * Returns the earliest version time of the database. This is the oldest timestamp that can be + * used to read old versions of the data. + */ + public Timestamp getEarliestVersionTime() { + return earliestVersionTime; + } + /** * Returns the {@link RestoreInfo} of the database if any is available, or null if no * {@link RestoreInfo} is available for this database. @@ -154,16 +197,21 @@ public boolean equals(Object o) { return id.equals(that.id) && state == that.state && Objects.equals(createTime, that.createTime) - && Objects.equals(restoreInfo, that.restoreInfo); + && Objects.equals(restoreInfo, that.restoreInfo) + && Objects.equals(versionRetentionPeriod, that.versionRetentionPeriod) + && Objects.equals(earliestVersionTime, that.earliestVersionTime); } @Override public int hashCode() { - return Objects.hash(id, state, createTime, restoreInfo); + return Objects.hash( + id, state, createTime, restoreInfo, versionRetentionPeriod, earliestVersionTime); } @Override public String toString() { - return String.format("Database[%s, %s, %s, %s]", id.getName(), state, createTime, restoreInfo); + return String.format( + "Database[%s, %s, %s, %s, %s, %s]", + id.getName(), state, createTime, restoreInfo, versionRetentionPeriod, earliestVersionTime); } } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/testing/RemoteSpannerHelper.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/testing/RemoteSpannerHelper.java index ef979a8706..6d2fe90227 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/testing/RemoteSpannerHelper.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/testing/RemoteSpannerHelper.java @@ -30,6 +30,7 @@ import java.util.Arrays; import java.util.List; import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; @@ -45,6 +46,8 @@ public class RemoteSpannerHelper { private final InstanceId instanceId; private static int dbSeq; private static int dbPrefix = new Random().nextInt(Integer.MAX_VALUE); + private static final AtomicInteger backupSeq = new AtomicInteger(); + private static int backupPrefix = new Random().nextInt(Integer.MAX_VALUE); private final List dbs = new ArrayList<>(); protected RemoteSpannerHelper(SpannerOptions options, InstanceId instanceId, Spanner client) { @@ -100,6 +103,13 @@ public String getUniqueDatabaseId() { return String.format("testdb_%d_%04d", dbPrefix, dbSeq++); } + /** + * Returns a backup id which is guaranteed to be unique within the context of this environment. + */ + public String getUniqueBackupId() { + return String.format("testbck_%06d_%04d", backupPrefix, backupSeq.incrementAndGet()); + } + /** * Creates a test database defined by {@code statements} in the test instance. A {@code CREATE * DATABASE ...} statement should not be included; an appropriate name will be chosen and the diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/testing/TimestampHelper.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/testing/TimestampHelper.java new file mode 100644 index 0000000000..021adf4dd7 --- /dev/null +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/testing/TimestampHelper.java @@ -0,0 +1,41 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner.testing; + +import com.google.cloud.Timestamp; +import java.util.concurrent.TimeUnit; + +public class TimestampHelper { + + public static Timestamp daysAgo(int days) { + return Timestamp.ofTimeMicroseconds( + TimeUnit.MICROSECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS) + - TimeUnit.MICROSECONDS.convert(days, TimeUnit.DAYS)); + } + + public static Timestamp afterDays(int days) { + return Timestamp.ofTimeMicroseconds( + TimeUnit.MICROSECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS) + + TimeUnit.MICROSECONDS.convert(days, TimeUnit.DAYS)); + } + + public static Timestamp afterMinutes(int minutes) { + return Timestamp.ofTimeMicroseconds( + TimeUnit.MICROSECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS) + + TimeUnit.MICROSECONDS.convert(minutes, TimeUnit.MINUTES)); + } +} diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BackupTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BackupTest.java index 3bdc673838..80b99f679b 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BackupTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/BackupTest.java @@ -46,6 +46,7 @@ public class BackupTest { "projects/test-project/instances/test-instance/backups/backup-1"; private static final String DB = "projects/test-project/instances/test-instance/databases/db-1"; private static final Timestamp EXP_TIME = Timestamp.ofTimeSecondsAndNanos(1000L, 1000); + private static final Timestamp VERSION_TIME = Timestamp.ofTimeSecondsAndNanos(2000L, 2000); @Mock DatabaseAdminClient dbClient; @@ -65,11 +66,13 @@ public Builder answer(InvocationOnMock invocation) { @Test public void build() { Timestamp expireTime = Timestamp.now(); + Timestamp versionTime = Timestamp.ofTimeMicroseconds(10L); Backup backup = dbClient .newBackupBuilder(BackupId.of("test-project", "instance-id", "backup-id")) .setDatabase(DatabaseId.of("test-project", "instance-id", "src-database")) .setExpireTime(expireTime) + .setVersionTime(versionTime) .setSize(100L) .setState(State.CREATING) .build(); @@ -77,6 +80,7 @@ public void build() { assertThat(copy.getId()).isEqualTo(backup.getId()); assertThat(copy.getDatabase()).isEqualTo(backup.getDatabase()); assertThat(copy.getExpireTime()).isEqualTo(backup.getExpireTime()); + assertThat(copy.getVersionTime()).isEqualTo(backup.getVersionTime()); assertThat(copy.getSize()).isEqualTo(backup.getSize()); assertThat(copy.getState()).isEqualTo(backup.getState()); } @@ -84,14 +88,16 @@ public void build() { @Test public void create() { Timestamp expireTime = Timestamp.now(); + Timestamp versionTime = Timestamp.ofTimeMicroseconds(10L); Backup backup = dbClient .newBackupBuilder(BackupId.of("test-project", "instance-id", "backup-id")) .setDatabase(DatabaseId.of("test-project", "instance-id", "src-database")) .setExpireTime(expireTime) + .setVersionTime(versionTime) .build(); backup.create(); - verify(dbClient).createBackup("instance-id", "backup-id", "src-database", expireTime); + verify(dbClient).createBackup(backup); } @Test @@ -125,6 +131,19 @@ public void createWithoutExpireTime() { } } + @Test + public void createWithoutVersionTimeShouldSucceed() { + final Timestamp expireTime = Timestamp.now(); + Backup backup = + dbClient + .newBackupBuilder(BackupId.of("test-project", "instance-id", "backup-id")) + .setDatabase(DatabaseId.of("test-project", "instance-id", "src-database")) + .setExpireTime(expireTime) + .build(); + backup.create(); + verify(dbClient).createBackup(backup); + } + @Test public void exists() { when(dbClient.getBackup("test-instance", "test-backup")) @@ -298,6 +317,7 @@ public void fromProto() { assertThat(backup.getId().getName()).isEqualTo(NAME); assertThat(backup.getState()).isEqualTo(BackupInfo.State.CREATING); assertThat(backup.getExpireTime()).isEqualTo(EXP_TIME); + assertThat(backup.getVersionTime()).isEqualTo(VERSION_TIME); } private Backup createBackup() { @@ -307,6 +327,8 @@ private Backup createBackup() { .setDatabase(DB) .setExpireTime( com.google.protobuf.Timestamp.newBuilder().setSeconds(1000L).setNanos(1000).build()) + .setVersionTime( + com.google.protobuf.Timestamp.newBuilder().setSeconds(2000L).setNanos(2000).build()) .setState(com.google.spanner.admin.database.v1.Backup.State.CREATING) .build(); return Backup.fromProto(proto, dbClient); diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseAdminClientImplTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseAdminClientImplTest.java index b2676473cf..fb617797aa 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseAdminClientImplTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseAdminClientImplTest.java @@ -48,6 +48,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import org.junit.Assert; import org.junit.Before; @@ -69,6 +70,8 @@ public class DatabaseAdminClientImplTest { private static final String BK_ID = "my-bk"; private static final String BK_NAME = "projects/my-project/instances/my-instance/backups/my-bk"; private static final String BK_NAME2 = "projects/my-project/instances/my-instance/backups/my-bk2"; + private static final Timestamp EARLIEST_VERSION_TIME = Timestamp.now(); + private static final String VERSION_RETENTION_PERIOD = "7d"; @Mock SpannerRpc rpc; DatabaseAdminClientImpl client; @@ -80,7 +83,12 @@ public void setUp() { } private Database getDatabaseProto() { - return Database.newBuilder().setName(DB_NAME).setState(Database.State.READY).build(); + return Database.newBuilder() + .setName(DB_NAME) + .setState(Database.State.READY) + .setEarliestVersionTime(EARLIEST_VERSION_TIME.toProto()) + .setVersionRetentionPeriod(VERSION_RETENTION_PERIOD) + .build(); } private Database getAnotherDatabaseProto() { @@ -116,6 +124,8 @@ public void getDatabase() { com.google.cloud.spanner.Database db = client.getDatabase(INSTANCE_ID, DB_ID); assertThat(db.getId().getName()).isEqualTo(DB_NAME); assertThat(db.getState()).isEqualTo(DatabaseInfo.State.READY); + assertThat(db.getEarliestVersionTime()).isEqualTo(EARLIEST_VERSION_TIME); + assertThat(db.getVersionRetentionPeriod()).isEqualTo(VERSION_RETENTION_PERIOD); } @Test @@ -292,7 +302,7 @@ public void testDatabaseIAMPermissions() { } @Test - public void createBackup() throws Exception { + public void createBackupWithParams() throws Exception { OperationFuture rawOperationFuture = OperationFutureUtil.immediateOperationFuture( "createBackup", getBackupProto(), CreateBackupMetadata.getDefaultInstance()); @@ -308,6 +318,40 @@ public void createBackup() throws Exception { assertThat(op.get().getId().getName()).isEqualTo(BK_NAME); } + @Test + public void createBackupWithBackupObject() throws ExecutionException, InterruptedException { + final OperationFuture rawOperationFuture = + OperationFutureUtil.immediateOperationFuture( + "createBackup", getBackupProto(), CreateBackupMetadata.getDefaultInstance()); + final Timestamp expireTime = + Timestamp.ofTimeMicroseconds( + TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis()) + + TimeUnit.HOURS.toMicros(28)); + final Timestamp versionTime = + Timestamp.ofTimeMicroseconds( + TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis()) - TimeUnit.DAYS.toMicros(2)); + final Backup expectedCallBackup = + Backup.newBuilder() + .setDatabase(DB_NAME) + .setExpireTime(expireTime.toProto()) + .setVersionTime(versionTime.toProto()) + .build(); + final com.google.cloud.spanner.Backup requestBackup = + client + .newBackupBuilder(BackupId.of(PROJECT_ID, INSTANCE_ID, BK_ID)) + .setDatabase(DatabaseId.of(PROJECT_ID, INSTANCE_ID, DB_ID)) + .setExpireTime(expireTime) + .setVersionTime(versionTime) + .build(); + + when(rpc.createBackup(INSTANCE_NAME, BK_ID, expectedCallBackup)).thenReturn(rawOperationFuture); + + final OperationFuture op = + client.createBackup(requestBackup); + assertThat(op.isDone()).isTrue(); + assertThat(op.get().getId().getName()).isEqualTo(BK_NAME); + } + @Test public void deleteBackup() { client.deleteBackup(INSTANCE_ID, BK_ID); diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseAdminClientTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseAdminClientTest.java index 8b30df912d..95641ef3ff 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseAdminClientTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseAdminClientTest.java @@ -16,6 +16,7 @@ package com.google.cloud.spanner; +import static com.google.cloud.spanner.testing.TimestampHelper.afterDays; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.fail; @@ -224,7 +225,7 @@ public void tearDown() { public void dbAdminCreateBackup() throws InterruptedException, ExecutionException { final String backupId = "other-backup-id"; OperationFuture op = - client.createBackup(INSTANCE_ID, backupId, DB_ID, after7Days()); + client.createBackup(INSTANCE_ID, backupId, DB_ID, afterDays(7)); Backup backup = op.get(); assertThat(backup.getId().getName()) .isEqualTo( @@ -240,7 +241,8 @@ public void backupCreate() throws InterruptedException, ExecutionException { client .newBackupBuilder(BackupId.of(PROJECT_ID, INSTANCE_ID, backupId)) .setDatabase(DatabaseId.of(PROJECT_ID, INSTANCE_ID, DB_ID)) - .setExpireTime(after7Days()) + .setExpireTime(afterDays(7)) + .setVersionTime(sevenDaysAgo()) .build(); OperationFuture op = backup.create(); backup = op.get(); @@ -251,6 +253,25 @@ public void backupCreate() throws InterruptedException, ExecutionException { assertThat(client.getBackup(INSTANCE_ID, backupId)).isEqualTo(backup); } + @Test + public void databaseAdminBackupCreate() throws ExecutionException, InterruptedException { + final String backupId = "other-backup-id"; + Backup backup = + client + .newBackupBuilder(BackupId.of(PROJECT_ID, INSTANCE_ID, backupId)) + .setDatabase(DatabaseId.of(PROJECT_ID, INSTANCE_ID, DB_ID)) + .setExpireTime(afterDays(7)) + .setVersionTime(sevenDaysAgo()) + .build(); + final OperationFuture op = client.createBackup(backup); + backup = op.get(); + assertThat(backup.getId().getName()) + .isEqualTo( + String.format( + "projects/%s/instances/%s/backups/%s", PROJECT_ID, INSTANCE_ID, backupId)); + assertThat(client.getBackup(INSTANCE_ID, backupId)).isEqualTo(backup); + } + @Test public void backupCreateCancel() { final String backupId = "other-backup-id"; @@ -299,7 +320,7 @@ public void databaseBackup() throws InterruptedException, ExecutionException { db.backup( client .newBackupBuilder(BackupId.of(PROJECT_ID, INSTANCE_ID, backupId)) - .setExpireTime(after7Days()) + .setExpireTime(afterDays(7)) .build()) .get(); assertThat(backup.getId().getName()) @@ -312,7 +333,7 @@ public void databaseBackup() throws InterruptedException, ExecutionException { @Test public void dbAdminCreateBackupAlreadyExists() throws InterruptedException { OperationFuture op = - client.createBackup(INSTANCE_ID, BCK_ID, DB_ID, after7Days()); + client.createBackup(INSTANCE_ID, BCK_ID, DB_ID, afterDays(7)); try { op.get(); fail("missing expected exception"); @@ -329,7 +350,7 @@ public void backupCreateAlreadyExists() throws InterruptedException { client .newBackupBuilder(BackupId.of(PROJECT_ID, INSTANCE_ID, BCK_ID)) .setDatabase(DatabaseId.of(PROJECT_ID, INSTANCE_ID, DB_ID)) - .setExpireTime(after7Days()) + .setExpireTime(afterDays(7)) .build(); try { backup.create().get(); @@ -348,7 +369,7 @@ public void databaseBackupAlreadyExists() throws InterruptedException { db.backup( client .newBackupBuilder(BackupId.of(PROJECT_ID, INSTANCE_ID, BCK_ID)) - .setExpireTime(after7Days()) + .setExpireTime(afterDays(7)) .build()); try { op.get(); @@ -364,7 +385,7 @@ public void databaseBackupAlreadyExists() throws InterruptedException { public void dbAdminCreateBackupDbNotFound() throws InterruptedException { final String backupId = "other-backup-id"; OperationFuture op = - client.createBackup(INSTANCE_ID, backupId, "does-not-exist", after7Days()); + client.createBackup(INSTANCE_ID, backupId, "does-not-exist", afterDays(7)); try { op.get(); fail("missing expected exception"); @@ -381,7 +402,7 @@ public void backupCreateDbNotFound() throws InterruptedException { client .newBackupBuilder(BackupId.of(PROJECT_ID, INSTANCE_ID, backupId)) .setDatabase(DatabaseId.of(PROJECT_ID, INSTANCE_ID, "does-not-exist")) - .setExpireTime(after7Days()) + .setExpireTime(afterDays(7)) .build(); try { backup.create().get(); @@ -402,7 +423,7 @@ public void databaseBackupDbNotFound() throws InterruptedException { db.backup( client .newBackupBuilder(BackupId.of(PROJECT_ID, INSTANCE_ID, backupId)) - .setExpireTime(after7Days()) + .setExpireTime(afterDays(7)) .build()); try { op.get(); @@ -499,7 +520,7 @@ public void dbClientListBackups() throws SpannerException, InterruptedException, ExecutionException { Backup backup = client.getBackup(INSTANCE_ID, BCK_ID); assertThat(client.listBackups(INSTANCE_ID).iterateAll()).containsExactly(backup); - Backup backup2 = client.createBackup(INSTANCE_ID, "backup2", DB_ID, after7Days()).get(); + Backup backup2 = client.createBackup(INSTANCE_ID, "backup2", DB_ID, afterDays(7)).get(); assertThat(client.listBackups(INSTANCE_ID).iterateAll()).containsExactly(backup, backup2); backup2.delete(); assertThat(client.listBackups(INSTANCE_ID).iterateAll()).containsExactly(backup); @@ -515,7 +536,7 @@ public void instanceListBackups() .build(); Backup backup = client.getBackup(INSTANCE_ID, BCK_ID); assertThat(instance.listBackups().iterateAll()).containsExactly(backup); - Backup backup2 = client.createBackup(INSTANCE_ID, "backup2", DB_ID, after7Days()).get(); + Backup backup2 = client.createBackup(INSTANCE_ID, "backup2", DB_ID, afterDays(7)).get(); assertThat(instance.listBackups().iterateAll()).containsExactly(backup, backup2); backup2.delete(); assertThat(instance.listBackups().iterateAll()).containsExactly(backup); @@ -532,7 +553,7 @@ public void instanceListBackupsWithFilter() Backup backup = client.getBackup(INSTANCE_ID, BCK_ID); assertThat(instance.listBackups().iterateAll()).containsExactly(backup); - Backup backup2 = client.createBackup(INSTANCE_ID, "backup2", DB_ID, after7Days()).get(); + Backup backup2 = client.createBackup(INSTANCE_ID, "backup2", DB_ID, afterDays(7)).get(); // All backups. assertThat(instance.listBackups().iterateAll()).containsExactly(backup, backup2); @@ -549,7 +570,7 @@ public void instanceListBackupsWithFilter() .containsExactly(backup, backup2); // All backups that expire before a certain time. - String ts = after14Days().toString(); + String ts = afterDays(14).toString(); filter = String.format("expire_time < \"%s\"", ts); mockDatabaseAdmin.addFilterMatches(filter, backup.getId().getName(), backup2.getId().getName()); assertThat(instance.listBackups(Options.filter(filter)).iterateAll()) @@ -701,7 +722,7 @@ public void databaseListDatabaseOperations() public void dbClientListBackupOperations() throws SpannerException, InterruptedException, ExecutionException { assertThat(client.listBackupOperations(INSTANCE_ID).iterateAll()).hasSize(1); - client.createBackup(INSTANCE_ID, "other-backup", DB_ID, after7Days()).get(); + client.createBackup(INSTANCE_ID, "other-backup", DB_ID, afterDays(7)).get(); assertThat(client.listBackupOperations(INSTANCE_ID).iterateAll()).hasSize(2); // Restore a backup. This creates 2 DATABASE operations: One to restore the database and // one to optimize it. @@ -723,7 +744,7 @@ public void instanceListBackupOperations() .backup( client .newBackupBuilder(BackupId.of(PROJECT_ID, INSTANCE_ID, "other-backup")) - .setExpireTime(after7Days()) + .setExpireTime(afterDays(7)) .build()) .get(); assertThat(instance.listBackupOperations().iterateAll()).hasSize(2); @@ -768,7 +789,7 @@ public void backupListBackupOperations() Backup backup = client.newBackupBuilder(BackupId.of(PROJECT_ID, INSTANCE_ID, BCK_ID)).build(); mockDatabaseAdmin.addFilterMatches("name:backups/" + BCK_ID, createBackupOperation.getName()); assertThat(backup.listBackupOperations().iterateAll()).hasSize(1); - client.createBackup(INSTANCE_ID, "other-backup", DB_ID, after7Days()).get(); + client.createBackup(INSTANCE_ID, "other-backup", DB_ID, afterDays(7)).get(); assertThat(backup.listBackupOperations().iterateAll()).hasSize(1); } @@ -791,16 +812,10 @@ public void testDatabaseIAMPermissions() { assertThat(permissions).containsExactly("spanner.databases.select"); } - private Timestamp after7Days() { - return Timestamp.ofTimeMicroseconds( - TimeUnit.MICROSECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS) - + TimeUnit.MICROSECONDS.convert(7, TimeUnit.DAYS)); - } - - private Timestamp after14Days() { + private Timestamp sevenDaysAgo() { return Timestamp.ofTimeMicroseconds( TimeUnit.MICROSECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS) - + TimeUnit.MICROSECONDS.convert(14, TimeUnit.DAYS)); + - TimeUnit.MICROSECONDS.convert(7, TimeUnit.DAYS)); } private void createTestDatabase() { @@ -814,7 +829,7 @@ private void createTestDatabase() { private void createTestBackup() { try { - createBackupOperation = client.createBackup(INSTANCE_ID, BCK_ID, DB_ID, after7Days()); + createBackupOperation = client.createBackup(INSTANCE_ID, BCK_ID, DB_ID, afterDays(7)); createBackupOperation.get(); } catch (InterruptedException | ExecutionException e) { throw SpannerExceptionFactory.newSpannerException(e); @@ -839,7 +854,7 @@ public void retryCreateBackupSlowResponse() throws Exception { SimulatedExecutionTime.ofException(Status.DEADLINE_EXCEEDED.asRuntimeException())); final String backupId = "other-backup-id"; OperationFuture op = - client.createBackup(INSTANCE_ID, backupId, DB_ID, after7Days()); + client.createBackup(INSTANCE_ID, backupId, DB_ID, afterDays(7)); Backup backup = op.get(); assertThat(backup.getId().getName()) .isEqualTo( @@ -857,7 +872,7 @@ public void retryCreateBackupSlowStartup() throws Exception { SimulatedExecutionTime.ofException(Status.DEADLINE_EXCEEDED.asRuntimeException())); final String backupId = "other-backup-id"; OperationFuture op = - client.createBackup(INSTANCE_ID, backupId, DB_ID, after7Days()); + client.createBackup(INSTANCE_ID, backupId, DB_ID, afterDays(7)); Backup backup = op.get(); assertThat(backup.getId().getName()) .isEqualTo( diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseTest.java index 9557615007..7c61720b21 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/DatabaseTest.java @@ -42,6 +42,9 @@ public class DatabaseTest { private static final String NAME = "projects/test-project/instances/test-instance/databases/database-1"; + private static final Timestamp EARLIEST_VERSION_TIME = Timestamp.now(); + private static final String VERSION_RETENTION_PERIOD = "7d"; + @Mock DatabaseAdminClient dbClient; @Before @@ -61,12 +64,13 @@ public Backup.Builder answer(InvocationOnMock invocation) { public void backup() { Timestamp expireTime = Timestamp.now(); Database db = createDatabase(); - db.backup( + Backup backup = dbClient .newBackupBuilder(BackupId.of("test-project", "test-instance", "test-backup")) .setExpireTime(expireTime) - .build()); - verify(dbClient).createBackup("test-instance", "test-backup", "database-1", expireTime); + .build(); + db.backup(backup); + verify(dbClient).createBackup(backup.toBuilder().setDatabase(db.getId()).build()); } @Test @@ -82,6 +86,8 @@ public void fromProto() { Database db = createDatabase(); assertThat(db.getId().getName()).isEqualTo(NAME); assertThat(db.getState()).isEqualTo(DatabaseInfo.State.CREATING); + assertThat(db.getVersionRetentionPeriod()).isEqualTo(VERSION_RETENTION_PERIOD); + assertThat(db.getEarliestVersionTime()).isEqualTo(EARLIEST_VERSION_TIME); } @Test @@ -119,6 +125,8 @@ private Database createDatabase() { com.google.spanner.admin.database.v1.Database.newBuilder() .setName(NAME) .setState(com.google.spanner.admin.database.v1.Database.State.CREATING) + .setEarliestVersionTime(EARLIEST_VERSION_TIME.toProto()) + .setVersionRetentionPeriod(VERSION_RETENTION_PERIOD) .build(); return Database.fromProto(proto, dbClient); } diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBackupTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBackupTest.java index 0fbd0702ec..e9b9887184 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBackupTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITBackupTest.java @@ -17,6 +17,9 @@ package com.google.cloud.spanner.it; import static com.google.cloud.spanner.testing.EmulatorSpannerHelper.isUsingEmulator; +import static com.google.cloud.spanner.testing.TimestampHelper.afterDays; +import static com.google.cloud.spanner.testing.TimestampHelper.afterMinutes; +import static com.google.cloud.spanner.testing.TimestampHelper.daysAgo; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.fail; import static org.junit.Assume.assumeFalse; @@ -58,7 +61,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; @@ -87,7 +89,6 @@ public class ITBackupTest { private InstanceAdminClient instanceAdminClient; private Instance instance; private RemoteSpannerHelper testHelper; - private final AtomicInteger backupSeq = new AtomicInteger(); private List databases = new ArrayList<>(); private List backups = new ArrayList<>(); private final Random random = new Random(); @@ -188,34 +189,6 @@ private void waitForDbOperations(String backupId) throws InterruptedException { } } - private String getUniqueBackupId() { - return String.format("testbck_%06d_%04d", random.nextInt(1000000), backupSeq.incrementAndGet()); - } - - private static Timestamp after7Days() { - return Timestamp.ofTimeMicroseconds( - TimeUnit.MICROSECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS) - + TimeUnit.MICROSECONDS.convert(7L, TimeUnit.DAYS)); - } - - private Timestamp after5Minutes() { - return Timestamp.ofTimeMicroseconds( - TimeUnit.MICROSECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS) - + TimeUnit.MICROSECONDS.convert(5L, TimeUnit.MINUTES)); - } - - private Timestamp tomorrow() { - return Timestamp.ofTimeMicroseconds( - TimeUnit.MICROSECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS) - + TimeUnit.MICROSECONDS.convert(1L, TimeUnit.DAYS)); - } - - private Timestamp yesterday() { - return Timestamp.ofTimeMicroseconds( - TimeUnit.MICROSECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS) - - TimeUnit.MICROSECONDS.convert(1L, TimeUnit.DAYS)); - } - @Test public void testBackups() throws InterruptedException, ExecutionException { // Create two test databases in parallel. @@ -250,9 +223,9 @@ public void testBackups() throws InterruptedException, ExecutionException { .build())); // Create two backups in parallel. - String backupId1 = getUniqueBackupId() + "_bck1"; - String backupId2 = getUniqueBackupId() + "_bck2"; - Timestamp expireTime = after7Days(); + String backupId1 = testHelper.getUniqueBackupId() + "_bck1"; + String backupId2 = testHelper.getUniqueBackupId() + "_bck2"; + Timestamp expireTime = afterDays(7); logger.info(String.format("Creating backups %s and %s in parallel", backupId1, backupId2)); OperationFuture op1 = dbAdminClient.createBackup( @@ -414,8 +387,8 @@ private void testMetadata( private void testCreateInvalidExpirationDate(Database db) throws InterruptedException { // This is not allowed, the expiration date must be at least 6 hours in the future. - Timestamp expireTime = yesterday(); - String backupId = getUniqueBackupId(); + Timestamp expireTime = daysAgo(1); + String backupId = testHelper.getUniqueBackupId(); logger.info(String.format("Creating backup %s with invalid expiration date", backupId)); OperationFuture op = dbAdminClient.createBackup( @@ -437,8 +410,8 @@ private void testCreateInvalidExpirationDate(Database db) throws InterruptedExce private void testCancelBackupOperation(Database db) throws InterruptedException, ExecutionException { - Timestamp expireTime = after7Days(); - String backupId = getUniqueBackupId(); + Timestamp expireTime = afterDays(7); + String backupId = testHelper.getUniqueBackupId(); logger.info(String.format("Starting to create backup %s", backupId)); OperationFuture op = dbAdminClient.createBackup( @@ -477,7 +450,7 @@ private void testGetBackup(Database db, String backupId, Timestamp expireTime) { private void testUpdateBackup(Backup backup) { // Update the expire time. - Timestamp tomorrow = tomorrow(); + Timestamp tomorrow = afterDays(1); backup = backup.toBuilder().setExpireTime(tomorrow).build(); logger.info( String.format("Updating expire time of backup %s to 1 week", backup.getId().getBackup())); @@ -488,7 +461,7 @@ private void testUpdateBackup(Backup backup) { assertThat(backup.getExpireTime()).isEqualTo(tomorrow); // Try to set the expire time to 5 minutes in the future. - Timestamp in5Minutes = after5Minutes(); + Timestamp in5Minutes = afterMinutes(5); backup = backup.toBuilder().setExpireTime(in5Minutes).build(); try { logger.info( diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITPitrBackupAndRestore.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITPitrBackupAndRestore.java new file mode 100644 index 0000000000..bfdaa60a25 --- /dev/null +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITPitrBackupAndRestore.java @@ -0,0 +1,244 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner.it; + +import static com.google.cloud.spanner.testing.EmulatorSpannerHelper.isUsingEmulator; +import static com.google.cloud.spanner.testing.TimestampHelper.afterDays; +import static com.google.cloud.spanner.testing.TimestampHelper.daysAgo; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assume.assumeFalse; + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.api.gax.paging.Page; +import com.google.cloud.Timestamp; +import com.google.cloud.spanner.Backup; +import com.google.cloud.spanner.BackupId; +import com.google.cloud.spanner.Database; +import com.google.cloud.spanner.DatabaseAdminClient; +import com.google.cloud.spanner.DatabaseId; +import com.google.cloud.spanner.InstanceId; +import com.google.cloud.spanner.IntegrationTestEnv; +import com.google.cloud.spanner.ParallelIntegrationTest; +import com.google.cloud.spanner.SpannerException; +import com.google.cloud.spanner.testing.RemoteSpannerHelper; +import com.google.spanner.admin.database.v1.CreateDatabaseMetadata; +import com.google.spanner.admin.database.v1.RestoreDatabaseMetadata; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@Category(ParallelIntegrationTest.class) +@RunWith(JUnit4.class) +public class ITPitrBackupAndRestore { + private static final Logger logger = Logger.getLogger(ITPitrBackupAndRestore.class.getName()); + + @ClassRule public static IntegrationTestEnv env = new IntegrationTestEnv(); + private static final long OP_TIMEOUT = 10; + private static final TimeUnit OP_TIMEOUT_UNIT = TimeUnit.MINUTES; + private static RemoteSpannerHelper testHelper; + private static DatabaseAdminClient dbAdminClient; + private static Database testDatabase; + private static final List backupsToDrop = new ArrayList<>(); + private static final List databasesToDrop = new ArrayList<>(); + + @BeforeClass + public static void doNotRunOnEmulator() { + assumeFalse("PITR features are not supported by the emulator", isUsingEmulator()); + } + + @BeforeClass + public static void setUp() throws Exception { + testHelper = env.getTestHelper(); + dbAdminClient = testHelper.getClient().getDatabaseAdminClient(); + testDatabase = createTestDatabase(); + } + + @AfterClass + public static void tearDown() { + int numDropped = 0; + for (Database database : databasesToDrop) { + try { + database.drop(); + numDropped++; + } catch (SpannerException e) { + logger.log(Level.SEVERE, "Failed to drop test database " + database.getId(), e); + } + } + logger.log(Level.INFO, "Dropped {0} test databases(s)", numDropped); + + numDropped = 0; + for (Backup backup : backupsToDrop) { + try { + backup.delete(); + numDropped++; + } catch (SpannerException e) { + logger.log(Level.SEVERE, "Failed to drop test backup " + backup.getId(), e); + } + } + logger.log(Level.INFO, "Dropped {0} test backup(s)", numDropped); + } + + @Test + @Ignore("backup and restore for pitr is not released yet") + public void backupCreationWithVersionTimeWithinVersionRetentionPeriodSucceeds() throws Exception { + final DatabaseId backupDatabaseId = testDatabase.getId(); + final String restoreDatabaseId = testHelper.getUniqueDatabaseId(); + final String projectId = backupDatabaseId.getInstanceId().getProject(); + final String instanceId = backupDatabaseId.getInstanceId().getInstance(); + final String backupId = testHelper.getUniqueBackupId(); + final Timestamp expireTime = afterDays(7); + final Timestamp versionTime = testDatabase.getEarliestVersionTime(); + final Backup backupToCreate = + dbAdminClient + .newBackupBuilder(BackupId.of(projectId, instanceId, backupId)) + .setDatabase(backupDatabaseId) + .setExpireTime(expireTime) + .setVersionTime(versionTime) + .build(); + + final Backup createdBackup = createBackup(backupToCreate); + assertThat(createdBackup.getVersionTime()).isEqualTo(versionTime); + + final RestoreDatabaseMetadata restoreDatabaseMetadata = + restoreDatabase(instanceId, backupId, restoreDatabaseId); + assertThat(Timestamp.fromProto(restoreDatabaseMetadata.getBackupInfo().getVersionTime())) + .isEqualTo(versionTime); + + final Database retrievedDatabase = dbAdminClient.getDatabase(instanceId, restoreDatabaseId); + assertThat(retrievedDatabase).isNotNull(); + assertThat( + Timestamp.fromProto( + retrievedDatabase.getRestoreInfo().getProto().getBackupInfo().getVersionTime())) + .isEqualTo(versionTime); + + final Database listedDatabase = listDatabase(instanceId, restoreDatabaseId); + assertThat(listedDatabase).isNotNull(); + assertThat( + Timestamp.fromProto( + listedDatabase.getRestoreInfo().getProto().getBackupInfo().getVersionTime())) + .isEqualTo(versionTime); + } + + @Test(expected = SpannerException.class) + @Ignore("backup and restore for pitr is not released yet") + public void backupCreationWithVersionTimeTooFarInThePastFails() throws Exception { + final DatabaseId databaseId = testDatabase.getId(); + final InstanceId instanceId = databaseId.getInstanceId(); + final String backupId = testHelper.getUniqueBackupId(); + final Timestamp expireTime = afterDays(7); + final Timestamp versionTime = daysAgo(30); + final Backup backupToCreate = + dbAdminClient + .newBackupBuilder(BackupId.of(instanceId, backupId)) + .setDatabase(databaseId) + .setExpireTime(expireTime) + .setVersionTime(versionTime) + .build(); + + createBackup(backupToCreate); + } + + @Test(expected = SpannerException.class) + @Ignore("backup and restore for pitr is not released yet") + public void backupCreationWithVersionTimeInTheFutureFails() throws Exception { + final DatabaseId databaseId = testDatabase.getId(); + final InstanceId instanceId = databaseId.getInstanceId(); + final String backupId = testHelper.getUniqueBackupId(); + final Timestamp expireTime = afterDays(7); + final Timestamp versionTime = afterDays(1); + final Backup backupToCreate = + dbAdminClient + .newBackupBuilder(BackupId.of(instanceId, backupId)) + .setDatabase(databaseId) + .setExpireTime(expireTime) + .setVersionTime(versionTime) + .build(); + + createBackup(backupToCreate); + } + + private Backup createBackup(Backup backupToCreate) + throws InterruptedException, ExecutionException, TimeoutException { + final Backup createdBackup = getOrThrow(dbAdminClient.createBackup(backupToCreate)); + backupsToDrop.add(createdBackup); + return createdBackup; + } + + private RestoreDatabaseMetadata restoreDatabase( + String instanceId, String backupId, String databaseId) + throws InterruptedException, ExecutionException, TimeoutException { + final OperationFuture op = + dbAdminClient.restoreDatabase(instanceId, backupId, instanceId, databaseId); + final Database database = getOrThrow(op); + databasesToDrop.add(database); + return op.getMetadata().get(OP_TIMEOUT, OP_TIMEOUT_UNIT); + } + + private Database listDatabase(String instanceId, String databaseId) { + Page page = dbAdminClient.listDatabases(instanceId); + while (page != null) { + for (Database database : page.getValues()) { + if (database.getId().getDatabase().equals(databaseId)) { + return database; + } + } + page = page.getNextPage(); + } + return null; + } + + private static Database createTestDatabase() + throws InterruptedException, ExecutionException, TimeoutException { + final String instanceId = testHelper.getInstanceId().getInstance(); + final String databaseId = testHelper.getUniqueDatabaseId(); + final OperationFuture op = + dbAdminClient.createDatabase( + instanceId, + databaseId, + Collections.singletonList( + "ALTER DATABASE " + databaseId + " SET OPTIONS (version_retention_period = '7d')")); + final Database database = getOrThrow(op); + databasesToDrop.add(database); + return database; + } + + private static T getOrThrow(OperationFuture op) + throws TimeoutException, InterruptedException, ExecutionException { + try { + return op.get(OP_TIMEOUT, OP_TIMEOUT_UNIT); + } catch (ExecutionException e) { + if (e.getCause() != null && e.getCause() instanceof SpannerException) { + throw (SpannerException) e.getCause(); + } else { + throw e; + } + } + } +} diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITPitrCreateDatabaseTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITPitrCreateDatabaseTest.java new file mode 100644 index 0000000000..f9fa081e41 --- /dev/null +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITPitrCreateDatabaseTest.java @@ -0,0 +1,144 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner.it; + +import static com.google.cloud.spanner.testing.EmulatorSpannerHelper.isUsingEmulator; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; +import static org.junit.Assume.assumeFalse; + +import com.google.cloud.spanner.Database; +import com.google.cloud.spanner.DatabaseAdminClient; +import com.google.cloud.spanner.DatabaseId; +import com.google.cloud.spanner.DatabaseNotFoundException; +import com.google.cloud.spanner.ErrorCode; +import com.google.cloud.spanner.IntegrationTestEnv; +import com.google.cloud.spanner.ParallelIntegrationTest; +import com.google.cloud.spanner.SpannerException; +import com.google.cloud.spanner.testing.RemoteSpannerHelper; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.threeten.bp.Duration; + +@Category(ParallelIntegrationTest.class) +@RunWith(JUnit4.class) +public class ITPitrCreateDatabaseTest { + + private static final Duration OPERATION_TIMEOUT = Duration.ofMinutes(2); + private static final String VERSION_RETENTION_PERIOD = "7d"; + + @ClassRule public static IntegrationTestEnv env = new IntegrationTestEnv(); + private RemoteSpannerHelper testHelper; + private DatabaseAdminClient dbAdminClient; + private List databasesToDrop; + + @BeforeClass + public static void doNotRunOnEmulator() { + assumeFalse("PITR-lite features are not supported by the emulator", isUsingEmulator()); + } + + @Before + public void setUp() { + testHelper = env.getTestHelper(); + dbAdminClient = testHelper.getClient().getDatabaseAdminClient(); + databasesToDrop = new ArrayList<>(); + } + + @After + public void tearDown() { + for (Database database : databasesToDrop) { + final DatabaseId id = database.getId(); + dbAdminClient.dropDatabase(id.getInstanceId().getInstance(), id.getDatabase()); + } + } + + @Test + public void returnsTheVersionRetentionPeriodSetThroughCreateDatabase() throws Exception { + final String instanceId = testHelper.getInstanceId().getInstance(); + final String databaseId = testHelper.getUniqueDatabaseId(); + final String extraStatement = + "ALTER DATABASE " + + databaseId + + " SET OPTIONS (version_retention_period = '" + + VERSION_RETENTION_PERIOD + + "')"; + + final Database database = createDatabase(instanceId, databaseId, extraStatement); + + assertThat(database.getVersionRetentionPeriod()).isEqualTo(VERSION_RETENTION_PERIOD); + assertThat(database.getEarliestVersionTime()).isNotNull(); + } + + @Test + public void returnsTheVersionRetentionPeriodSetThroughGetDatabase() throws Exception { + final String instanceId = testHelper.getInstanceId().getInstance(); + final String databaseId = testHelper.getUniqueDatabaseId(); + final String extraStatement = + "ALTER DATABASE " + + databaseId + + " SET OPTIONS (version_retention_period = '" + + VERSION_RETENTION_PERIOD + + "')"; + + createDatabase(instanceId, databaseId, extraStatement); + final Database database = dbAdminClient.getDatabase(instanceId, databaseId); + + assertThat(database.getVersionRetentionPeriod()).isEqualTo(VERSION_RETENTION_PERIOD); + assertThat(database.getEarliestVersionTime()).isNotNull(); + } + + @Test(expected = DatabaseNotFoundException.class) + public void returnsAnErrorWhenAnInvalidVersionRetentionPeriodIsGiven() { + final String instanceId = testHelper.getInstanceId().getInstance(); + final String databaseId = testHelper.getUniqueDatabaseId(); + final String extraStatement = + "ALTER DATABASE " + databaseId + " SET OPTIONS (version_retention_period = '0d')"; + + try { + createDatabase(instanceId, databaseId, extraStatement); + fail("Expected invalid argument error when setting invalid version retention period"); + } catch (Exception e) { + SpannerException spannerException = (SpannerException) e.getCause(); + assertThat(spannerException.getErrorCode()).isEqualTo(ErrorCode.INVALID_ARGUMENT); + } + + // Expects a database not found exception + dbAdminClient.getDatabase(instanceId, databaseId); + } + + private Database createDatabase( + final String instanceId, final String databaseId, final String extraStatement) + throws Exception { + final Database database = + dbAdminClient + .createDatabase(instanceId, databaseId, Collections.singletonList(extraStatement)) + .get(OPERATION_TIMEOUT.toNanos(), TimeUnit.NANOSECONDS); + databasesToDrop.add(database); + + return database; + } +} diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITPitrUpdateDatabaseTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITPitrUpdateDatabaseTest.java new file mode 100644 index 0000000000..72a52b3907 --- /dev/null +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/it/ITPitrUpdateDatabaseTest.java @@ -0,0 +1,207 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner.it; + +import static com.google.cloud.spanner.testing.EmulatorSpannerHelper.isUsingEmulator; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; +import static org.junit.Assume.assumeFalse; + +import com.google.api.gax.longrunning.OperationFuture; +import com.google.api.gax.paging.Page; +import com.google.cloud.spanner.Database; +import com.google.cloud.spanner.DatabaseAdminClient; +import com.google.cloud.spanner.DatabaseClient; +import com.google.cloud.spanner.DatabaseId; +import com.google.cloud.spanner.ErrorCode; +import com.google.cloud.spanner.IntegrationTestEnv; +import com.google.cloud.spanner.ParallelIntegrationTest; +import com.google.cloud.spanner.ResultSet; +import com.google.cloud.spanner.SpannerException; +import com.google.cloud.spanner.Statement; +import com.google.cloud.spanner.testing.RemoteSpannerHelper; +import com.google.spanner.admin.database.v1.UpdateDatabaseDdlMetadata; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.threeten.bp.Duration; + +@Category(ParallelIntegrationTest.class) +@RunWith(JUnit4.class) +public class ITPitrUpdateDatabaseTest { + + private static final Duration OPERATION_TIMEOUT = Duration.ofMinutes(2); + private static final String VERSION_RETENTION_PERIOD = "7d"; + + @ClassRule public static IntegrationTestEnv env = new IntegrationTestEnv(); + private static DatabaseAdminClient dbAdminClient; + private static DatabaseClient dbClient; + private static String instanceId; + private static String databaseId; + private static UpdateDatabaseDdlMetadata metadata; + + @BeforeClass + public static void setUp() throws Exception { + assumeFalse("PITR-lite features are not supported by the emulator", isUsingEmulator()); + + final RemoteSpannerHelper testHelper = env.getTestHelper(); + final String projectId = testHelper.getOptions().getProjectId(); + instanceId = testHelper.getInstanceId().getInstance(); + databaseId = testHelper.getUniqueDatabaseId(); + dbAdminClient = testHelper.getClient().getDatabaseAdminClient(); + + createDatabase(dbAdminClient, instanceId, databaseId, Collections.emptyList()); + metadata = + updateVersionRetentionPeriod( + dbAdminClient, instanceId, databaseId, VERSION_RETENTION_PERIOD); + + dbClient = + testHelper.getClient().getDatabaseClient(DatabaseId.of(projectId, instanceId, databaseId)); + } + + @AfterClass + public static void tearDown() { + if (!isUsingEmulator()) { + dbAdminClient.dropDatabase(instanceId, databaseId); + } + } + + @Test + public void checksThatTheOperationWasNotThrottled() { + assertThat(metadata.getThrottled()).isFalse(); + } + + @Test + public void returnsTheVersionRetentionPeriodSetThroughGetDatabase() { + final Database database = dbAdminClient.getDatabase(instanceId, databaseId); + + assertThat(database.getVersionRetentionPeriod()).isEqualTo(VERSION_RETENTION_PERIOD); + assertThat(database.getEarliestVersionTime()).isNotNull(); + } + + @Test + public void returnsTheVersionRetentionPeriodSetThroughListDatabases() { + final Page page = dbAdminClient.listDatabases(instanceId); + + for (Database database : page.iterateAll()) { + if (!database.getId().getDatabase().equals(databaseId)) { + continue; + } + assertThat(database.getVersionRetentionPeriod()).isEqualTo(VERSION_RETENTION_PERIOD); + assertThat(database.getEarliestVersionTime()).isNotNull(); + } + } + + @Test + public void returnsTheVersionRetentionPeriodSetThroughGetDatabaseDdl() { + final List ddls = dbAdminClient.getDatabaseDdl(instanceId, databaseId); + + boolean hasVersionRetentionPeriodStatement = false; + for (String ddl : ddls) { + hasVersionRetentionPeriodStatement = + ddl.contains("version_retention_period = '" + VERSION_RETENTION_PERIOD + "'"); + if (hasVersionRetentionPeriodStatement) { + break; + } + } + assertThat(hasVersionRetentionPeriodStatement).isTrue(); + } + + @Test + public void returnsTheVersionRetentionPeriodSetThroughInformationSchema() { + try (final ResultSet rs = + dbClient + .singleUse() + .executeQuery( + Statement.of( + "SELECT OPTION_VALUE AS version_retention_period " + + "FROM INFORMATION_SCHEMA.DATABASE_OPTIONS " + + "WHERE SCHEMA_NAME = '' AND OPTION_NAME = 'version_retention_period'"))) { + + String versionRetentionPeriod = null; + while (rs.next()) { + versionRetentionPeriod = rs.getString("version_retention_period"); + } + + assertThat(versionRetentionPeriod).isEqualTo(VERSION_RETENTION_PERIOD); + } + } + + @Test + public void returnsAnErrorWhenAnInvalidRetentionPeriodIsGiven() { + try { + dbAdminClient + .updateDatabaseDdl( + instanceId, + databaseId, + Collections.singletonList( + "ALTER DATABASE " + + databaseId + + " SET OPTIONS (version_retention_period = '0d')"), + "op_invalid_retention_period_" + databaseId) + .get(OPERATION_TIMEOUT.toNanos(), TimeUnit.NANOSECONDS); + fail("Expected invalid argument error when setting invalid version retention period"); + } catch (Exception e) { + SpannerException spannerException = (SpannerException) e.getCause(); + assertThat(spannerException.getErrorCode()).isEqualTo(ErrorCode.INVALID_ARGUMENT); + } + + final Database database = dbAdminClient.getDatabase(instanceId, databaseId); + + assertThat(database.getVersionRetentionPeriod()).isEqualTo(VERSION_RETENTION_PERIOD); + assertThat(database.getEarliestVersionTime()).isNotNull(); + } + + private static Database createDatabase( + final DatabaseAdminClient dbAdminClient, + final String instanceId, + final String databaseId, + final Iterable extraStatements) + throws Exception { + return dbAdminClient + .createDatabase(instanceId, databaseId, extraStatements) + .get(OPERATION_TIMEOUT.toNanos(), TimeUnit.NANOSECONDS); + } + + private static UpdateDatabaseDdlMetadata updateVersionRetentionPeriod( + final DatabaseAdminClient dbAdminClient, + final String instanceId, + final String databaseId, + final String versionRetentionPeriod) + throws Exception { + final OperationFuture op = + dbAdminClient.updateDatabaseDdl( + instanceId, + databaseId, + Collections.singletonList( + "ALTER DATABASE " + + databaseId + + " SET OPTIONS ( version_retention_period = '" + + versionRetentionPeriod + + "' )"), + "updateddl_version_retention_period"); + op.get(OPERATION_TIMEOUT.toNanos(), TimeUnit.NANOSECONDS); + return op.getMetadata().get(OPERATION_TIMEOUT.toNanos(), TimeUnit.NANOSECONDS); + } +}