Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
feat!: customer-managed encryption keys for Spanner (#666)
* feat: add support for encrypted databases

* fix: fix deps and clirr failures

* tests: add additional tests for keys

* tests: remove IT and add unit

* fix: set null instead of default instance

* fix: does not set encryption info if null

Does not set encryption info in the request if it is null

* fix: fixes dependencies

* feature: adds support for encrypted backup

Adds the possibility to set encryption config info in the creation of a
backup.

* feature: adds support for restoring encrypted dbs

* Revert "tests: remove IT and add unit"

This reverts commit cc19cf2.

* fix: makes the setEncryptionConfigInfo public

This is so a backup can be encrypted

* feature: adds tests for cmek

Adds tests for creating encrypted database, creating encrypted backups
and restoring encrypted databases.

* fix: removes keys after test finishes

Destroy keys used in CMEK tests

* fix: fixes clirr errors

* fix: ignores failing cmek tests

Ignores the failing CMEK tests until the backend support is enabled in
production.

* fix: uses wrapper encryption info for backups

* fix: fixes clirr issues

* fix: re-orders clirr issues

* fix: addresses PR comments

* test: fixes database admin client tests

* chore: re-formats the code

* chore: fixes clirr checks

* tests: adds unit tests for domain classes

Adds unit tests for EncryptionConfigInfo, EncryptionConfig, Backup and
Restore.

* chore: renames EncryptionConfigInfo

Renames EncryptionConfigInfo to EncryptionConfig in order to mirror what
is the protobuf definition.

* tests: do not create a key on CMEK test

Instead use an existing key and fails if the key is not present.

* feat: allows multiple encryption configs

Allows customer managed encryption for create databases (google default
encryption is just nullifying the value here).
Allows customer managed encryption, google default encryption and
database encryption for create backups.
Allows customer managed encryption, google default encryption and backup
encryption for restore databases.

* docs: adds java doc to Restore class

* chore: refactors pom.xml

Uses variables to define project id and instance id for running
integration tests.

* test: fixes cmek integration test

* chore: fixes linting

* Revert "chore: refactors pom.xml"

This reverts commit d182b83.

* test: unifies cmek backup and restore tests

* chore: adds toString to encryption classes

* docs: updates DatabaseInfo javadoc

Co-authored-by: Knut Olav Løite <koloite@gmail.com>

* docs: updates Restore javadocs

Co-authored-by: Knut Olav Løite <koloite@gmail.com>

* docs: updates DatabaseInfo javadocs

Co-authored-by: Knut Olav Løite <koloite@gmail.com>

* fix: addresses PR comments

* tests: reformats

Co-authored-by: Olav Loite <koloite@gmail.com>
  • Loading branch information
thiagotnunes and olavloite committed Mar 18, 2021
1 parent 0837496 commit 8338116
Show file tree
Hide file tree
Showing 29 changed files with 1,614 additions and 165 deletions.
71 changes: 69 additions & 2 deletions google-cloud-spanner/clirr-ignored-differences.xml
Expand Up @@ -319,7 +319,7 @@
<className>com/google/cloud/spanner/Value</className>
<method>java.util.List getNumericArray()</method>
</difference>

<!-- Async Connection API -->
<difference>
<differenceType>7012</differenceType>
Expand Down Expand Up @@ -406,7 +406,7 @@
<className>com/google/cloud/spanner/AbstractLazyInitializer</className>
<method>java.lang.Object initialize()</method>
</difference>

<!-- TransactionOptions and UpdateOptions -->
<difference>
<differenceType>7004</differenceType>
Expand Down Expand Up @@ -504,4 +504,71 @@
<className>com/google/cloud/spanner/DatabaseAdminClient</className>
<method>com.google.api.gax.longrunning.OperationFuture createBackup(com.google.cloud.spanner.Backup)</method>
</difference>

<!-- Support creating encrypted databases -->
<difference>
<differenceType>7004</differenceType>
<className>com/google/cloud/spanner/spi/v1/SpannerRpc</className>
<method>com.google.api.gax.longrunning.OperationFuture createDatabase(java.lang.String, java.lang.String, java.lang.Iterable)</method>
</difference>
<difference>
<differenceType>7004</differenceType>
<className>com/google/cloud/spanner/spi/v1/SpannerRpc</className>
<method>com.google.api.gax.longrunning.OperationFuture createBackup(java.lang.String, java.lang.String, com.google.spanner.admin.database.v1.Backup)</method>
</difference>
<difference>
<differenceType>7004</differenceType>
<className>com/google/cloud/spanner/spi/v1/SpannerRpc</className>
<method>com.google.api.gax.longrunning.OperationFuture restoreDatabase(java.lang.String, java.lang.String, java.lang.String)</method>
</difference>
<difference>
<differenceType>7004</differenceType>
<className>com/google/cloud/spanner/spi/v1/GapicSpannerRpc</className>
<method>com.google.api.gax.longrunning.OperationFuture createDatabase(java.lang.String, java.lang.String, java.lang.Iterable)</method>
</difference>
<difference>
<differenceType>7004</differenceType>
<className>com/google/cloud/spanner/spi/v1/GapicSpannerRpc</className>
<method>com.google.api.gax.longrunning.OperationFuture createBackup(java.lang.String, java.lang.String, com.google.spanner.admin.database.v1.Backup)</method>
</difference>
<difference>
<differenceType>7004</differenceType>
<className>com/google/cloud/spanner/spi/v1/GapicSpannerRpc</className>
<method>com.google.api.gax.longrunning.OperationFuture restoreDatabase(java.lang.String, java.lang.String, java.lang.String)</method>
</difference>
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/spanner/DatabaseAdminClient</className>
<method>com.google.api.gax.longrunning.OperationFuture createDatabase(com.google.cloud.spanner.Database, java.lang.Iterable)</method>
</difference>
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/spanner/DatabaseAdminClient</className>
<method>com.google.api.gax.longrunning.OperationFuture createBackup(com.google.cloud.spanner.Backup)</method>
</difference>
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/spanner/DatabaseAdminClient</className>
<method>com.google.api.gax.longrunning.OperationFuture restoreDatabase(com.google.cloud.spanner.Restore)</method>
</difference>
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/spanner/DatabaseAdminClient</className>
<method>com.google.cloud.spanner.Database$Builder newDatabaseBuilder(com.google.cloud.spanner.DatabaseId)</method>
</difference>
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/spanner/DatabaseAdminClient</className>
<method>com.google.cloud.spanner.Restore$Builder newRestoreBuilder(com.google.cloud.spanner.BackupId, com.google.cloud.spanner.DatabaseId)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/spanner/DatabaseInfo$Builder</className>
<method>com.google.cloud.spanner.DatabaseInfo$Builder setEncryptionConfig(com.google.cloud.spanner.encryption.CustomerManagedEncryption)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/spanner/BackupInfo$Builder</className>
<method>com.google.cloud.spanner.BackupInfo$Builder setEncryptionConfig(com.google.cloud.spanner.encryption.BackupEncryptionConfig)</method>
</difference>
</differences>
3 changes: 2 additions & 1 deletion google-cloud-spanner/pom.xml
Expand Up @@ -73,6 +73,7 @@
<spanner.testenv.config.class>com.google.cloud.spanner.GceTestEnvConfig</spanner.testenv.config.class>
<spanner.testenv.instance>projects/gcloud-devel/instances/spanner-testing</spanner.testenv.instance>
<spanner.gce.config.project_id>gcloud-devel</spanner.gce.config.project_id>
<spanner.testenv.kms_key.name>projects/gcloud-devel/locations/us-central1/keyRings/spanner-test-keyring/cryptoKeys/spanner-test-key</spanner.testenv.kms_key.name>
</systemPropertyVariables>
<forkedProcessTimeoutInSeconds>3000</forkedProcessTimeoutInSeconds>
</configuration>
Expand Down Expand Up @@ -383,7 +384,7 @@
</build>
</profile>
<profile>
<!-- Profile for generating new sql test scripts. See ConnectionImplGeneratedSqlScriptTest
<!-- Profile for generating new sql test scripts. See ConnectionImplGeneratedSqlScriptTest
for more information. -->
<id>generate-test-sql-scripts</id>
<build>
Expand Down
Expand Up @@ -23,6 +23,7 @@
import com.google.api.gax.paging.Page;
import com.google.cloud.Policy;
import com.google.cloud.Timestamp;
import com.google.cloud.spanner.encryption.EncryptionInfo;
import com.google.longrunning.Operation;
import com.google.spanner.admin.database.v1.CreateBackupMetadata;
import com.google.spanner.admin.database.v1.RestoreDatabaseMetadata;
Expand Down Expand Up @@ -61,10 +62,6 @@ public Backup build() {

/** Creates a backup on the server based on the source of this {@link Backup} instance. */
public OperationFuture<Backup, CreateBackupMetadata> create() {
Preconditions.checkState(
getExpireTime() != null, "Cannot create a backup without an expire time");
Preconditions.checkState(
getDatabase() != null, "Cannot create a backup without a source database");
return dbClient.createBackup(this);
}

Expand Down Expand Up @@ -184,6 +181,7 @@ static Backup fromProto(
.setExpireTime(Timestamp.fromProto(proto.getExpireTime()))
.setVersionTime(Timestamp.fromProto(proto.getVersionTime()))
.setDatabase(DatabaseId.of(proto.getDatabase()))
.setEncryptionInfo(EncryptionInfo.fromProtoOrNull(proto.getEncryptionInfo()))
.setProto(proto)
.build();
}
Expand Down
Expand Up @@ -18,6 +18,8 @@

import com.google.api.client.util.Preconditions;
import com.google.cloud.Timestamp;
import com.google.cloud.spanner.encryption.BackupEncryptionConfig;
import com.google.cloud.spanner.encryption.EncryptionInfo;
import com.google.spanner.admin.database.v1.Database;
import java.util.Objects;
import javax.annotation.Nullable;
Expand All @@ -29,8 +31,29 @@ public abstract static class Builder {

abstract Builder setSize(long size);

/**
* Returned when retrieving a backup.
*
* <p>The encryption information for the backup. If the encryption key protecting this resource
* is customer managed, then kms_key_version will be filled.
*/
abstract Builder setEncryptionInfo(EncryptionInfo encryptionInfo);

abstract Builder setProto(com.google.spanner.admin.database.v1.Backup proto);

/**
* Optional for creating a new backup.
*
* <p>The encryption configuration to be used for the backup. The possible configurations are
* {@link com.google.cloud.spanner.encryption.CustomerManagedEncryption}, {@link
* com.google.cloud.spanner.encryption.GoogleDefaultEncryption} and {@link
* com.google.cloud.spanner.encryption.UseDatabaseEncryption}.
*
* <p>If no encryption config is given the backup will be created with the same encryption as
* set by the database ({@link com.google.cloud.spanner.encryption.UseDatabaseEncryption}).
*/
public abstract Builder setEncryptionConfig(BackupEncryptionConfig encryptionConfig);

/**
* Required for creating a new backup.
*
Expand Down Expand Up @@ -70,6 +93,8 @@ abstract static class BuilderImpl extends Builder {
private Timestamp versionTime;
private DatabaseId database;
private long size;
private BackupEncryptionConfig encryptionConfig;
private EncryptionInfo encryptionInfo;
private com.google.spanner.admin.database.v1.Backup proto;

BuilderImpl(BackupId id) {
Expand All @@ -83,6 +108,8 @@ abstract static class BuilderImpl extends Builder {
this.versionTime = other.versionTime;
this.database = other.database;
this.size = other.size;
this.encryptionConfig = other.encryptionConfig;
this.encryptionInfo = other.encryptionInfo;
this.proto = other.proto;
}

Expand Down Expand Up @@ -113,12 +140,24 @@ public Builder setDatabase(DatabaseId database) {
return this;
}

@Override
public Builder setEncryptionConfig(BackupEncryptionConfig encryptionConfig) {
this.encryptionConfig = encryptionConfig;
return this;
}

@Override
Builder setSize(long size) {
this.size = size;
return this;
}

@Override
Builder setEncryptionInfo(EncryptionInfo encryptionInfo) {
this.encryptionInfo = encryptionInfo;
return this;
}

@Override
Builder setProto(@Nullable com.google.spanner.admin.database.v1.Backup proto) {
this.proto = proto;
Expand All @@ -142,12 +181,16 @@ public enum State {
private final Timestamp versionTime;
private final DatabaseId database;
private final long size;
private final BackupEncryptionConfig encryptionConfig;
private final EncryptionInfo encryptionInfo;
private final com.google.spanner.admin.database.v1.Backup proto;

BackupInfo(BuilderImpl builder) {
this.id = builder.id;
this.state = builder.state;
this.size = builder.size;
this.encryptionConfig = builder.encryptionConfig;
this.encryptionInfo = builder.encryptionInfo;
this.expireTime = builder.expireTime;
this.versionTime = builder.versionTime;
this.database = builder.database;
Expand All @@ -174,6 +217,22 @@ public long getSize() {
return size;
}

/**
* Returns the {@link BackupEncryptionConfig} to encrypt the backup during its creation. Returns
* <code>null</code> if no customer-managed encryption key should be used.
*/
public BackupEncryptionConfig getEncryptionConfig() {
return encryptionConfig;
}

/**
* Returns the {@link EncryptionInfo} of the backup if the backup is encrypted, or <code>null
* </code> if this backup is not encrypted.
*/
public EncryptionInfo getEncryptionInfo() {
return encryptionInfo;
}

/** Returns the expire time of the backup. */
public Timestamp getExpireTime() {
return expireTime;
Expand Down Expand Up @@ -206,20 +265,30 @@ public boolean equals(Object o) {
return id.equals(that.id)
&& state == that.state
&& size == that.size
&& Objects.equals(encryptionConfig, that.encryptionConfig)
&& Objects.equals(encryptionInfo, that.encryptionInfo)
&& Objects.equals(expireTime, that.expireTime)
&& Objects.equals(versionTime, that.versionTime)
&& Objects.equals(database, that.database);
}

@Override
public int hashCode() {
return Objects.hash(id, state, size, expireTime, versionTime, database);
return Objects.hash(
id, state, size, encryptionConfig, encryptionInfo, expireTime, versionTime, database);
}

@Override
public String toString() {
return String.format(
"Backup[%s, %s, %d, %s, %s, %s]",
id.getName(), state, size, expireTime, versionTime, database);
"Backup[%s, %s, %d, %s, %s, %s, %s, %s]",
id.getName(),
state,
size,
encryptionConfig,
encryptionInfo,
expireTime,
versionTime,
database);
}
}
Expand Up @@ -22,6 +22,7 @@
import com.google.api.gax.paging.Page;
import com.google.cloud.Policy;
import com.google.cloud.Timestamp;
import com.google.cloud.spanner.encryption.CustomerManagedEncryption;
import com.google.common.base.Preconditions;
import com.google.longrunning.Operation;
import com.google.spanner.admin.database.v1.CreateBackupMetadata;
Expand Down Expand Up @@ -185,6 +186,7 @@ static Database fromProto(
.setRestoreInfo(RestoreInfo.fromProtoOrNullIfDefaultInstance(proto.getRestoreInfo()))
.setVersionRetentionPeriod(proto.getVersionRetentionPeriod())
.setEarliestVersionTime(Timestamp.fromProto(proto.getEarliestVersionTime()))
.setEncryptionConfig(CustomerManagedEncryption.fromProtoOrNull(proto.getEncryptionConfig()))
.setProto(proto)
.build();
}
Expand Down

0 comments on commit 8338116

Please sign in to comment.