Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
feat: adding utility to transform protobuf into model object (#299)
* feat: adding utility to transform protobuf into model object

With this commit, User shall be able to transform protobuf object to bigtable client's specific model objects.

* chore: addressed feedback comments

 - updated the JavaDoc
 - marked the utility as `@BetaApi`

* chore: addressed feedback comments

 - Rephrased Javadoc to include more explanation
 - change `Mutation#fromProto` visibility to package only

* chore: removed VisibleForTesting annotation

As Mutation#fromProto is being used by RowMutation#fromProto, So removed `@VisibleForTesting` annotation

* chore: extended unit tests to verify project and instance override
  • Loading branch information
rahulKQL committed May 14, 2020
1 parent bc215c5 commit 00f6d2d
Show file tree
Hide file tree
Showing 10 changed files with 205 additions and 0 deletions.
Expand Up @@ -15,6 +15,7 @@
*/
package com.google.cloud.bigtable.data.v2.models;

import com.google.api.core.BetaApi;
import com.google.api.core.InternalApi;
import com.google.bigtable.v2.MutateRowsRequest;
import com.google.cloud.bigtable.data.v2.internal.NameUtil;
Expand Down Expand Up @@ -114,6 +115,26 @@ public MutateRowsRequest toProto(RequestContext requestContext) {
.build();
}

/**
* Wraps the protobuf {@link MutateRowsRequest}.
*
* <p>This is meant for advanced usage only. Please ensure that the MutateRowsRequest does not use
* server side timestamps. The BigtableDataClient assumes that mutation present in BulkMutation
* are idempotent and is configured to enable retries by default. If serverside timestamps are
* enabled then that can lead to duplicate mutations.
*
* <p>WARNING: when applied, the resulting mutation object will ignore the project id and instance
* id in the table_name and instead apply the configuration in the client.
*/
@BetaApi
public static BulkMutation fromProto(@Nonnull MutateRowsRequest request) {
BulkMutation bulkMutation =
BulkMutation.create(NameUtil.extractTableIdFromTableName(request.getTableName()));
bulkMutation.builder = request.toBuilder();

return bulkMutation;
}

/** Creates a copy of {@link BulkMutation}. */
@Override
public BulkMutation clone() {
Expand Down
Expand Up @@ -15,6 +15,7 @@
*/
package com.google.cloud.bigtable.data.v2.models;

import com.google.api.core.BetaApi;
import com.google.api.core.InternalApi;
import com.google.bigtable.v2.CheckAndMutateRowRequest;
import com.google.cloud.bigtable.data.v2.internal.NameUtil;
Expand Down Expand Up @@ -136,4 +137,20 @@ public CheckAndMutateRowRequest toProto(RequestContext requestContext) {
.setAppProfileId(requestContext.getAppProfileId())
.build();
}

/**
* Wraps the protobuf {@link CheckAndMutateRowRequest}.
*
* <p>WARNING: Please note that the table_name will be overwritten by the configuration in the
* BigtableDataClient.
*/
@BetaApi
public static ConditionalRowMutation fromProto(@Nonnull CheckAndMutateRowRequest request) {
String tableId = NameUtil.extractTableIdFromTableName(request.getTableName());
ConditionalRowMutation rowMutation =
ConditionalRowMutation.create(tableId, request.getRowKey());
rowMutation.builder = request.toBuilder();

return rowMutation;
}
}
Expand Up @@ -87,6 +87,22 @@ public static Mutation fromProtoUnsafe(List<com.google.bigtable.v2.Mutation> pro
return mutation;
}

/**
* Constructs a row mutation from an existing protobuf object.
*
* <p>Callers must ensure that the protobuf argument is not using serverside timestamps. The
* client assumes that all mutations are idempotent and will retry in case of transient errors.
* This can lead to row duplication.
*
* <p>When applied, the resulting Mutation object will ignore the project id and instance id in
* the table_name and instead apply the configuration in the client
*/
static Mutation fromProto(List<com.google.bigtable.v2.Mutation> protos) {
Mutation mutation = new Mutation(false);
mutation.mutations.addAll(protos);
return mutation;
}

private Mutation(boolean allowServersideTimestamp) {
this.allowServersideTimestamp = allowServersideTimestamp;
}
Expand Down
Expand Up @@ -15,6 +15,7 @@
*/
package com.google.cloud.bigtable.data.v2.models;

import com.google.api.core.BetaApi;
import com.google.api.core.InternalApi;
import com.google.bigtable.v2.ReadModifyWriteRowRequest;
import com.google.bigtable.v2.ReadModifyWriteRule;
Expand Down Expand Up @@ -137,4 +138,20 @@ public ReadModifyWriteRowRequest toProto(RequestContext requestContext) {
.setAppProfileId(requestContext.getAppProfileId())
.build();
}

/**
* Wraps the protobuf {@link ReadModifyWriteRowRequest}.
*
* <p>WARNING: Please note that the table_name will be overwritten by the configuration in the
* BigtableDataClient.
*/
@BetaApi
public static ReadModifyWriteRow fromProto(@Nonnull ReadModifyWriteRowRequest request) {
String tableId = NameUtil.extractTableIdFromTableName(request.getTableName());

ReadModifyWriteRow row = ReadModifyWriteRow.create(tableId, request.getRowKey());
row.builder = request.toBuilder();

return row;
}
}
Expand Up @@ -15,6 +15,7 @@
*/
package com.google.cloud.bigtable.data.v2.models;

import com.google.api.core.BetaApi;
import com.google.api.core.InternalApi;
import com.google.bigtable.v2.MutateRowRequest;
import com.google.bigtable.v2.MutateRowsRequest;
Expand Down Expand Up @@ -214,4 +215,23 @@ public MutateRowsRequest toBulkProto(RequestContext requestContext) {
Entry.newBuilder().setRowKey(key).addAllMutations(mutation.getMutations()).build())
.build();
}

/**
* Wraps the protobuf {@link MutateRowRequest}.
*
* <p>This is meant for advanced usage only. Please ensure that the MutateRowRequest does not use
* server side timestamps. The BigtableDataClient assumes that RowMutations are idempotent and is
* configured to enable retries by default. If serverside timestamps are enabled, this can lead to
* duplicate mutations.
*
* <p>WARNING: when applied, the resulting mutation object will ignore the project id and instance
* id in the table_name and instead apply the configuration in the client.
*/
@BetaApi
public static RowMutation fromProto(@Nonnull MutateRowRequest request) {
String tableId = NameUtil.extractTableIdFromTableName(request.getTableName());

return RowMutation.create(
tableId, request.getRowKey(), Mutation.fromProto(request.getMutationsList()));
}
}
Expand Up @@ -147,4 +147,29 @@ public void addRowMutationEntry() {
bulkMutation.add(entry);
assertThat(bulkMutation.toProto(REQUEST_CONTEXT).getEntriesList()).contains(entry.toProto());
}

@Test
public void fromProtoTest() {
BulkMutation expected =
BulkMutation.create(TABLE_ID)
.add(
"key",
Mutation.create().setCell("fake-family", "fake-qualifier", 10_000L, "fake-value"));

MutateRowsRequest protoRequest = expected.toProto(REQUEST_CONTEXT);
BulkMutation actualBulkMutation = BulkMutation.fromProto(protoRequest);

assertThat(actualBulkMutation.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest);

String projectId = "fresh-project";
String instanceId = "fresh-instance";
String appProfile = "fresh-app-profile";
MutateRowsRequest overriddenRequest =
actualBulkMutation.toProto(RequestContext.create(projectId, instanceId, appProfile));

assertThat(overriddenRequest).isNotEqualTo(protoRequest);
assertThat(overriddenRequest.getTableName())
.matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID));
assertThat(overriddenRequest.getAppProfileId()).matches(appProfile);
}
}
Expand Up @@ -162,4 +162,29 @@ public void serializationTest() throws IOException, ClassNotFoundException {
ConditionalRowMutation actual = (ConditionalRowMutation) ois.readObject();
assertThat(actual.toProto(REQUEST_CONTEXT)).isEqualTo(expected.toProto(REQUEST_CONTEXT));
}

@Test
public void fromProtoTest() {
ConditionalRowMutation mutation =
ConditionalRowMutation.create(TABLE_ID, TEST_KEY)
.condition(Filters.FILTERS.key().regex("test"))
.then(Mutation.create().setCell("family1", "qualifier1", 10_000L, "value"))
.otherwise(Mutation.create().deleteFamily("family"));

CheckAndMutateRowRequest protoRequest = mutation.toProto(REQUEST_CONTEXT);
ConditionalRowMutation actualRequest = ConditionalRowMutation.fromProto(protoRequest);

assertThat(actualRequest.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest);

String projectId = "fresh-project";
String instanceId = "fresh-instance";
String appProfile = "fresh-app-profile";
CheckAndMutateRowRequest overriddenRequest =
actualRequest.toProto(RequestContext.create(projectId, instanceId, appProfile));

assertThat(overriddenRequest).isNotEqualTo(protoRequest);
assertThat(overriddenRequest.getTableName())
.matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID));
assertThat(overriddenRequest.getAppProfileId()).matches(appProfile);
}
}
Expand Up @@ -254,4 +254,20 @@ public void testWithLongValue() {
.setValue(ByteString.copyFrom(Longs.toByteArray(20_000L)))
.build());
}

@Test
public void fromProtoTest() {
mutation
.setCell(
"fake-family",
ByteString.copyFromUtf8("fake-qualifier"),
1_000,
ByteString.copyFromUtf8("fake-value"))
.deleteCells("fake-family", ByteString.copyFromUtf8("fake-qualifier"))
.deleteFamily("fake-family2");

List<com.google.bigtable.v2.Mutation> protoMutation = mutation.getMutations();

assertThat(Mutation.fromProto(protoMutation).getMutations()).isEqualTo(protoMutation);
}
}
Expand Up @@ -113,4 +113,28 @@ public void serializationTest() throws IOException, ClassNotFoundException {
ReadModifyWriteRow actual = (ReadModifyWriteRow) ois.readObject();
assertThat(actual.toProto(REQUEST_CONTEXT)).isEqualTo(expected.toProto(REQUEST_CONTEXT));
}

@Test
public void fromProtoTest() {
ReadModifyWriteRow expected =
ReadModifyWriteRow.create(TABLE_ID, "row-key")
.increment("fake-family", ByteString.copyFromUtf8("fake-qualifier"), 1)
.append("fake-family", "fake-qualifier", "fake-value");

ReadModifyWriteRowRequest protoRequest = expected.toProto(REQUEST_CONTEXT);
ReadModifyWriteRow actualRequest = ReadModifyWriteRow.fromProto(protoRequest);

assertThat(actualRequest.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest);

String projectId = "fresh-project";
String instanceId = "fresh-instance";
String appProfile = "fresh-app-profile";
ReadModifyWriteRowRequest overriddenRequest =
actualRequest.toProto(RequestContext.create(projectId, instanceId, appProfile));

assertThat(overriddenRequest).isNotEqualTo(protoRequest);
assertThat(overriddenRequest.getTableName())
.matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID));
assertThat(overriddenRequest.getAppProfileId()).matches(appProfile);
}
}
Expand Up @@ -139,4 +139,28 @@ public void testWithLongValue() {
.setValue(ByteString.copyFrom(Longs.toByteArray(100_000L)))
.build());
}

@Test
public void fromProtoTest() {
RowMutation rowMutation =
RowMutation.create("fake-table", "fake-key")
.setCell("fake-family", "fake-qualifier-1", "fake-value")
.setCell("fake-family", "fake-qualifier-2", 30_000L, "fake-value-2");

MutateRowRequest protoRequest = rowMutation.toProto(REQUEST_CONTEXT);
RowMutation actualRequest = RowMutation.fromProto(protoRequest);

assertThat(actualRequest.toProto(REQUEST_CONTEXT)).isEqualTo(protoRequest);

String projectId = "fresh-project";
String instanceId = "fresh-instance";
String appProfile = "fresh-app-profile";
MutateRowRequest overriddenRequest =
actualRequest.toProto(RequestContext.create(projectId, instanceId, appProfile));

assertThat(overriddenRequest).isNotEqualTo(protoRequest);
assertThat(overriddenRequest.getTableName())
.matches(NameUtil.formatTableName(projectId, instanceId, TABLE_ID));
assertThat(overriddenRequest.getAppProfileId()).matches(appProfile);
}
}

0 comments on commit 00f6d2d

Please sign in to comment.