From d6bfd30ea9484b7433a048bdaad66153ffaa6a79 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Mon, 6 Jan 2020 13:25:23 -0500 Subject: [PATCH] feat: add BigtableDataClientFactory to create lightweight data clients (#112) * feat: add BigtableDataClientFactory to create lightweight data clients The new factory allows users to construct a single heavy factory object that can create many lightweight clients. This is meant to be used in situations when a single application needs to access multiple instances or use different application profiles * code format * disclaimer comment --- .../data/v2/BigtableDataClientFactory.java | 204 +++++++++++++++ .../data/v2/BigtableDataSettings.java | 26 +- .../v2/stub/EnhancedBigtableStubSettings.java | 16 ++ .../v2/BigtableDataClientFactoryTest.java | 236 ++++++++++++++++++ 4 files changed, 478 insertions(+), 4 deletions(-) create mode 100644 google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactory.java create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactoryTest.java diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactory.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactory.java new file mode 100644 index 000000000..b08613cc6 --- /dev/null +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactory.java @@ -0,0 +1,204 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2; + +import com.google.api.core.BetaApi; +import com.google.api.gax.core.BackgroundResource; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.api.gax.core.FixedExecutorProvider; +import com.google.api.gax.rpc.ClientContext; +import com.google.api.gax.rpc.FixedHeaderProvider; +import com.google.api.gax.rpc.FixedTransportChannelProvider; +import com.google.api.gax.rpc.FixedWatchdogProvider; +import com.google.api.gax.rpc.StubSettings; +import java.io.IOException; +import javax.annotation.Nonnull; + +/** + * A factory to create multiple {@link BigtableDataClient} instances that all share the same channel + * pool. + * + *

This allows multiple client instances to share the same gRPC channel pool, which makes client + * creation very cheap. The intended use case is for applications that need to access multiple + * Bigtable Instances from the same process. + * + *

Example Usage: + * + * 

{@code
+ * BigtableDataSettings defaultSettings = BigtableDataSettings.newBuilder()
+ *   .setProject("my-default-project")
+ *   .setInstance("my-default-instance")
+ *   .build();
+ *
+ * BigtableDataClientFactory clientFactory = BigtableDataClientFactory.create(defaultSettings);
+ *
+ * // Create a new client for "my-default-instance" in "my-default-project";
+ * BigtableDataClient defaultInstanceClient = clientFactory.createDefault();
+ *
+ * // Create a new client for a different application profile
+ * BigtableDataClient otherAppProfileClient = clientFactory.createForAppProfile("other-app-profile");
+ *
+ * // Create a new client for a completely different instance and application profile.
+ * BigtableDataClient otherInstanceClient = clientFactory
+ *   .createForInstance("my-other-project", "my-other-instance", "my-other-app-profile");
+ *
+ * // Clean up: make sure close the clients AND the factory.
+ * defaultInstanceClient.close();
+ * otherAppProfileClient.close();
+ * otherInstanceClient.close();
+ *
+ * clientFactory.close();
+ *
+ * 
Please note that this is an experimental feature and might be changed or removed in future.
+ * }
+ */ +@BetaApi("This feature is currently experimental and can change in the future") +public final class BigtableDataClientFactory implements AutoCloseable { + private final BigtableDataSettings defaultSettings; + private final ClientContext sharedClientContext; + + /** + * Create a instance of this factory. + * + *

The factory will be used to create clients using the provided settings as the base. Make + * sure to call {@link #close()} on the factory after closing all clients. + */ + public static BigtableDataClientFactory create(BigtableDataSettings defaultSettings) + throws IOException { + ClientContext sharedClientContext = ClientContext.create(defaultSettings.getStubSettings()); + return new BigtableDataClientFactory(sharedClientContext, defaultSettings); + } + + private BigtableDataClientFactory( + ClientContext sharedClientContext, BigtableDataSettings defaultSettings) { + this.sharedClientContext = sharedClientContext; + this.defaultSettings = defaultSettings; + } + + /** + * Release all of the resources associated with this factory. + * + *

This will close the underlying channel pooling, disconnecting all create clients. + */ + @Override + public void close() throws Exception { + for (BackgroundResource resource : sharedClientContext.getBackgroundResources()) { + resource.close(); + } + } + + /** + * Create a lightweight client using the default settings in this factory. This will use the + * factory default project, instance and application profile ids. The client will also share + * resources like the channel pool with other clients created using this factory. + * + *

The client should be closed when it is no longer needed. Closing the client will release + * client specific resources, but will leave shared resources like the channel pool open. To + * release all resources, first close all of the created clients and then this factory instance. + */ + public BigtableDataClient createDefault() { + BigtableDataSettings.Builder settingsBuilder = defaultSettings.toBuilder(); + patchStubSettings(settingsBuilder.stubSettings()); + BigtableDataSettings settings = settingsBuilder.build(); + + try { + return BigtableDataClient.create(settings); + } catch (IOException e) { + // Should never happen because the connection has been established already + throw new RuntimeException( + "Failed to create a new client using factory default settings and shared resources."); + } + } + + /** + * Create a lightweight client with an overriden application profile and the factory default + * project and instance ids. The client will also share resources like the channel pool with other + * clients created using this factory. + * + *

The client should be closed when it is no longer needed. Closing the client will release + * client specific resources, but will leave shared resources like the channel pool open. To + * release all resources, first close all of the created clients and then this factory instance. + */ + public BigtableDataClient createForAppProfile(@Nonnull String appProfileId) throws IOException { + BigtableDataSettings.Builder settingsBuilder = + defaultSettings.toBuilder().setAppProfileId(appProfileId); + + patchStubSettings(settingsBuilder.stubSettings()); + + return BigtableDataClient.create(settingsBuilder.build()); + } + + /** + * Create a lightweight client with the specified project and instance id. The resulting client + * will use the server default application profile. The client will also share resources like the + * channel pool with other clients created using this factory. + * + *

The client should be closed when it is no longer needed. Closing the client will release + * client specific resources, but will leave shared resources like the channel pool open. To + * release all resources, first close all of the created clients and then this factory instance. + */ + public BigtableDataClient createForInstance(@Nonnull String projectId, @Nonnull String instanceId) + throws IOException { + BigtableDataSettings.Builder settingsBuilder = + defaultSettings + .toBuilder() + .setProjectId(projectId) + .setInstanceId(instanceId) + .setDefaultAppProfileId(); + + patchStubSettings(settingsBuilder.stubSettings()); + + return BigtableDataClient.create(settingsBuilder.build()); + } + + /** + * Create a lightweight client to the specified project, instance and application profile id. The + * client will share resources like the channel pool with other clients created using this + * factory. + * + *

The client should be closed when it is no longer needed. Closing the client will release + * client specific resources, but will leave shared resources like the channel pool open. To + * release all resources, first close all of the created clients and then this factory instance. + */ + public BigtableDataClient createForInstance( + @Nonnull String projectId, @Nonnull String instanceId, @Nonnull String appProfileId) + throws IOException { + BigtableDataSettings.Builder settingsBuilder = + defaultSettings + .toBuilder() + .setProjectId(projectId) + .setInstanceId(instanceId) + .setAppProfileId(appProfileId); + + patchStubSettings(settingsBuilder.stubSettings()); + + return BigtableDataClient.create(settingsBuilder.build()); + } + + // Update stub settings to use shared resources in this factory + private void patchStubSettings(StubSettings.Builder stubSettings) { + stubSettings + .setTransportChannelProvider( + FixedTransportChannelProvider.create(sharedClientContext.getTransportChannel())) + .setCredentialsProvider( + FixedCredentialsProvider.create(sharedClientContext.getCredentials())) + .setExecutorProvider(FixedExecutorProvider.create(sharedClientContext.getExecutor())) + .setStreamWatchdogProvider( + FixedWatchdogProvider.create(sharedClientContext.getStreamWatchdog())) + .setHeaderProvider(FixedHeaderProvider.create(sharedClientContext.getHeaders())) + .setClock(sharedClientContext.getClock()); + } +} diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java index ff9740880..437ebc653 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/BigtableDataSettings.java @@ -255,16 +255,34 @@ public String getInstanceId() { } /** - * Sets the AppProfile to use. An application profile (sometimes also shortened to "app - * profile") is a group of configuration parameters for an individual use case. A client will - * identify itself with an application profile ID at connection time, and the requests will be - * handled according to that application profile. + * Sets the AppProfile to use. + * + *

An application profile (sometimes also shortened to "app profile") is a group of + * configuration parameters for an individual use case. A client will identify itself with an + * application profile ID at connection time, and the requests will be handled according to that + * application profile. */ public Builder setAppProfileId(@Nonnull String appProfileId) { stubSettings.setAppProfileId(appProfileId); return this; } + /** + * Resets the AppProfile id to the default for the instance. + * + *

An application profile (sometimes also shortened to "app profile") is a group of + * configuration parameters for an individual use case. A client will identify itself with an + * application profile ID at connection time, and the requests will be handled according to that + * application profile. + * + *

Every Bigtable Instance has a default application profile associated with it, this method + * configures the client to use it. + */ + public Builder setDefaultAppProfileId() { + stubSettings.setDefaultAppProfileId(); + return this; + } + /** Gets the app profile id that was previously set on this Builder. */ public String getAppProfileId() { return stubSettings.getAppProfileId(); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java index 11d472608..4fc544061 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStubSettings.java @@ -610,6 +610,22 @@ public Builder setAppProfileId(@Nonnull String appProfileId) { return this; } + /** + * Resets the AppProfile id to the default for the instance. + * + *

An application profile (sometimes also shortened to "app profile") is a group of + * configuration parameters for an individual use case. A client will identify itself with an + * application profile ID at connection time, and the requests will be handled according to that + * application profile. + * + *

Every Bigtable Instance has a default application profile associated with it, this method + * configures the client to use it. + */ + public Builder setDefaultAppProfileId() { + setAppProfileId(SERVER_DEFAULT_APP_PROFILE_ID); + return this; + } + /** Gets the app profile id that was previously set on this Builder. */ public String getAppProfileId() { return appProfileId; diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactoryTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactoryTest.java new file mode 100644 index 000000000..612569fd2 --- /dev/null +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/BigtableDataClientFactoryTest.java @@ -0,0 +1,236 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.data.v2; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.core.ApiClock; +import com.google.api.gax.core.CredentialsProvider; +import com.google.api.gax.core.ExecutorProvider; +import com.google.api.gax.rpc.TransportChannelProvider; +import com.google.api.gax.rpc.WatchdogProvider; +import com.google.bigtable.v2.BigtableGrpc; +import com.google.bigtable.v2.MutateRowRequest; +import com.google.bigtable.v2.MutateRowResponse; +import com.google.cloud.bigtable.data.v2.internal.NameUtil; +import com.google.cloud.bigtable.data.v2.models.RowMutation; +import io.grpc.Server; +import io.grpc.ServerBuilder; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.ServerSocket; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; +import org.mockito.stubbing.Answer; + +@RunWith(JUnit4.class) +public class BigtableDataClientFactoryTest { + @Rule public MockitoRule mockitoRule = MockitoJUnit.rule(); + + private static final String DEFAULT_PROJECT_ID = "fake-project"; + private static final String DEFAULT_INSTANCE_ID = "fake-instance"; + private static final String DEFAULT_APP_PROFILE_ID = "fake-app-profile"; + + private Server fakeServer; + private FakeBigtableService service; + + private TransportChannelProvider transportChannelProvider; + private CredentialsProvider credentialsProvider; + private ExecutorProvider executorProvider; + private WatchdogProvider watchdogProvider; + private ApiClock apiClock; + private BigtableDataSettings defaultSettings; + + @Before + public void setUp() throws IOException { + service = new FakeBigtableService(); + + // Create a fake server for the client to connect to + final int port; + try (ServerSocket ss = new ServerSocket(0)) { + port = ss.getLocalPort(); + } + fakeServer = ServerBuilder.forPort(port).addService(service).build(); + fakeServer.start(); + + BigtableDataSettings.Builder builder = + BigtableDataSettings.newBuilderForEmulator(port) + .setProjectId(DEFAULT_PROJECT_ID) + .setInstanceId(DEFAULT_INSTANCE_ID) + .setAppProfileId(DEFAULT_APP_PROFILE_ID); + + transportChannelProvider = + Mockito.mock( + TransportChannelProvider.class, + new BuilderAnswer<>( + TransportChannelProvider.class, + builder.stubSettings().getTransportChannelProvider())); + + credentialsProvider = + Mockito.mock( + CredentialsProvider.class, + new BuilderAnswer<>( + CredentialsProvider.class, builder.stubSettings().getCredentialsProvider())); + + executorProvider = + Mockito.mock( + ExecutorProvider.class, + new BuilderAnswer<>( + ExecutorProvider.class, builder.stubSettings().getExecutorProvider())); + + watchdogProvider = + Mockito.mock( + WatchdogProvider.class, + new BuilderAnswer<>( + WatchdogProvider.class, builder.stubSettings().getStreamWatchdogProvider())); + + apiClock = builder.stubSettings().getClock(); + + builder + .stubSettings() + .setTransportChannelProvider(transportChannelProvider) + .setCredentialsProvider(credentialsProvider) + .setExecutorProvider(executorProvider) + .setStreamWatchdogProvider(watchdogProvider) + .setClock(apiClock); + + defaultSettings = builder.build(); + } + + @After + public void tearDown() { + if (fakeServer != null) { + fakeServer.shutdownNow(); + } + } + + @Test + public void testNewClientsShareTransportChannel() throws Exception { + BigtableDataClientFactory factory = BigtableDataClientFactory.create(defaultSettings); + + // Create 3 lightweight clients + BigtableDataClient client1 = factory.createForInstance("project1", "instance1"); + BigtableDataClient client2 = factory.createForInstance("project2", "instance2"); + BigtableDataClient client3 = factory.createForInstance("project3", "instance3"); + + // Make sure that only 1 instance is created by each provider + Mockito.verify(transportChannelProvider, Mockito.times(1)).getTransportChannel(); + Mockito.verify(credentialsProvider, Mockito.times(1)).getCredentials(); + Mockito.verify(executorProvider, Mockito.times(1)).getExecutor(); + Mockito.verify(watchdogProvider, Mockito.times(1)).getWatchdog(); + + // clean up + client1.close(); + client2.close(); + client3.close(); + factory.close(); + } + + @Test + public void testCreateDefaultKeepsSettings() throws Exception { + BigtableDataClientFactory factory = BigtableDataClientFactory.create(defaultSettings); + BigtableDataClient client = factory.createDefault(); + + client.mutateRow(RowMutation.create("some-table", "some-key").deleteRow()); + + assertThat(service.lastRequest.getTableName()) + .isEqualTo(NameUtil.formatTableName(DEFAULT_PROJECT_ID, DEFAULT_INSTANCE_ID, "some-table")); + assertThat(service.lastRequest.getAppProfileId()).isEqualTo(DEFAULT_APP_PROFILE_ID); + } + + @Test + public void testCreateForAppProfileHasCorrectSettings() throws Exception { + BigtableDataClientFactory factory = BigtableDataClientFactory.create(defaultSettings); + BigtableDataClient client = factory.createForAppProfile("other-app-profile"); + + client.mutateRow(RowMutation.create("some-table", "some-key").deleteRow()); + + assertThat(service.lastRequest.getTableName()) + .isEqualTo(NameUtil.formatTableName(DEFAULT_PROJECT_ID, DEFAULT_INSTANCE_ID, "some-table")); + assertThat(service.lastRequest.getAppProfileId()).isEqualTo("other-app-profile"); + } + + @Test + public void testCreateForInstanceHasCorrectSettings() throws Exception { + BigtableDataClientFactory factory = BigtableDataClientFactory.create(defaultSettings); + BigtableDataClient client = factory.createForInstance("other-project", "other-instance"); + + client.mutateRow(RowMutation.create("some-table", "some-key").deleteRow()); + + assertThat(service.lastRequest.getTableName()) + .isEqualTo(NameUtil.formatTableName("other-project", "other-instance", "some-table")); + // app profile should be reset to default + assertThat(service.lastRequest.getAppProfileId()).isEmpty(); + } + + @Test + public void testCreateForInstanceWithAppProfileHasCorrectSettings() throws Exception { + BigtableDataClientFactory factory = BigtableDataClientFactory.create(defaultSettings); + BigtableDataClient client = + factory.createForInstance("other-project", "other-instance", "other-app-profile"); + + client.mutateRow(RowMutation.create("some-table", "some-key").deleteRow()); + + assertThat(service.lastRequest.getTableName()) + .isEqualTo(NameUtil.formatTableName("other-project", "other-instance", "some-table")); + // app profile should be reset to default + assertThat(service.lastRequest.getAppProfileId()).isEqualTo("other-app-profile"); + } + + private static class FakeBigtableService extends BigtableGrpc.BigtableImplBase { + volatile MutateRowRequest lastRequest; + + @Override + public void mutateRow( + MutateRowRequest request, StreamObserver responseObserver) { + lastRequest = request; + responseObserver.onNext(MutateRowResponse.getDefaultInstance()); + responseObserver.onCompleted(); + } + } + + private static class BuilderAnswer implements Answer { + private final Class targetClass; + private T targetInstance; + + public BuilderAnswer(Class targetClass, T targetInstance) { + this.targetClass = targetClass; + this.targetInstance = targetInstance; + } + + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + Method method = invocation.getMethod(); + Object r = invocation.getMethod().invoke(targetInstance, invocation.getArguments()); + + if (method.getName().startsWith("with") + && targetClass.isAssignableFrom(method.getReturnType())) { + this.targetInstance = (T) r; + r = invocation.getMock(); + } + return r; + } + } +}