From d9a00a81c454ae793f9687d0e2de2bcc58d96502 Mon Sep 17 00:00:00 2001 From: Mayur Kale Date: Thu, 6 Feb 2020 14:45:06 -0800 Subject: [PATCH] feat: instrument Spanner client with OpenCensus metrics (#54) * feat: add session metrics Add active_sessions (The number of active sessions in use) and max_sessions (The number of max sessions configured the user) metrics * Fix code reviews Use maxSessionsInUse instead of numSessionsInUse and update the description. * Change active sessions description * add numSessionsInUse metric * Fix package structure * rename metric name and description * fix nits * createMockSession for metrics validations --- .../spanner/MetricRegistryConstants.java | 53 ++++++ .../com/google/cloud/spanner/SessionPool.java | 123 +++++++++++++- .../com/google/cloud/spanner/SpannerImpl.java | 11 +- .../spanner/MetricRegistryTestUtils.java | 156 ++++++++++++++++++ .../google/cloud/spanner/SessionPoolTest.java | 53 ++++++ 5 files changed, 391 insertions(+), 5 deletions(-) create mode 100644 google-cloud-spanner/src/main/java/com/google/cloud/spanner/MetricRegistryConstants.java create mode 100644 google-cloud-spanner/src/test/java/com/google/cloud/spanner/MetricRegistryTestUtils.java diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MetricRegistryConstants.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MetricRegistryConstants.java new file mode 100644 index 0000000000..a79fe550d7 --- /dev/null +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/MetricRegistryConstants.java @@ -0,0 +1,53 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.spanner; + +import com.google.common.collect.ImmutableList; +import io.opencensus.metrics.LabelKey; +import io.opencensus.metrics.LabelValue; + +/** A helper class that holds OpenCensus's related constants. */ +class MetricRegistryConstants { + + // The label keys are used to uniquely identify timeseries. + private static final LabelKey DATABASE = LabelKey.create("database", "Target database"); + private static final LabelKey INSTANCE_ID = + LabelKey.create("instance_id", "Name of the instance"); + private static final LabelKey LIBRARY_VERSION = + LabelKey.create("library_version", "Library version"); + + /** The label value is used to represent missing value. */ + private static final LabelValue UNSET_LABEL = LabelValue.create(null); + + static final ImmutableList SPANNER_LABEL_KEYS = + ImmutableList.of(DATABASE, INSTANCE_ID, LIBRARY_VERSION); + + static final ImmutableList SPANNER_DEFAULT_LABEL_VALUES = + ImmutableList.of(UNSET_LABEL, UNSET_LABEL, UNSET_LABEL); + + /** Unit to represent counts. */ + static final String COUNT = "1"; + + // The Metric name and description + static final String MAX_IN_USE_SESSIONS = "cloud.google.com/java/spanner/max_in_use_session"; + static final String MAX_ALLOWED_SESSIONS = "cloud.google.com/java/spanner/max_allowed_sessions"; + static final String IN_USE_SESSIONS = "cloud.google.com/java/spanner/in_use_sessions"; + static final String MAX_IN_USE_SESSIONS_DESCRIPTION = + "The maximum number of sessions in use during the last 10 minute interval."; + static final String MAX_ALLOWED_SESSIONS_DESCRIPTION = + "The maximum number of sessions allowed. Configurable by the user."; + static final String IN_USE_SESSIONS_DESCRIPTION = "The number of sessions currently in use."; +} diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SessionPool.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SessionPool.java index 45112ff4b4..76953713ee 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SessionPool.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SessionPool.java @@ -16,6 +16,15 @@ package com.google.cloud.spanner; +import static com.google.cloud.spanner.MetricRegistryConstants.COUNT; +import static com.google.cloud.spanner.MetricRegistryConstants.IN_USE_SESSIONS; +import static com.google.cloud.spanner.MetricRegistryConstants.IN_USE_SESSIONS_DESCRIPTION; +import static com.google.cloud.spanner.MetricRegistryConstants.MAX_ALLOWED_SESSIONS; +import static com.google.cloud.spanner.MetricRegistryConstants.MAX_ALLOWED_SESSIONS_DESCRIPTION; +import static com.google.cloud.spanner.MetricRegistryConstants.MAX_IN_USE_SESSIONS; +import static com.google.cloud.spanner.MetricRegistryConstants.MAX_IN_USE_SESSIONS_DESCRIPTION; +import static com.google.cloud.spanner.MetricRegistryConstants.SPANNER_DEFAULT_LABEL_VALUES; +import static com.google.cloud.spanner.MetricRegistryConstants.SPANNER_LABEL_KEYS; import static com.google.cloud.spanner.SpannerExceptionFactory.newSpannerException; import com.google.api.core.ApiFuture; @@ -40,6 +49,12 @@ import com.google.common.util.concurrent.Uninterruptibles; import com.google.protobuf.Empty; import io.opencensus.common.Scope; +import io.opencensus.common.ToLongFunction; +import io.opencensus.metrics.DerivedLongGauge; +import io.opencensus.metrics.LabelValue; +import io.opencensus.metrics.MetricOptions; +import io.opencensus.metrics.MetricRegistry; +import io.opencensus.metrics.Metrics; import io.opencensus.trace.Annotation; import io.opencensus.trace.AttributeValue; import io.opencensus.trace.Span; @@ -49,6 +64,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; +import java.util.List; import java.util.Queue; import java.util.Random; import java.util.Set; @@ -1116,11 +1132,15 @@ private static enum Position { * Return pool is immediately ready for use, though getting a session might block for sessions to * be created. */ - static SessionPool createPool(SpannerOptions spannerOptions, SessionClient sessionClient) { + static SessionPool createPool( + SpannerOptions spannerOptions, SessionClient sessionClient, List labelValues) { return createPool( spannerOptions.getSessionPoolOptions(), ((GrpcTransportOptions) spannerOptions.getTransportOptions()).getExecutorFactory(), - sessionClient); + sessionClient, + new Clock(), + Metrics.getMetricRegistry(), + labelValues); } static SessionPool createPool( @@ -1135,8 +1155,31 @@ static SessionPool createPool( ExecutorFactory executorFactory, SessionClient sessionClient, Clock clock) { + return createPool( + poolOptions, + executorFactory, + sessionClient, + clock, + Metrics.getMetricRegistry(), + SPANNER_DEFAULT_LABEL_VALUES); + } + + static SessionPool createPool( + SessionPoolOptions poolOptions, + ExecutorFactory executorFactory, + SessionClient sessionClient, + Clock clock, + MetricRegistry metricRegistry, + List labelValues) { SessionPool pool = - new SessionPool(poolOptions, executorFactory, executorFactory.get(), sessionClient, clock); + new SessionPool( + poolOptions, + executorFactory, + executorFactory.get(), + sessionClient, + clock, + metricRegistry, + labelValues); pool.initPool(); return pool; } @@ -1146,13 +1189,16 @@ private SessionPool( ExecutorFactory executorFactory, ScheduledExecutorService executor, SessionClient sessionClient, - Clock clock) { + Clock clock, + MetricRegistry metricRegistry, + List labelValues) { this.options = options; this.executorFactory = executorFactory; this.executor = executor; this.sessionClient = sessionClient; this.clock = clock; this.poolMaintainer = new PoolMaintainer(); + this.initMetricsCollection(metricRegistry, labelValues); } @VisibleForTesting @@ -1766,4 +1812,73 @@ public void onSessionCreateFailure(Throwable t, int createFailureForSessionCount } } } + + /** + * Initializes and creates Spanner session relevant metrics. When coupled with an exporter, it + * allows users to monitor client behavior. + */ + private void initMetricsCollection(MetricRegistry metricRegistry, List labelValues) { + DerivedLongGauge maxInUseSessionsMetric = + metricRegistry.addDerivedLongGauge( + MAX_IN_USE_SESSIONS, + MetricOptions.builder() + .setDescription(MAX_IN_USE_SESSIONS_DESCRIPTION) + .setUnit(COUNT) + .setLabelKeys(SPANNER_LABEL_KEYS) + .build()); + + DerivedLongGauge maxAllowedSessionsMetric = + metricRegistry.addDerivedLongGauge( + MAX_ALLOWED_SESSIONS, + MetricOptions.builder() + .setDescription(MAX_ALLOWED_SESSIONS_DESCRIPTION) + .setUnit(COUNT) + .setLabelKeys(SPANNER_LABEL_KEYS) + .build()); + + DerivedLongGauge numInUseSessionsMetric = + metricRegistry.addDerivedLongGauge( + IN_USE_SESSIONS, + MetricOptions.builder() + .setDescription(IN_USE_SESSIONS_DESCRIPTION) + .setUnit(COUNT) + .setLabelKeys(SPANNER_LABEL_KEYS) + .build()); + + // The value of a maxSessionsInUse is observed from a callback function. This function is + // invoked whenever metrics are collected. + maxInUseSessionsMetric.createTimeSeries( + labelValues, + this, + new ToLongFunction() { + @Override + public long applyAsLong(SessionPool sessionPool) { + return sessionPool.maxSessionsInUse; + } + }); + + // The value of a maxSessions is observed from a callback function. This function is invoked + // whenever metrics are collected. + maxAllowedSessionsMetric.createTimeSeries( + labelValues, + options, + new ToLongFunction() { + @Override + public long applyAsLong(SessionPoolOptions options) { + return options.getMaxSessions(); + } + }); + + // The value of a numSessionsInUse is observed from a callback function. This function is + // invoked whenever metrics are collected. + numInUseSessionsMetric.createTimeSeries( + labelValues, + this, + new ToLongFunction() { + @Override + public long applyAsLong(SessionPool sessionPool) { + return sessionPool.numSessionsInUse; + } + }); + } } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerImpl.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerImpl.java index aa8abbacb4..07cdc91190 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerImpl.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerImpl.java @@ -16,6 +16,7 @@ package com.google.cloud.spanner; +import com.google.api.gax.core.GaxProperties; import com.google.api.gax.paging.Page; import com.google.cloud.BaseService; import com.google.cloud.PageImpl; @@ -27,8 +28,10 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import io.opencensus.metrics.LabelValue; import io.opencensus.trace.Tracer; import io.opencensus.trace.Tracing; import java.util.ArrayList; @@ -142,8 +145,14 @@ public DatabaseClient getDatabaseClient(DatabaseId db) { if (dbClients.containsKey(db)) { return dbClients.get(db); } else { + List labelValues = + ImmutableList.of( + LabelValue.create(db.getDatabase()), + LabelValue.create(db.getInstanceId().getName()), + LabelValue.create(GaxProperties.getLibraryVersion(getOptions().getClass()))); SessionPool pool = - SessionPool.createPool(getOptions(), SpannerImpl.this.getSessionClient(db)); + SessionPool.createPool( + getOptions(), SpannerImpl.this.getSessionClient(db), labelValues); DatabaseClientImpl dbClient = createDatabaseClient(pool); dbClients.put(db, dbClient); return dbClient; diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MetricRegistryTestUtils.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MetricRegistryTestUtils.java new file mode 100644 index 0000000000..fe9252d1f9 --- /dev/null +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/MetricRegistryTestUtils.java @@ -0,0 +1,156 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.spanner; + +import com.google.common.collect.Maps; +import io.opencensus.common.ToLongFunction; +import io.opencensus.metrics.DerivedDoubleCumulative; +import io.opencensus.metrics.DerivedDoubleGauge; +import io.opencensus.metrics.DerivedLongCumulative; +import io.opencensus.metrics.DerivedLongGauge; +import io.opencensus.metrics.DoubleCumulative; +import io.opencensus.metrics.DoubleGauge; +import io.opencensus.metrics.LabelKey; +import io.opencensus.metrics.LabelValue; +import io.opencensus.metrics.LongCumulative; +import io.opencensus.metrics.LongGauge; +import io.opencensus.metrics.MetricOptions; +import io.opencensus.metrics.MetricRegistry; +import java.util.List; +import java.util.Map; + +class MetricRegistryTestUtils { + + static class PointWithFunction { + private final T ref; + private final ToLongFunction function; + + PointWithFunction(T obj, ToLongFunction function) { + this.ref = obj; + this.function = function; + } + + long get() { + return function.applyAsLong(ref); + } + } + + static class MetricsRecord { + private final Map metrics; + private final Map, List> labels; + + private MetricsRecord() { + this.metrics = Maps.newHashMap(); + this.labels = Maps.newHashMap(); + } + + Map getMetrics() { + Map copy = Maps.newHashMap(); + for (Map.Entry entry : metrics.entrySet()) { + copy.put(entry.getKey(), entry.getValue().get()); + } + return copy; + } + + Map, List> getLabels() { + return this.labels; + } + } + + public static final class FakeDerivedLongGauge extends DerivedLongGauge { + private final MetricsRecord record; + private final String name; + private final List labelKeys; + + private FakeDerivedLongGauge( + FakeMetricRegistry metricRegistry, String name, List labelKeys) { + this.record = metricRegistry.record; + this.labelKeys = labelKeys; + this.name = name; + } + + @Override + public void createTimeSeries( + List labelValues, T t, ToLongFunction toLongFunction) { + this.record.metrics.put(this.name, new PointWithFunction(t, toLongFunction)); + this.record.labels.put(this.labelKeys, labelValues); + } + + @Override + public void removeTimeSeries(List list) {} + + @Override + public void clear() {} + } + + /** + * A {@link MetricRegistry} implementation that saves metrics records to be accessible from {@link + * #pollRecord()}. + */ + public static final class FakeMetricRegistry extends MetricRegistry { + + private MetricsRecord record; + + FakeMetricRegistry() { + record = new MetricsRecord(); + } + + MetricsRecord pollRecord() { + return record; + } + + @Override + public DerivedLongGauge addDerivedLongGauge(String s, MetricOptions metricOptions) { + return new FakeDerivedLongGauge(this, s, metricOptions.getLabelKeys()); + } + + @Override + public LongGauge addLongGauge(String s, MetricOptions metricOptions) { + throw new UnsupportedOperationException(); + } + + @Override + public DoubleGauge addDoubleGauge(String s, MetricOptions metricOptions) { + throw new UnsupportedOperationException(); + } + + @Override + public DerivedDoubleGauge addDerivedDoubleGauge(String s, MetricOptions metricOptions) { + throw new UnsupportedOperationException(); + } + + @Override + public LongCumulative addLongCumulative(String s, MetricOptions metricOptions) { + throw new UnsupportedOperationException(); + } + + @Override + public DoubleCumulative addDoubleCumulative(String s, MetricOptions metricOptions) { + throw new UnsupportedOperationException(); + } + + @Override + public DerivedLongCumulative addDerivedLongCumulative(String s, MetricOptions metricOptions) { + throw new UnsupportedOperationException(); + } + + @Override + public DerivedDoubleCumulative addDerivedDoubleCumulative( + String s, MetricOptions metricOptions) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionPoolTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionPoolTest.java index 7bd06af092..507d94b12a 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionPoolTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/SessionPoolTest.java @@ -16,6 +16,7 @@ package com.google.cloud.spanner; +import static com.google.cloud.spanner.MetricRegistryConstants.SPANNER_LABEL_KEYS; import static com.google.cloud.spanner.SpannerMatchers.isSpannerException; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.fail; @@ -35,6 +36,8 @@ import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutures; import com.google.cloud.Timestamp; +import com.google.cloud.spanner.MetricRegistryTestUtils.FakeMetricRegistry; +import com.google.cloud.spanner.MetricRegistryTestUtils.MetricsRecord; import com.google.cloud.spanner.ReadContext.QueryAnalyzeMode; import com.google.cloud.spanner.SessionClient.SessionConsumer; import com.google.cloud.spanner.SessionPool.Clock; @@ -53,6 +56,8 @@ import com.google.spanner.v1.ExecuteSqlRequest; import com.google.spanner.v1.ResultSetStats; import com.google.spanner.v1.RollbackRequest; +import io.opencensus.metrics.LabelValue; +import io.opencensus.metrics.MetricRegistry; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -110,6 +115,17 @@ private SessionPool createPool(Clock clock) { options, new TestExecutorFactory(), client.getSessionClient(db), clock); } + private SessionPool createPool( + Clock clock, MetricRegistry metricRegistry, List labelValues) { + return SessionPool.createPool( + options, + new TestExecutorFactory(), + client.getSessionClient(db), + clock, + metricRegistry, + labelValues); + } + @Before public void setUp() throws Exception { initMocks(this); @@ -1544,6 +1560,43 @@ public void run() { assertThat(impl.executePartitionedUpdate(statement)).isEqualTo(1L); } + @Test + public void testSessionMetrics() { + options = + SessionPoolOptions.newBuilder() + .setMinSessions(1) + .setMaxSessions(3) + .setMaxIdleSessions(0) + .build(); + FakeClock clock = new FakeClock(); + clock.currentTimeMillis = System.currentTimeMillis(); + FakeMetricRegistry metricRegistry = new FakeMetricRegistry(); + List labelValues = + Arrays.asList( + LabelValue.create("database1"), + LabelValue.create("instance1"), + LabelValue.create("1.0.0")); + + setupMockSessionCreation(); + pool = createPool(clock, metricRegistry, labelValues); + Session session1 = pool.getReadSession(); + Session session2 = pool.getReadSession(); + + MetricsRecord record = metricRegistry.pollRecord(); + assertThat(record.getMetrics()).containsEntry(MetricRegistryConstants.IN_USE_SESSIONS, 2L); + assertThat(record.getMetrics()).containsEntry(MetricRegistryConstants.MAX_IN_USE_SESSIONS, 2L); + assertThat(record.getMetrics()) + .containsEntry( + MetricRegistryConstants.MAX_ALLOWED_SESSIONS, (long) options.getMaxSessions()); + assertThat(record.getLabels()).containsEntry(SPANNER_LABEL_KEYS, labelValues); + + session2.close(); + session1.close(); + + assertThat(record.getMetrics()).containsEntry(MetricRegistryConstants.IN_USE_SESSIONS, 0L); + assertThat(record.getMetrics()).containsEntry(MetricRegistryConstants.MAX_IN_USE_SESSIONS, 2L); + } + private void mockKeepAlive(Session session) { ReadContext context = mock(ReadContext.class); ResultSet resultSet = mock(ResultSet.class);