Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
feat: adds query optimizer statistics support (#385)
* feat: adds optimizer_statistics_package option

Adds the possibility to set the optimizer statistics package when
executing queries. This option can be set in different levels as follows:
1. Through statement hints
2. Through query level configuration
3. Through an environment variable
4. Through application level configuration

If more than one package is set (in different levels) the precedence is from 1 to 4 (1 has top priority).

* test: adds integration tests

Adds integration tests for setting the query options optimizer
statistics package.

* fix: addresses PR comments

Fix missing value in the documentation of the optimizer statistics
package for the connection class.

* fix: adds tests for invalid stats packages

Adds tests for invalid statistics packages (whitespace only ones).

* fix: adds integration tests for query options

Adds an integration test to run the sql script with several expectations
for the query options. These are tests for the optimizer version and
optimizer statistics package.

* fix: formatting of ClientSideStatements.json

This file is using a mix of tabs and spaces. For now I have formatted as
the other lines, so that the diff is clear. In a further PR I will
reformat the whole file to use only spaces.

* tests: fix Connection test

Fixes connection test when using environment variables for retrieving
configurations. This only works when a first connection is created, so
we moved this specific test to its own subclass.

* fix: fix clirr checks

Provides default interface implementations and fixes clirr checks
  • Loading branch information
thiagotnunes committed Jun 5, 2021
1 parent 8a23ad0 commit e294532
Show file tree
Hide file tree
Showing 22 changed files with 568 additions and 50 deletions.
19 changes: 19 additions & 0 deletions google-cloud-spanner/clirr-ignored-differences.xml
Expand Up @@ -618,4 +618,23 @@
<className>com/google/cloud/spanner/TransactionContext</className>
<method>com.google.api.core.ApiFuture bufferAsync(java.lang.Iterable)</method>
</difference>

<!-- Query stats optimiser statistics package -->
<!-- These are not breaking changes, since we provide default interface implementation -->
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/spanner/SpannerOptions$SpannerEnvironment</className>
<method>java.lang.String getOptimizerStatisticsPackage()</method>
</difference>
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/spanner/connection/Connection</className>
<method>java.lang.String getOptimizerStatisticsPackage()</method>
</difference>
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/spanner/connection/Connection</className>
<method>void setOptimizerStatisticsPackage(java.lang.String)</method>
</difference>

</differences>
Expand Up @@ -585,6 +585,15 @@ public interface SpannerEnvironment {
*/
@Nonnull
String getOptimizerVersion();

/**
* The optimizer statistics package to use. Must return an empty string to indicate that no
* value has been set.
*/
@Nonnull
default String getOptimizerStatisticsPackage() {
throw new UnsupportedOperationException("Unimplemented");
}
}

/**
Expand All @@ -594,13 +603,21 @@ public interface SpannerEnvironment {
private static class SpannerEnvironmentImpl implements SpannerEnvironment {
private static final SpannerEnvironmentImpl INSTANCE = new SpannerEnvironmentImpl();
private static final String SPANNER_OPTIMIZER_VERSION_ENV_VAR = "SPANNER_OPTIMIZER_VERSION";
private static final String SPANNER_OPTIMIZER_STATISTICS_PACKAGE_ENV_VAR =
"SPANNER_OPTIMIZER_STATISTICS_PACKAGE";

private SpannerEnvironmentImpl() {}

@Override
public String getOptimizerVersion() {
return MoreObjects.firstNonNull(System.getenv(SPANNER_OPTIMIZER_VERSION_ENV_VAR), "");
}

@Override
public String getOptimizerStatisticsPackage() {
return MoreObjects.firstNonNull(
System.getenv(SPANNER_OPTIMIZER_STATISTICS_PACKAGE_ENV_VAR), "");
}
}

/** Builder for {@link SpannerOptions} instances. */
Expand Down Expand Up @@ -957,6 +974,7 @@ public Builder setDefaultQueryOptions(DatabaseId database, QueryOptions defaultQ
QueryOptions getEnvironmentQueryOptions() {
return QueryOptions.newBuilder()
.setOptimizerVersion(environment.getOptimizerVersion())
.setOptimizerStatisticsPackage(environment.getOptimizerStatisticsPackage())
.build();
}

Expand Down
Expand Up @@ -96,6 +96,11 @@
* <li><code>
* SET OPTIMIZER_VERSION='&lt;version&gt;' | 'LATEST'
* </code>: Sets the value of <code>OPTIMIZER_VERSION</code> for this connection.
* <li><code>SHOW OPTIMIZER_STATISTICS_PACKAGE</code>: Returns the current value of <code>
* OPTIMIZER_STATISTICS_PACKAGE</code> of this connection as a {@link ResultSet}
* <li><code>
* SET OPTIMIZER_STATISTICS_PACKAGE='&lt;package&gt;' | ''
* </code>: Sets the value of <code>OPTIMIZER_STATISTICS_PACKAGE</code> for this connection.
* <li><code>BEGIN [TRANSACTION]</code>: Begins a new transaction. This statement is optional when
* the connection is not in autocommit mode, as a new transaction will automatically be
* started when a query or update statement is issued. In autocommit mode, this statement will
Expand Down Expand Up @@ -448,6 +453,29 @@ public interface Connection extends AutoCloseable {
*/
String getOptimizerVersion();

/**
* Sets the query optimizer statistics package
*
* @param optimizerStatisticsPackage The query optimizer statistics package to use. Must be a
* string composed of letters, numbers, dashes and underscores or an empty string. The empty
* string will instruct the connection to use the optimizer statistics package that is defined
* the environment variable <code>SPANNER_OPTIMIZER_STATISTICS_PACKAGE</code>. If no value is
* specified in the environment variable, the client level query optimizer is used. If none is
* set, the default query optimizer of Cloud Spanner is used.
*/
default void setOptimizerStatisticsPackage(String optimizerStatisticsPackage) {
throw new UnsupportedOperationException("Unimplemented");
}

/**
* Gets the current query optimizer statistics package of this connection.
*
* @return The query optimizer statistics package that is currently used by this connection.
*/
default String getOptimizerStatisticsPackage() {
throw new UnsupportedOperationException("Unimplemented");
}

/**
* Sets whether this connection should request commit statistics from Cloud Spanner for read/write
* transactions and DML statements in autocommit mode.
Expand Down
Expand Up @@ -433,6 +433,20 @@ public String getOptimizerVersion() {
return this.queryOptions.getOptimizerVersion();
}

@Override
public void setOptimizerStatisticsPackage(String optimizerStatisticsPackage) {
Preconditions.checkNotNull(optimizerStatisticsPackage);
ConnectionPreconditions.checkState(!isClosed(), CLOSED_ERROR_MSG);
this.queryOptions =
queryOptions.toBuilder().setOptimizerStatisticsPackage(optimizerStatisticsPackage).build();
}

@Override
public String getOptimizerStatisticsPackage() {
ConnectionPreconditions.checkState(!isClosed(), CLOSED_ERROR_MSG);
return this.queryOptions.getOptimizerStatisticsPackage();
}

@Override
public void setStatementTimeout(long timeout, TimeUnit unit) {
Preconditions.checkArgument(timeout > 0L, "Zero or negative timeout values are not allowed");
Expand Down
Expand Up @@ -157,6 +157,7 @@ public String[] getValidValues() {
private static final String DEFAULT_NUM_CHANNELS = null;
private static final String DEFAULT_USER_AGENT = null;
private static final String DEFAULT_OPTIMIZER_VERSION = "";
private static final String DEFAULT_OPTIMIZER_STATISTICS_PACKAGE = "";
private static final boolean DEFAULT_RETURN_COMMIT_STATS = false;
private static final boolean DEFAULT_LENIENT = false;

Expand Down Expand Up @@ -190,6 +191,9 @@ public String[] getValidValues() {
private static final String USER_AGENT_PROPERTY_NAME = "userAgent";
/** Query optimizer version to use for a connection. */
private static final String OPTIMIZER_VERSION_PROPERTY_NAME = "optimizerVersion";
/** Query optimizer statistics package to use for a connection. */
private static final String OPTIMIZER_STATISTICS_PACKAGE_PROPERTY_NAME =
"optimizerStatisticsPackage";
/** Name of the 'lenientMode' connection property. */
public static final String LENIENT_PROPERTY_NAME = "lenient";

Expand Down Expand Up @@ -238,6 +242,8 @@ public String[] getValidValues() {
ConnectionProperty.createStringProperty(
OPTIMIZER_VERSION_PROPERTY_NAME,
"Sets the default query optimizer version to use for this connection."),
ConnectionProperty.createStringProperty(
OPTIMIZER_STATISTICS_PACKAGE_PROPERTY_NAME, ""),
ConnectionProperty.createBooleanProperty("returnCommitStats", "", false),
ConnectionProperty.createBooleanProperty(
"autoConfigEmulator",
Expand Down Expand Up @@ -521,6 +527,7 @@ private ConnectionOptions(Builder builder) {
this.userAgent = parseUserAgent(this.uri);
QueryOptions.Builder queryOptionsBuilder = QueryOptions.newBuilder();
queryOptionsBuilder.setOptimizerVersion(parseOptimizerVersion(this.uri));
queryOptionsBuilder.setOptimizerStatisticsPackage(parseOptimizerStatisticsPackage(this.uri));
this.queryOptions = queryOptionsBuilder.build();
this.returnCommitStats = parseReturnCommitStats(this.uri);
this.autoConfigEmulator = parseAutoConfigEmulator(this.uri);
Expand Down Expand Up @@ -695,6 +702,12 @@ static String parseOptimizerVersion(String uri) {
return value != null ? value : DEFAULT_OPTIMIZER_VERSION;
}

@VisibleForTesting
static String parseOptimizerStatisticsPackage(String uri) {
String value = parseUriProperty(uri, OPTIMIZER_STATISTICS_PACKAGE_PROPERTY_NAME);
return value != null ? value : DEFAULT_OPTIMIZER_STATISTICS_PACKAGE;
}

@VisibleForTesting
static boolean parseReturnCommitStats(String uri) {
String value = parseUriProperty(uri, "returnCommitStats");
Expand Down
Expand Up @@ -66,6 +66,10 @@ interface ConnectionStatementExecutor {

StatementResult statementShowOptimizerVersion();

StatementResult statementSetOptimizerStatisticsPackage(String optimizerStatisticsPackage);

StatementResult statementShowOptimizerStatisticsPackage();

StatementResult statementSetReturnCommitStats(Boolean returnCommitStats);

StatementResult statementShowReturnCommitStats();
Expand Down
Expand Up @@ -23,6 +23,7 @@
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.RUN_BATCH;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SET_AUTOCOMMIT;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SET_AUTOCOMMIT_DML_MODE;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SET_OPTIMIZER_STATISTICS_PACKAGE;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SET_OPTIMIZER_VERSION;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SET_READONLY;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SET_READ_ONLY_STALENESS;
Expand All @@ -34,6 +35,7 @@
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SHOW_AUTOCOMMIT_DML_MODE;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SHOW_COMMIT_RESPONSE;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SHOW_COMMIT_TIMESTAMP;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SHOW_OPTIMIZER_STATISTICS_PACKAGE;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SHOW_OPTIMIZER_VERSION;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SHOW_READONLY;
import static com.google.cloud.spanner.connection.StatementResult.ClientSideStatementType.SHOW_READ_ONLY_STALENESS;
Expand Down Expand Up @@ -230,6 +232,20 @@ public StatementResult statementShowOptimizerVersion() {
"OPTIMIZER_VERSION", getConnection().getOptimizerVersion(), SHOW_OPTIMIZER_VERSION);
}

@Override
public StatementResult statementSetOptimizerStatisticsPackage(String optimizerStatisticsPackage) {
getConnection().setOptimizerStatisticsPackage(optimizerStatisticsPackage);
return noResult(SET_OPTIMIZER_STATISTICS_PACKAGE);
}

@Override
public StatementResult statementShowOptimizerStatisticsPackage() {
return resultSet(
"OPTIMIZER_STATISTICS_PACKAGE",
getConnection().getOptimizerStatisticsPackage(),
SHOW_OPTIMIZER_STATISTICS_PACKAGE);
}

@Override
public StatementResult statementSetReturnCommitStats(Boolean returnCommitStats) {
getConnection().setReturnCommitStats(returnCommitStats);
Expand Down
Expand Up @@ -65,6 +65,8 @@ enum ClientSideStatementType {
SET_READ_ONLY_STALENESS,
SHOW_OPTIMIZER_VERSION,
SET_OPTIMIZER_VERSION,
SHOW_OPTIMIZER_STATISTICS_PACKAGE,
SET_OPTIMIZER_STATISTICS_PACKAGE,
SHOW_RETURN_COMMIT_STATS,
SET_RETURN_COMMIT_STATS,
BEGIN,
Expand Down
Expand Up @@ -76,6 +76,14 @@
"method": "statementShowOptimizerVersion",
"exampleStatements": ["show variable optimizer_version"]
},
{
"name": "SHOW VARIABLE OPTIMIZER_STATISTICS_PACKAGE",
"executorName": "ClientSideStatementNoParamExecutor",
"resultType": "RESULT_SET",
"regex": "(?is)\\A\\s*show\\s+variable\\s+optimizer_statistics_package\\s*\\z",
"method": "statementShowOptimizerStatisticsPackage",
"exampleStatements": ["show variable optimizer_statistics_package"]
},
{
"name": "SHOW VARIABLE RETURN_COMMIT_STATS",
"executorName": "ClientSideStatementNoParamExecutor",
Expand Down Expand Up @@ -281,6 +289,20 @@
"converterName": "ClientSideStatementValueConverters$StringValueConverter"
}
},
{
"name": "SET OPTIMIZER_STATISTICS_PACKAGE = '<package>'|''",
"executorName": "ClientSideStatementSetExecutor",
"resultType": "NO_RESULT",
"regex": "(?is)\\A\\s*set\\s+optimizer_statistics_package\\s*(?:=)\\s*(.*)\\z",
"method": "statementSetOptimizerStatisticsPackage",
"exampleStatements": ["set optimizer_statistics_package='auto_20191128_14_47_22UTC'", "set optimizer_statistics_package=''"],
"setStatement": {
"propertyName": "OPTIMIZER_STATISTICS_PACKAGE",
"separator": "=",
"allowedValues": "'((\\S+)|())'",
"converterName": "ClientSideStatementValueConverters$StringValueConverter"
}
},
{
"name": "SET RETURN_COMMIT_STATS = TRUE|FALSE",
"executorName": "ClientSideStatementSetExecutor",
Expand All @@ -296,4 +318,4 @@
}
}
]
}
}
Expand Up @@ -52,7 +52,12 @@ public static Collection<Object[]> parameters() {
List<Object[]> params = new ArrayList<>();
params.add(new Object[] {QueryOptions.getDefaultInstance()});
params.add(
new Object[] {QueryOptions.newBuilder().setOptimizerVersion("some-version").build()});
new Object[] {
QueryOptions.newBuilder()
.setOptimizerVersion("some-version")
.setOptimizerStatisticsPackage("some-package")
.build()
});
return params;
}

Expand Down Expand Up @@ -134,14 +139,20 @@ public void executeSqlRequestBuilderWithQueryOptions() {
context
.getExecuteSqlRequestBuilder(
Statement.newBuilder("SELECT FOO FROM BAR")
.withQueryOptions(QueryOptions.newBuilder().setOptimizerVersion("2.0").build())
.withQueryOptions(
QueryOptions.newBuilder()
.setOptimizerVersion("2.0")
.setOptimizerStatisticsPackage("custom-package")
.build())
.build(),
QueryMode.NORMAL,
Options.fromQueryOptions(),
true)
.build();
assertThat(request.getSql()).isEqualTo("SELECT FOO FROM BAR");
assertThat(request.getQueryOptions().getOptimizerVersion()).isEqualTo("2.0");
assertThat(request.getQueryOptions().getOptimizerStatisticsPackage())
.isEqualTo("custom-package");
}

@Test
Expand Down

0 comments on commit e294532

Please sign in to comment.