From d33b64594f9b3217b0fc89d255618b558a9d45e8 Mon Sep 17 00:00:00 2001 From: Praful Makani Date: Fri, 20 Mar 2020 05:33:12 +0530 Subject: [PATCH] feat: add materialize view (#174) * feat: add materialize view * feat: add unit test case * feat: modified comment * feat: add additional fields and code coverage * feat: add integration test * feat: run formatter --- .../bigquery/MaterializedViewDefinition.java | 155 ++++++++++++++++++ .../cloud/bigquery/TableDefinition.java | 10 ++ .../MaterializedViewDefinitionTest.java | 99 +++++++++++ .../cloud/bigquery/it/ITBigQueryTest.java | 51 ++++-- 4 files changed, 303 insertions(+), 12 deletions(-) create mode 100644 google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/MaterializedViewDefinition.java create mode 100644 google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/MaterializedViewDefinitionTest.java diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/MaterializedViewDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/MaterializedViewDefinition.java new file mode 100644 index 000000000..bf3a913a9 --- /dev/null +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/MaterializedViewDefinition.java @@ -0,0 +1,155 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import com.google.api.services.bigquery.model.Table; +import com.google.auto.value.AutoValue; +import javax.annotation.Nullable; + +@AutoValue +public abstract class MaterializedViewDefinition extends TableDefinition { + + private static final long serialVersionUID = 5898696389126164276L; + + @AutoValue.Builder + public abstract static class Builder + extends TableDefinition.Builder { + + /** + * [Output-only] The time when this materialized view was last modified, in milliseconds since + * the epoch. + */ + abstract Builder setLastRefreshTime(Long lastRefreshTime); + + /** Sets the query whose result is persisted. */ + public abstract Builder setQuery(String query); + + /** + * Set enable automatic refresh of the materialized view when the base table is updated. The + * default value is "true". + */ + public abstract Builder setEnableRefresh(Boolean enableRefresh); + + /** + * Set a maximum frequency at which this materialized view will be refreshed. The default value + * is "1800000" (30 minutes). + */ + public abstract Builder setRefreshIntervalMs(Long refreshIntervalMs); + + /** Sets the table schema. */ + @Override + public abstract Builder setSchema(Schema schema); + + @Override + public abstract Builder setType(Type type); + + /** Creates a {@code MaterializedViewDefinition} object. */ + @Override + public abstract MaterializedViewDefinition build(); + } + + /** + * Returns time when this materialized view was last modified, in milliseconds since the epoch. + */ + @Nullable + public abstract Long getLastRefreshTime(); + + /** Returns a query whose result is persisted. */ + @Nullable + public abstract String getQuery(); + + /** + * Returns enable automatic refresh of the materialized view when the base table is updated. The + * default value is "true". + */ + @Nullable + public abstract Boolean getEnableRefresh(); + + /** + * Returns a maximum frequency at which this materialized view will be refreshed. The default + * value is "1800000" (30 minutes). + */ + @Nullable + public abstract Long getRefreshIntervalMs(); + + /** Returns a builder for the {@code MaterializedViewDefinition} object. */ + public abstract Builder toBuilder(); + + @Override + Table toPb() { + Table tablePb = super.toPb(); + com.google.api.services.bigquery.model.MaterializedViewDefinition materializedViewDefinition = + new com.google.api.services.bigquery.model.MaterializedViewDefinition(); + if (getQuery() != null) { + materializedViewDefinition.setQuery(getQuery()); + } + if (getLastRefreshTime() != null) { + materializedViewDefinition.setLastRefreshTime(getLastRefreshTime()); + } + if (getEnableRefresh() != null) { + materializedViewDefinition.setEnableRefresh(getEnableRefresh()); + } + if (getRefreshIntervalMs() != null) { + materializedViewDefinition.setRefreshIntervalMs(getRefreshIntervalMs()); + } + tablePb.setMaterializedView(materializedViewDefinition); + return tablePb; + } + + static Builder newBuilder() { + return new AutoValue_MaterializedViewDefinition.Builder().setType(Type.MATERIALIZED_VIEW); + } + + /** + * Returns a builder for a BigQuery materialized view definition. + * + * @param query the query used to generate the materialized view + */ + public static Builder newBuilder(String query) { + return newBuilder().setQuery(query); + } + + /** + * Returns a builder for a BigQuery materialized view definition. + * + * @param query the query used to generate the materialized view + */ + public static MaterializedViewDefinition of(String query) { + return newBuilder(query).build(); + } + + static MaterializedViewDefinition fromPb(Table tablePb) { + Builder builder = newBuilder().table(tablePb); + if (tablePb.getMaterializedView() != null) { + com.google.api.services.bigquery.model.MaterializedViewDefinition materializedViewDefinition = + tablePb.getMaterializedView(); + if (materializedViewDefinition.getQuery() != null) { + builder.setQuery(materializedViewDefinition.getQuery()); + } + if (materializedViewDefinition.getLastRefreshTime() != null) { + builder.setLastRefreshTime(materializedViewDefinition.getLastRefreshTime()); + } + if (materializedViewDefinition.getEnableRefresh() != null) { + builder.setEnableRefresh(materializedViewDefinition.getEnableRefresh()); + } + if (materializedViewDefinition.getRefreshIntervalMs() != null) { + builder.setRefreshIntervalMs(materializedViewDefinition.getRefreshIntervalMs()); + } + } + return builder.build(); + } +} diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableDefinition.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableDefinition.java index 75d4bf403..6babd4e6f 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableDefinition.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/TableDefinition.java @@ -57,6 +57,14 @@ public Type apply(String constant) { */ public static final Type VIEW = type.createAndRegister("VIEW"); + /** + * SQL query whose result is persisted. Instances of {@code MaterializedViewDefinition} for this + * type are implemented by {@link MaterializedViewDefinition}. + * + * @see Views + */ + public static final Type MATERIALIZED_VIEW = type.createAndRegister("MATERIALIZED_VIEW"); + /** * A BigQuery table backed by external data. Instances of {@code TableDefinition} for this type * are implemented by {@link ExternalTableDefinition}. @@ -151,6 +159,8 @@ static T fromPb(Table tablePb) { return (T) StandardTableDefinition.fromPb(tablePb); case "VIEW": return (T) ViewDefinition.fromPb(tablePb); + case "MATERIALIZED_VIEW": + return (T) MaterializedViewDefinition.fromPb(tablePb); case "EXTERNAL": return (T) ExternalTableDefinition.fromPb(tablePb); case "MODEL": diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/MaterializedViewDefinitionTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/MaterializedViewDefinitionTest.java new file mode 100644 index 000000000..ab1caab07 --- /dev/null +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/MaterializedViewDefinitionTest.java @@ -0,0 +1,99 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public class MaterializedViewDefinitionTest { + + private static final String MATERIALIZED_VIEW_QUERY = "MATERIALIZED_VIEW_QUERY"; + private static final Long LAST_REFRESH_TIME = 1580302008L; + private static final Boolean ENABLE_REFRESH = false; + private static final Long REFRESH_INTERVAL_MS = 60000L; + private static final Schema SCHEMA = Schema.of(); + private static final MaterializedViewDefinition MATERIALIZED_VIEW_DEFINITION = + MaterializedViewDefinition.newBuilder() + .setSchema(SCHEMA) + .setQuery(MATERIALIZED_VIEW_QUERY) + .setLastRefreshTime(LAST_REFRESH_TIME) + .setEnableRefresh(ENABLE_REFRESH) + .setRefreshIntervalMs(REFRESH_INTERVAL_MS) + .build(); + + @Test + public void testToBuilder() { + compareMaterializedView( + MATERIALIZED_VIEW_DEFINITION, MATERIALIZED_VIEW_DEFINITION.toBuilder().build()); + MaterializedViewDefinition materializedViewDefinition = + MATERIALIZED_VIEW_DEFINITION.toBuilder().setQuery("NEW QUERY").build(); + assertEquals("NEW QUERY", materializedViewDefinition.getQuery()); + materializedViewDefinition = + materializedViewDefinition.toBuilder().setQuery(MATERIALIZED_VIEW_QUERY).build(); + compareMaterializedView(MATERIALIZED_VIEW_DEFINITION, materializedViewDefinition); + } + + @Test + public void testToBuilderIncomplete() { + TableDefinition materializedViewDefinition = + MaterializedViewDefinition.of(MATERIALIZED_VIEW_QUERY); + assertEquals(materializedViewDefinition, materializedViewDefinition.toBuilder().build()); + } + + @Test + public void testBuilder() { + assertEquals(MATERIALIZED_VIEW_QUERY, MATERIALIZED_VIEW_DEFINITION.getQuery()); + assertEquals(TableDefinition.Type.MATERIALIZED_VIEW, MATERIALIZED_VIEW_DEFINITION.getType()); + assertEquals(LAST_REFRESH_TIME, MATERIALIZED_VIEW_DEFINITION.getLastRefreshTime()); + MaterializedViewDefinition materializedViewDefinition = + MaterializedViewDefinition.newBuilder() + .setSchema(SCHEMA) + .setQuery(MATERIALIZED_VIEW_QUERY) + .setLastRefreshTime(LAST_REFRESH_TIME) + .setEnableRefresh(ENABLE_REFRESH) + .setRefreshIntervalMs(REFRESH_INTERVAL_MS) + .build(); + assertEquals(MATERIALIZED_VIEW_DEFINITION, materializedViewDefinition); + } + + @Test + public void testToAndFromPb() { + MaterializedViewDefinition materializedViewDefinition = + MATERIALIZED_VIEW_DEFINITION.toBuilder().build(); + assertTrue( + TableDefinition.fromPb(materializedViewDefinition.toPb()) + instanceof MaterializedViewDefinition); + compareMaterializedView( + materializedViewDefinition, + TableDefinition.fromPb(materializedViewDefinition.toPb())); + } + + private void compareMaterializedView( + MaterializedViewDefinition expected, MaterializedViewDefinition actual) { + assertEquals(expected.getType(), actual.getType()); + assertEquals(expected.getSchema(), actual.getSchema()); + assertEquals(expected.getQuery(), actual.getQuery()); + assertEquals(expected.getLastRefreshTime(), actual.getLastRefreshTime()); + assertEquals(expected.getEnableRefresh(), actual.getEnableRefresh()); + assertEquals(expected.getRefreshIntervalMs(), actual.getRefreshIntervalMs()); + assertEquals(expected.toString(), actual.toString()); + assertEquals(expected.hashCode(), actual.hashCode()); + assertEquals(expected, actual); + } +} diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java index f2e747319..6e2d89e1a 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java @@ -32,6 +32,7 @@ import com.google.auth.oauth2.ServiceAccountCredentials; import com.google.cloud.Date; import com.google.cloud.RetryOption; +import com.google.cloud.ServiceOptions; import com.google.cloud.bigquery.Acl; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; @@ -65,6 +66,7 @@ import com.google.cloud.bigquery.JobStatistics.LoadStatistics; import com.google.cloud.bigquery.LegacySQLTypeName; import com.google.cloud.bigquery.LoadJobConfiguration; +import com.google.cloud.bigquery.MaterializedViewDefinition; import com.google.cloud.bigquery.Model; import com.google.cloud.bigquery.ModelId; import com.google.cloud.bigquery.ModelInfo; @@ -134,6 +136,7 @@ public class ITBigQueryTest { private static final String OTHER_DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String MODEL_DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String ROUTINE_DATASET = RemoteBigQueryHelper.generateDatasetName(); + private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); private static final Map LABELS = ImmutableMap.of( "example-label1", "example-value1", @@ -219,6 +222,17 @@ public class ITBigQueryTest { Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN) .setMode(Field.Mode.NULLABLE) .build()); + private static final Schema VIEW_SCHEMA = + Schema.of( + Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("StringField", LegacySQLTypeName.STRING) + .setMode(Field.Mode.NULLABLE) + .build(), + Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN) + .setMode(Field.Mode.NULLABLE) + .build()); private static final RangePartitioning.Range RANGE = RangePartitioning.Range.newBuilder().setStart(1L).setInterval(2L).setEnd(20L).build(); private static final RangePartitioning RANGE_PARTITIONING = @@ -640,18 +654,7 @@ public void testCreateViewTable() throws InterruptedException { assertNotNull(remoteTable); assertEquals(createdTable.getTableId(), remoteTable.getTableId()); assertTrue(remoteTable.getDefinition() instanceof ViewDefinition); - Schema expectedSchema = - Schema.of( - Field.newBuilder("TimestampField", LegacySQLTypeName.TIMESTAMP) - .setMode(Field.Mode.NULLABLE) - .build(), - Field.newBuilder("StringField", LegacySQLTypeName.STRING) - .setMode(Field.Mode.NULLABLE) - .build(), - Field.newBuilder("BooleanField", LegacySQLTypeName.BOOLEAN) - .setMode(Field.Mode.NULLABLE) - .build()); - assertEquals(expectedSchema, remoteTable.getDefinition().getSchema()); + assertEquals(VIEW_SCHEMA, remoteTable.getDefinition().getSchema()); QueryJobConfiguration config = QueryJobConfiguration.newBuilder("SELECT * FROM " + tableName) .setDefaultDataset(DatasetId.of(DATASET)) @@ -678,6 +681,30 @@ public void testCreateViewTable() throws InterruptedException { assertTrue(remoteTable.delete()); } + @Test + public void testCreateMaterializedViewTable() { + String tableName = "test_materialized_view_table"; + TableId tableId = TableId.of(DATASET, tableName); + MaterializedViewDefinition viewDefinition = + MaterializedViewDefinition.newBuilder( + String.format( + "SELECT MAX(TimestampField) AS TimestampField,StringField, MAX(BooleanField) AS BooleanField FROM %s.%s.%s GROUP BY StringField", + PROJECT_ID, DATASET, TABLE_ID.getTable())) + .build(); + TableInfo tableInfo = TableInfo.of(tableId, viewDefinition); + Table createdTable = bigquery.create(tableInfo); + assertNotNull(createdTable); + assertEquals(DATASET, createdTable.getTableId().getDataset()); + assertEquals(tableName, createdTable.getTableId().getTable()); + Table remoteTable = bigquery.getTable(DATASET, tableName); + assertNotNull(remoteTable); + assertEquals(createdTable.getTableId(), remoteTable.getTableId()); + assertEquals(createdTable.getTableId(), remoteTable.getTableId()); + assertTrue(remoteTable.getDefinition() instanceof MaterializedViewDefinition); + assertEquals(VIEW_SCHEMA, remoteTable.getDefinition().getSchema()); + assertTrue(remoteTable.delete()); + } + @Test public void testListTables() { String tableName = "test_list_tables";