diff --git a/samples/src/main/java/com/example/bigquery/CreateDataset.java b/samples/src/main/java/com/example/bigquery/CreateDataset.java index f61fce657..3e2fe0f42 100644 --- a/samples/src/main/java/com/example/bigquery/CreateDataset.java +++ b/samples/src/main/java/com/example/bigquery/CreateDataset.java @@ -27,17 +27,18 @@ public class CreateDataset { public static void runCreateDataset() { // TODO(developer): Replace these variables before running the sample. - String datasetName = "my-dataset-name"; + String datasetName = "MY_DATASET_NAME"; createDataset(datasetName); } public static void createDataset(String datasetName) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - - DatasetInfo datasetInfo = DatasetInfo.newBuilder(datasetName).build(); try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + + DatasetInfo datasetInfo = DatasetInfo.newBuilder(datasetName).build(); + Dataset newDataset = bigquery.create(datasetInfo); String newDatasetName = newDataset.getDatasetId().getDataset(); System.out.println(newDatasetName + " created successfully"); diff --git a/samples/src/main/java/com/example/bigquery/CreateTable.java b/samples/src/main/java/com/example/bigquery/CreateTable.java index 0f56495b4..d00000e55 100644 --- a/samples/src/main/java/com/example/bigquery/CreateTable.java +++ b/samples/src/main/java/com/example/bigquery/CreateTable.java @@ -1,3 +1,19 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.example.bigquery; // [START bigquery_create_table] @@ -16,26 +32,26 @@ public class CreateTable { public static void runCreateTable() { // TODO(developer): Replace these variables before running the sample. - String datasetName = "my-dataset-name"; - String tableName = "my_table_name"; + String datasetName = "MY_DATASET_NAME"; + String tableName = "MY_TABLE_NAME"; Schema schema = Schema.of( - // LegacySQLTypeName will be updated to StandardSQLTypeName once release rolls out + // INFO: LegacySQLTypeName will be updated to StandardSQLTypeName in release 1.103.0 Field.of("stringField", LegacySQLTypeName.STRING), Field.of("booleanField", LegacySQLTypeName.BOOLEAN)); createTable(datasetName, tableName, schema); } public static void createTable(String datasetName, String tableName, Schema schema) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - TableId tableId = TableId.of(datasetName, tableName); - TableDefinition tableDefinition = StandardTableDefinition.of(schema); - TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build(); + TableId tableId = TableId.of(datasetName, tableName); + TableDefinition tableDefinition = StandardTableDefinition.of(schema); + TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build(); - try { bigquery.create(tableInfo); System.out.println("Table created successfully"); } catch (BigQueryException e) { diff --git a/samples/src/main/java/com/example/bigquery/DeleteDataset.java b/samples/src/main/java/com/example/bigquery/DeleteDataset.java index f0f8ac96a..7018d0397 100644 --- a/samples/src/main/java/com/example/bigquery/DeleteDataset.java +++ b/samples/src/main/java/com/example/bigquery/DeleteDataset.java @@ -19,6 +19,7 @@ // [START bigquery_delete_dataset] import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; +import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.DatasetId; @@ -26,22 +27,26 @@ public class DeleteDataset { public static void runDeleteDataset() { // TODO(developer): Replace these variables before running the sample.\ - String projectId = "my-project-id"; - String datasetName = "my-dataset-name"; + String projectId = "MY_PROJECT_ID"; + String datasetName = "MY_DATASET_NAME"; deleteDataset(projectId, datasetName); } public static void deleteDataset(String projectId, String datasetName) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - DatasetId datasetId = DatasetId.of(projectId, datasetName); - boolean success = bigquery.delete(datasetId, DatasetDeleteOption.deleteContents()); - if (success) { - System.out.println("Dataset deleted successfully"); - } else { - System.out.println("Dataset was not found"); + DatasetId datasetId = DatasetId.of(projectId, datasetName); + boolean success = bigquery.delete(datasetId, DatasetDeleteOption.deleteContents()); + if (success) { + System.out.println("Dataset deleted successfully"); + } else { + System.out.println("Dataset was not found"); + } + } catch (BigQueryException e) { + System.out.println("Dataset was not deleted. \n" + e.toString()); } } } diff --git a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java deleted file mode 100644 index a9b4a9abe..000000000 --- a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.example.bigquery; - -// [START bigquery_extract_table] -import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.Table; - -public class ExtractTableToJSON { - - public static void runExtractTableToJSON() { - // TODO(developer): Replace these variables before running the sample. - Table table = null; - String format = "CSV"; - String bucketName = "my-bucket"; - String gcsFileName = "gs://" + bucketName + "/extractTest.csv"; - extractTableToJSON(table, format, gcsFileName); - } - - // Exports my-dataset-name:my_table to gcs://my-bucket/my-file as raw CSV - public static void extractTableToJSON(Table table, String format, String gcsFileName) { - - try { - table.extract(format, gcsFileName); - System.out.println("Table extraction job completed successfully"); - } catch (BigQueryException e) { - System.out.println("Table extraction job was interrupted. \n" + e.toString()); - } - } -} -// [END bigquery_extract_table] diff --git a/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java b/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java new file mode 100644 index 000000000..20e29135e --- /dev/null +++ b/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java @@ -0,0 +1,72 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigquery; + +// [START bigquery_extract_table] +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.Job; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableId; + +public class ExtractTableToJson { + + public static void runExtractTableToJson() { + // TODO(developer): Replace these variables before running the sample. + String projectId = "bigquery-public-data"; + String datasetName = "samples"; + String tableName = "shakespeare"; + String bucketName = "my-bucket"; + String destinationUri = "gs://" + bucketName + "/path/to/file"; + extractTableToJson(projectId, datasetName, tableName, destinationUri); + } + + // Exports datasetName:tableName to destinationUri as raw CSV + public static void extractTableToJson( + String projectId, String datasetName, String tableName, String destinationUri) { + try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + + TableId tableId = TableId.of(projectId, datasetName, tableName); + Table table = bigquery.getTable(tableId); + + // For more information on export formats available see: + // https://cloud.google.com/bigquery/docs/exporting-data#export_formats_and_compression_types + // For more information on Job see: + // https://googleapis.dev/java/google-cloud-clients/latest/index.html?com/google/cloud/bigquery/package-summary.html + Job job = table.extract("CSV", destinationUri); + + // Blocks until this job completes its execution, either failing or succeeding. + Job completedJob = job.waitFor(); + if (completedJob == null) { + System.out.println("Job not executed since it no longer exists."); + return; + } else if (completedJob.getStatus().getError() != null) { + System.out.println( + "BigQuery was unable to extract due to an error: \n" + job.getStatus().getError()); + return; + } + System.out.println("Table export successful. Check in GCS bucket for the CSV file."); + } catch (BigQueryException | InterruptedException e) { + System.out.println("Table extraction job was interrupted. \n" + e.toString()); + } + } +} +// [END bigquery_extract_table] diff --git a/samples/src/main/java/com/example/bigquery/ListDatasets.java b/samples/src/main/java/com/example/bigquery/ListDatasets.java index 69f8b91af..2c86c71aa 100644 --- a/samples/src/main/java/com/example/bigquery/ListDatasets.java +++ b/samples/src/main/java/com/example/bigquery/ListDatasets.java @@ -28,17 +28,16 @@ public class ListDatasets { public static void runListDatasets() { // TODO(developer): Replace these variables before running the sample. - String projectId = "my-project-id"; + String projectId = "MY_PROJECT_ID"; listDatasets(projectId); } public static void listDatasets(String projectId) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - - // List datasets in a specified project try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + Page datasets = bigquery.listDatasets(projectId, DatasetListOption.pageSize(100)); for (Dataset dataset : datasets.iterateAll()) { System.out.println(dataset.getDatasetId() + " dataset in project listed successfully"); diff --git a/samples/src/main/java/com/example/bigquery/UpdateDatasetAccess.java b/samples/src/main/java/com/example/bigquery/UpdateDatasetAccess.java index 1166a2baa..5719dcffa 100644 --- a/samples/src/main/java/com/example/bigquery/UpdateDatasetAccess.java +++ b/samples/src/main/java/com/example/bigquery/UpdateDatasetAccess.java @@ -30,28 +30,27 @@ public class UpdateDatasetAccess { public static void runUpdateDatasetAccess() { // TODO(developer): Replace these variables before running the sample. - String datasetName = "my-dataset-name"; + String datasetName = "MY_DATASET_NAME"; updateDatasetAccess(datasetName); } public static void updateDatasetAccess(String datasetName) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - Dataset dataset = bigquery.getDataset(datasetName); + Dataset dataset = bigquery.getDataset(datasetName); - // Create a new ACL granting the READER role to "sample.bigquery.dev@gmail.com" - // For more information on the types of ACLs available see: - // https://cloud.google.com/storage/docs/access-control/lists - Acl newEntry = Acl.of(new User("sample.bigquery.dev@gmail.com"), Role.READER); + // Create a new ACL granting the READER role to "sample.bigquery.dev@gmail.com" + // For more information on the types of ACLs available see: + // https://cloud.google.com/storage/docs/access-control/lists + Acl newEntry = Acl.of(new User("sample.bigquery.dev@gmail.com"), Role.READER); - // Get a copy of the ACLs list from the dataset and append the new entry - ArrayList acls = new ArrayList<>(dataset.getAcl()); - acls.add(newEntry); + // Get a copy of the ACLs list from the dataset and append the new entry + ArrayList acls = new ArrayList<>(dataset.getAcl()); + acls.add(newEntry); - // Update the dataset to use the new ACLs - try { bigquery.update(dataset.toBuilder().setAcl(acls).build()); System.out.println("Dataset Access Control updated successfully"); } catch (BigQueryException e) { diff --git a/samples/src/main/java/com/example/bigquery/UpdateDatasetDescription.java b/samples/src/main/java/com/example/bigquery/UpdateDatasetDescription.java index 8f6ab8a5c..5738ec0e5 100644 --- a/samples/src/main/java/com/example/bigquery/UpdateDatasetDescription.java +++ b/samples/src/main/java/com/example/bigquery/UpdateDatasetDescription.java @@ -26,20 +26,18 @@ public class UpdateDatasetDescription { public static void runUpdateDatasetDescription() { // TODO(developer): Replace these variables before running the sample. - String datasetName = "my-dataset-name"; + String datasetName = "MY_DATASET_NAME"; String newDescription = "this is the new dataset description"; updateDatasetDescription(datasetName, newDescription); } public static void updateDatasetDescription(String datasetName, String newDescription) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - - Dataset dataset = bigquery.getDataset(datasetName); - - // Update dataset description try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + + Dataset dataset = bigquery.getDataset(datasetName); bigquery.update(dataset.toBuilder().setDescription(newDescription).build()); System.out.println("Dataset description updated successfully to " + newDescription); } catch (BigQueryException e) { diff --git a/samples/src/main/java/com/example/bigquery/UpdateDatasetExpiration.java b/samples/src/main/java/com/example/bigquery/UpdateDatasetExpiration.java index 4cb16b4f4..bea27624f 100644 --- a/samples/src/main/java/com/example/bigquery/UpdateDatasetExpiration.java +++ b/samples/src/main/java/com/example/bigquery/UpdateDatasetExpiration.java @@ -27,20 +27,20 @@ public class UpdateDatasetExpiration { public static void runUpdateDatasetExpiration() { // TODO(developer): Replace these variables before running the sample. - String datasetName = "my-dataset-name"; + String datasetName = "MY_DATASET_NAME"; updateDatasetExpiration(datasetName); } public static void updateDatasetExpiration(String datasetName) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - Dataset dataset = bigquery.getDataset(datasetName); + // Update dataset expiration to one day + Long newExpiration = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS); - // Update dataset expiration to one day - Long newExpiration = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS); - try { + Dataset dataset = bigquery.getDataset(datasetName); bigquery.update(dataset.toBuilder().setDefaultTableLifetime(newExpiration).build()); System.out.println("Dataset description updated successfully to " + newExpiration); } catch (BigQueryException e) { diff --git a/samples/src/test/java/com/example/bigquery/CreateTableIT.java b/samples/src/test/java/com/example/bigquery/CreateTableIT.java index 09c490e2d..2572f6069 100644 --- a/samples/src/test/java/com/example/bigquery/CreateTableIT.java +++ b/samples/src/test/java/com/example/bigquery/CreateTableIT.java @@ -1,3 +1,19 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.example.bigquery; import static com.google.common.truth.Truth.assertThat; @@ -36,7 +52,7 @@ public void testCreateTable() { CreateDataset.createDataset(generatedDatasetName); // Create an empty table with specific schema in the dataset just created - String tableName = "my_table_name"; + String tableName = "MY_TABLE_NAME"; Schema schema = Schema.of( Field.of("stringField", LegacySQLTypeName.STRING), diff --git a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java deleted file mode 100644 index d515d5098..000000000 --- a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java +++ /dev/null @@ -1,76 +0,0 @@ -package com.example.bigquery; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.BigQueryOptions; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.LegacySQLTypeName; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.StandardTableDefinition; -import com.google.cloud.bigquery.Table; -import com.google.cloud.bigquery.TableDefinition; -import com.google.cloud.bigquery.TableId; -import com.google.cloud.bigquery.TableInfo; -import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class ExtractTableToJSONIT { - private ByteArrayOutputStream bout; - private PrintStream out; - - @Before - public void setUp() throws Exception { - bout = new ByteArrayOutputStream(); - out = new PrintStream(bout); - System.setOut(out); - } - - @After - public void tearDown() { - System.setOut(null); - } - - @Test - public void testExtractTableToJSON() { - String generatedDatasetName = RemoteBigQueryHelper.generateDatasetName(); - - // Create a new dataset to create a new table in - CreateDataset.createDataset(generatedDatasetName); - - // Create a new table to extract to GCS for - String tableName = "my_table_name"; - Schema schema = - Schema.of( - Field.of("stringField", LegacySQLTypeName.STRING), - Field.of("booleanField", LegacySQLTypeName.BOOLEAN)); - Table table = createTableHelper(generatedDatasetName, tableName, schema); - - // Extract table content to GCS in CSV format - ExtractTableToJSON.extractTableToJSON(table, "CSV", "gs://my-bucket/extractTest.csv"); - assertThat(bout.toString()).contains("Table extraction job completed successfully"); - } - - private static Table createTableHelper(String datasetName, String tableName, Schema schema) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - - TableId tableId = TableId.of(datasetName, tableName); - TableDefinition tableDefinition = StandardTableDefinition.of(schema); - TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build(); - - try { - Table table = bigquery.create(tableInfo); - return table; - } catch (BigQueryException e) { - System.out.println("Table was not created. \n" + e.toString()); - return null; - } - } -} diff --git a/samples/src/test/java/com/example/bigquery/ExtractTableToJsonIT.java b/samples/src/test/java/com/example/bigquery/ExtractTableToJsonIT.java new file mode 100644 index 000000000..5d2a2a0a0 --- /dev/null +++ b/samples/src/test/java/com/example/bigquery/ExtractTableToJsonIT.java @@ -0,0 +1,71 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigquery; + +import static com.google.common.truth.Truth.assertThat; +import static junit.framework.TestCase.assertNotNull; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ExtractTableToJsonIT { + private ByteArrayOutputStream bout; + private PrintStream out; + + private static final String GCS_BUCKET = System.getenv("GCS_BUCKET"); + + private static void requireEnvVar(String varName) { + assertNotNull( + "Environment variable '%s' is required to perform these tests.".format(varName), + System.getenv(varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnvVar("GCS_BUCKET"); + } + + @Before + public void setUp() throws Exception { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() { + System.setOut(null); + } + + @Test + public void testExtractTableToJson() { + String projectId = "bigquery-public-data"; + String datasetName = "samples"; + String tableName = "shakespeare"; + String destinationUri = "gs://" + GCS_BUCKET + "/extractTest.csv"; + System.out.println(destinationUri); + + // Extract table content to GCS in CSV format + ExtractTableToJson.extractTableToJson(projectId, datasetName, tableName, destinationUri); + assertThat(bout.toString()) + .contains("Table export successful. Check in GCS bucket for the CSV file."); + } +}