From 6da113884c4856bb1cacc0dd0453e8ccc4b3ed8b Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 17 Dec 2019 11:22:11 -0500 Subject: [PATCH 01/14] feat: add extract table to json sample --- .../example/bigquery/ExtractTableToJSON.java | 55 +++++++++++++++++++ .../bigquery/ExtractTableToJSONIT.java | 38 +++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java create mode 100644 samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java diff --git a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java new file mode 100644 index 000000000..b0b97877a --- /dev/null +++ b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java @@ -0,0 +1,55 @@ +package com.example.bigquery; + +// [START bigquery_extract_table] +import com.google.cloud.RetryOption; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQuery.TableOption; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.Job; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; +import org.threeten.bp.Duration; + +public class ExtractTableToJSON { + + public static void runExtractTableToJSON() { + // TODO(developer): Replace these variables before running the sample. + String datasetName = "my-dataset-name"; + String format = "CSV"; + String tableName = "my_table"; + String bucketName = "my-bucket"; + String gcsFileName = "gs://" + bucketName + "/extractTest.csv"; + extractTableToJSON(datasetName, tableName, format, gcsFileName); + } + + + // Exports my-dataset-name:my_table to gcs://my-bucket/my-file as raw CSV + public static void extractTableToJSON(String datasetName, String tableName, String format, String gcsFileName) { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + + //Create a table to extract to GCS + StandardTableDefinition.Builder builder = StandardTableDefinition.newBuilder(); + Table table = bigquery.create(TableInfo.of(TableId.of(datasetName, tableName), builder.build())); + + Job job = table.extract(format, gcsFileName); + try { + Job completedJob = + job.waitFor( + RetryOption.initialRetryDelay(Duration.ofSeconds(1)), + RetryOption.totalTimeout(Duration.ofMinutes(3))); + if (completedJob != null && completedJob.getStatus().getError() == null) { + System.out.println("table extraction job completed successfully"); + } else { + System.out.println("table extraction job failed" + table.toString() + completedJob.toString()); + } + } catch (InterruptedException e) { + System.out.println("table extraction job was interrupted. \n" + e.toString()); + } + } +} +// [END bigquery_extract_table] \ No newline at end of file diff --git a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java new file mode 100644 index 000000000..7889d0e00 --- /dev/null +++ b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java @@ -0,0 +1,38 @@ +package com.example.bigquery; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.*; + +import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class ExtractTableToJSONIT { + private ByteArrayOutputStream bout; + private PrintStream out; + + @Before + public void setUp() throws Exception { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() { + System.setOut(null); + } + + @Test + public void testExtractTableToJSON() { + String generatedDatasetName = RemoteBigQueryHelper.generateDatasetName(); + CreateDataset.createDataset(generatedDatasetName); + + ExtractTableToJSON.extractTableToJSON(generatedDatasetName, "my_table","CSV", "gs://my-bucket/extractTest.csv"); + assertThat(bout.toString()) + .contains("table extraction job completed successfully"); + } +} \ No newline at end of file From 15f09458e9932fbd5454e084c3da418a5b0a0b09 Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 17 Dec 2019 17:13:13 -0500 Subject: [PATCH 02/14] feat: add create table sample --- .../com/example/bigquery/CreateTable.java | 46 +++++++++++++++++ .../com/example/bigquery/CreateTableIT.java | 49 +++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 samples/src/main/java/com/example/bigquery/CreateTable.java create mode 100644 samples/src/test/java/com/example/bigquery/CreateTableIT.java diff --git a/samples/src/main/java/com/example/bigquery/CreateTable.java b/samples/src/main/java/com/example/bigquery/CreateTable.java new file mode 100644 index 000000000..e2cb956e2 --- /dev/null +++ b/samples/src/main/java/com/example/bigquery/CreateTable.java @@ -0,0 +1,46 @@ +package com.example.bigquery; + +// [START bigquery_create_table] +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.LegacySQLTypeName; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.TableDefinition; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; + +public class CreateTable { + + public static void runCreateTable() { + // TODO(developer): Replace these variables before running the sample. + String datasetName = "my-dataset-name"; + String tableName = "my_table_name"; + Schema schema = + Schema.of( + Field.of("stringField", LegacySQLTypeName.STRING), + Field.of("booleanField", LegacySQLTypeName.BOOLEAN)); + createTable(datasetName, tableName, schema); + } + + public static void createTable(String datasetName, String tableName, Schema schema) { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + + TableId tableId = TableId.of(datasetName, tableName); + TableDefinition tableDefinition = StandardTableDefinition.of(schema); + TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build(); + + try { + bigquery.create(tableInfo); + System.out.println("Table created successfully"); + } catch(BigQueryException e) { + System.out.println("Table was not created. \n" + e.toString()); + } + } +} +// [END bigquery_create_table] \ No newline at end of file diff --git a/samples/src/test/java/com/example/bigquery/CreateTableIT.java b/samples/src/test/java/com/example/bigquery/CreateTableIT.java new file mode 100644 index 000000000..228f86a30 --- /dev/null +++ b/samples/src/test/java/com/example/bigquery/CreateTableIT.java @@ -0,0 +1,49 @@ +package com.example.bigquery; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.*; + +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.LegacySQLTypeName; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class CreateTableIT { + private ByteArrayOutputStream bout; + private PrintStream out; + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() { + System.setOut(null); + } + + @Test + public void testCreateTable() { + String generatedDatasetName = RemoteBigQueryHelper.generateDatasetName(); + + //Create a new dataset to create a table in + CreateDataset.createDataset(generatedDatasetName); + + //Create an empty table with specific schema in the dataset just created + String tableName = "my_table_name"; + Schema schema = + Schema.of( + Field.of("stringField", LegacySQLTypeName.STRING), + Field.of("booleanField", LegacySQLTypeName.BOOLEAN)); + CreateTable.createTable(generatedDatasetName, tableName, schema); + + assertThat(bout.toString()).contains("Table created successfully"); + } +} \ No newline at end of file From 7032ee825886d31589e32fb8fce0397367d79739 Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 17 Dec 2019 17:13:41 -0500 Subject: [PATCH 03/14] feat: add extract table to json sample --- .../example/bigquery/ExtractTableToJSON.java | 20 ++++----- .../bigquery/ExtractTableToJSONIT.java | 42 ++++++++++++++++++- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java index b0b97877a..622986c8f 100644 --- a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java +++ b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java @@ -6,9 +6,13 @@ import com.google.cloud.bigquery.BigQuery.TableOption; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.Job; +import com.google.cloud.bigquery.LegacySQLTypeName; +import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardTableDefinition; import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableDefinition; import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableInfo; import org.threeten.bp.Duration; @@ -17,25 +21,15 @@ public class ExtractTableToJSON { public static void runExtractTableToJSON() { // TODO(developer): Replace these variables before running the sample. - String datasetName = "my-dataset-name"; + Table table = null; String format = "CSV"; - String tableName = "my_table"; String bucketName = "my-bucket"; String gcsFileName = "gs://" + bucketName + "/extractTest.csv"; - extractTableToJSON(datasetName, tableName, format, gcsFileName); + extractTableToJSON(table, format, gcsFileName); } - // Exports my-dataset-name:my_table to gcs://my-bucket/my-file as raw CSV - public static void extractTableToJSON(String datasetName, String tableName, String format, String gcsFileName) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - - //Create a table to extract to GCS - StandardTableDefinition.Builder builder = StandardTableDefinition.newBuilder(); - Table table = bigquery.create(TableInfo.of(TableId.of(datasetName, tableName), builder.build())); - + public static void extractTableToJSON(Table table, String format, String gcsFileName) { Job job = table.extract(format, gcsFileName); try { Job completedJob = diff --git a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java index 7889d0e00..cb2d8b21d 100644 --- a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java +++ b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java @@ -3,6 +3,17 @@ import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.*; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.LegacySQLTypeName; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableDefinition; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; import java.io.ByteArrayOutputStream; import java.io.PrintStream; @@ -29,10 +40,39 @@ public void tearDown() { @Test public void testExtractTableToJSON() { String generatedDatasetName = RemoteBigQueryHelper.generateDatasetName(); + + //Create a new dataset to create a new table in CreateDataset.createDataset(generatedDatasetName); - ExtractTableToJSON.extractTableToJSON(generatedDatasetName, "my_table","CSV", "gs://my-bucket/extractTest.csv"); + //Create a new table to extract to GCS for + String tableName = "my_table_name"; + Schema schema = + Schema.of( + Field.of("stringField", LegacySQLTypeName.STRING), + Field.of("booleanField", LegacySQLTypeName.BOOLEAN)); + Table table = createTableHelper(generatedDatasetName, tableName, schema); + + //Extract table content to GCS + ExtractTableToJSON.extractTableToJSON(table,"CSV", "gs://my-bucket/extractTest.csv"); assertThat(bout.toString()) .contains("table extraction job completed successfully"); } + + private static Table createTableHelper(String datasetName, String tableName, Schema schema) { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + + TableId tableId = TableId.of(datasetName, tableName); + TableDefinition tableDefinition = StandardTableDefinition.of(schema); + TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build(); + + try { + Table table = bigquery.create(tableInfo); + return table; + } catch(BigQueryException e) { + System.out.println("Table was not created. \n" + e.toString()); + return null; + } + } } \ No newline at end of file From dcfc812baa9bca52b955b18888e470671e28253c Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 17 Dec 2019 18:06:46 -0500 Subject: [PATCH 04/14] chore: clean up formatting --- .../com/example/bigquery/CreateTable.java | 2 +- .../example/bigquery/ExtractTableToJSON.java | 31 +++---------------- .../com/example/bigquery/CreateTableIT.java | 1 - .../bigquery/ExtractTableToJSONIT.java | 5 ++- 4 files changed, 8 insertions(+), 31 deletions(-) diff --git a/samples/src/main/java/com/example/bigquery/CreateTable.java b/samples/src/main/java/com/example/bigquery/CreateTable.java index e2cb956e2..764869bec 100644 --- a/samples/src/main/java/com/example/bigquery/CreateTable.java +++ b/samples/src/main/java/com/example/bigquery/CreateTable.java @@ -7,7 +7,6 @@ import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.LegacySQLTypeName; import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.StandardTableDefinition; import com.google.cloud.bigquery.TableDefinition; import com.google.cloud.bigquery.TableId; @@ -21,6 +20,7 @@ public static void runCreateTable() { String tableName = "my_table_name"; Schema schema = Schema.of( + // LegacySQLTypeName will be updated to StandardSQLTypeName once release rolls out Field.of("stringField", LegacySQLTypeName.STRING), Field.of("booleanField", LegacySQLTypeName.BOOLEAN)); createTable(datasetName, tableName, schema); diff --git a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java index 622986c8f..1db448b23 100644 --- a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java +++ b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java @@ -1,22 +1,8 @@ package com.example.bigquery; // [START bigquery_extract_table] -import com.google.cloud.RetryOption; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.BigQuery.TableOption; import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.BigQueryOptions; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.Job; -import com.google.cloud.bigquery.LegacySQLTypeName; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.StandardTableDefinition; import com.google.cloud.bigquery.Table; -import com.google.cloud.bigquery.TableDefinition; -import com.google.cloud.bigquery.TableId; -import com.google.cloud.bigquery.TableInfo; -import org.threeten.bp.Duration; - public class ExtractTableToJSON { public static void runExtractTableToJSON() { @@ -30,19 +16,12 @@ public static void runExtractTableToJSON() { // Exports my-dataset-name:my_table to gcs://my-bucket/my-file as raw CSV public static void extractTableToJSON(Table table, String format, String gcsFileName) { - Job job = table.extract(format, gcsFileName); + try { - Job completedJob = - job.waitFor( - RetryOption.initialRetryDelay(Duration.ofSeconds(1)), - RetryOption.totalTimeout(Duration.ofMinutes(3))); - if (completedJob != null && completedJob.getStatus().getError() == null) { - System.out.println("table extraction job completed successfully"); - } else { - System.out.println("table extraction job failed" + table.toString() + completedJob.toString()); - } - } catch (InterruptedException e) { - System.out.println("table extraction job was interrupted. \n" + e.toString()); + table.extract(format, gcsFileName); + System.out.println("Table extraction job completed successfully"); + } catch (BigQueryException e) { + System.out.println("Table extraction job was interrupted. \n" + e.toString()); } } } diff --git a/samples/src/test/java/com/example/bigquery/CreateTableIT.java b/samples/src/test/java/com/example/bigquery/CreateTableIT.java index 228f86a30..56295222c 100644 --- a/samples/src/test/java/com/example/bigquery/CreateTableIT.java +++ b/samples/src/test/java/com/example/bigquery/CreateTableIT.java @@ -1,7 +1,6 @@ package com.example.bigquery; import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.*; import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.LegacySQLTypeName; diff --git a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java index cb2d8b21d..e474bc86a 100644 --- a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java +++ b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java @@ -1,7 +1,6 @@ package com.example.bigquery; import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.*; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryException; @@ -52,10 +51,10 @@ public void testExtractTableToJSON() { Field.of("booleanField", LegacySQLTypeName.BOOLEAN)); Table table = createTableHelper(generatedDatasetName, tableName, schema); - //Extract table content to GCS + //Extract table content to GCS in CSV format ExtractTableToJSON.extractTableToJSON(table,"CSV", "gs://my-bucket/extractTest.csv"); assertThat(bout.toString()) - .contains("table extraction job completed successfully"); + .contains("Table extraction job completed successfully"); } private static Table createTableHelper(String datasetName, String tableName, Schema schema) { From 166ea16346078a0fb48cd00954b54d4e67662462 Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 18 Dec 2019 10:56:08 -0500 Subject: [PATCH 05/14] chore: update formatting according to comment --- samples/src/main/java/com/example/bigquery/CreateTable.java | 2 +- .../src/main/java/com/example/bigquery/ExtractTableToJSON.java | 2 +- samples/src/test/java/com/example/bigquery/CreateTableIT.java | 2 +- .../test/java/com/example/bigquery/ExtractTableToJSONIT.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/src/main/java/com/example/bigquery/CreateTable.java b/samples/src/main/java/com/example/bigquery/CreateTable.java index 764869bec..11fcd8889 100644 --- a/samples/src/main/java/com/example/bigquery/CreateTable.java +++ b/samples/src/main/java/com/example/bigquery/CreateTable.java @@ -43,4 +43,4 @@ public static void createTable(String datasetName, String tableName, Schema sche } } } -// [END bigquery_create_table] \ No newline at end of file +// [END bigquery_create_table] diff --git a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java index 1db448b23..2ec66e67f 100644 --- a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java +++ b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java @@ -25,4 +25,4 @@ public static void extractTableToJSON(Table table, String format, String gcsFile } } } -// [END bigquery_extract_table] \ No newline at end of file +// [END bigquery_extract_table] diff --git a/samples/src/test/java/com/example/bigquery/CreateTableIT.java b/samples/src/test/java/com/example/bigquery/CreateTableIT.java index 56295222c..af5dca585 100644 --- a/samples/src/test/java/com/example/bigquery/CreateTableIT.java +++ b/samples/src/test/java/com/example/bigquery/CreateTableIT.java @@ -45,4 +45,4 @@ public void testCreateTable() { assertThat(bout.toString()).contains("Table created successfully"); } -} \ No newline at end of file +} diff --git a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java index e474bc86a..95288fe0f 100644 --- a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java +++ b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java @@ -74,4 +74,4 @@ private static Table createTableHelper(String datasetName, String tableName, Sch return null; } } -} \ No newline at end of file +} From 116f215e932b8c2270f914e841dc3f69c855ddb6 Mon Sep 17 00:00:00 2001 From: stephwang Date: Wed, 18 Dec 2019 11:07:48 -0500 Subject: [PATCH 06/14] chore: update formatting according to comment --- .../main/java/com/example/bigquery/CreateTable.java | 2 +- .../com/example/bigquery/ExtractTableToJSON.java | 1 + .../java/com/example/bigquery/CreateTableIT.java | 4 ++-- .../com/example/bigquery/ExtractTableToJSONIT.java | 13 ++++++------- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/samples/src/main/java/com/example/bigquery/CreateTable.java b/samples/src/main/java/com/example/bigquery/CreateTable.java index 11fcd8889..0f56495b4 100644 --- a/samples/src/main/java/com/example/bigquery/CreateTable.java +++ b/samples/src/main/java/com/example/bigquery/CreateTable.java @@ -38,7 +38,7 @@ public static void createTable(String datasetName, String tableName, Schema sche try { bigquery.create(tableInfo); System.out.println("Table created successfully"); - } catch(BigQueryException e) { + } catch (BigQueryException e) { System.out.println("Table was not created. \n" + e.toString()); } } diff --git a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java index 2ec66e67f..a9b4a9abe 100644 --- a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java +++ b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java @@ -3,6 +3,7 @@ // [START bigquery_extract_table] import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.Table; + public class ExtractTableToJSON { public static void runExtractTableToJSON() { diff --git a/samples/src/test/java/com/example/bigquery/CreateTableIT.java b/samples/src/test/java/com/example/bigquery/CreateTableIT.java index af5dca585..09c490e2d 100644 --- a/samples/src/test/java/com/example/bigquery/CreateTableIT.java +++ b/samples/src/test/java/com/example/bigquery/CreateTableIT.java @@ -32,10 +32,10 @@ public void tearDown() { public void testCreateTable() { String generatedDatasetName = RemoteBigQueryHelper.generateDatasetName(); - //Create a new dataset to create a table in + // Create a new dataset to create a table in CreateDataset.createDataset(generatedDatasetName); - //Create an empty table with specific schema in the dataset just created + // Create an empty table with specific schema in the dataset just created String tableName = "my_table_name"; Schema schema = Schema.of( diff --git a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java index 95288fe0f..d515d5098 100644 --- a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java +++ b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java @@ -40,10 +40,10 @@ public void tearDown() { public void testExtractTableToJSON() { String generatedDatasetName = RemoteBigQueryHelper.generateDatasetName(); - //Create a new dataset to create a new table in + // Create a new dataset to create a new table in CreateDataset.createDataset(generatedDatasetName); - //Create a new table to extract to GCS for + // Create a new table to extract to GCS for String tableName = "my_table_name"; Schema schema = Schema.of( @@ -51,10 +51,9 @@ public void testExtractTableToJSON() { Field.of("booleanField", LegacySQLTypeName.BOOLEAN)); Table table = createTableHelper(generatedDatasetName, tableName, schema); - //Extract table content to GCS in CSV format - ExtractTableToJSON.extractTableToJSON(table,"CSV", "gs://my-bucket/extractTest.csv"); - assertThat(bout.toString()) - .contains("Table extraction job completed successfully"); + // Extract table content to GCS in CSV format + ExtractTableToJSON.extractTableToJSON(table, "CSV", "gs://my-bucket/extractTest.csv"); + assertThat(bout.toString()).contains("Table extraction job completed successfully"); } private static Table createTableHelper(String datasetName, String tableName, Schema schema) { @@ -69,7 +68,7 @@ private static Table createTableHelper(String datasetName, String tableName, Sch try { Table table = bigquery.create(tableInfo); return table; - } catch(BigQueryException e) { + } catch (BigQueryException e) { System.out.println("Table was not created. \n" + e.toString()); return null; } From d4a92236b7bad9eb71cb865664681a4d8787305f Mon Sep 17 00:00:00 2001 From: stephwang Date: Thu, 19 Dec 2019 14:22:12 -0500 Subject: [PATCH 07/14] update based on kurtis' comments --- .../com/example/bigquery/CreateDataset.java | 2 +- .../com/example/bigquery/CreateTable.java | 22 ++++++- .../com/example/bigquery/DeleteDataset.java | 4 +- .../example/bigquery/ExtractTableToJSON.java | 65 +++++++++++++++++-- .../com/example/bigquery/ListDatasets.java | 2 +- .../example/bigquery/UpdateDatasetAccess.java | 2 +- .../bigquery/UpdateDatasetDescription.java | 2 +- .../bigquery/UpdateDatasetExpiration.java | 2 +- .../com/example/bigquery/CreateTableIT.java | 2 +- .../bigquery/ExtractTableToJSONIT.java | 2 +- 10 files changed, 88 insertions(+), 17 deletions(-) diff --git a/samples/src/main/java/com/example/bigquery/CreateDataset.java b/samples/src/main/java/com/example/bigquery/CreateDataset.java index f61fce657..ca6c944d8 100644 --- a/samples/src/main/java/com/example/bigquery/CreateDataset.java +++ b/samples/src/main/java/com/example/bigquery/CreateDataset.java @@ -27,7 +27,7 @@ public class CreateDataset { public static void runCreateDataset() { // TODO(developer): Replace these variables before running the sample. - String datasetName = "my-dataset-name"; + String datasetName = "MY_DATASET_NAME"; createDataset(datasetName); } diff --git a/samples/src/main/java/com/example/bigquery/CreateTable.java b/samples/src/main/java/com/example/bigquery/CreateTable.java index 0f56495b4..47f80d0c0 100644 --- a/samples/src/main/java/com/example/bigquery/CreateTable.java +++ b/samples/src/main/java/com/example/bigquery/CreateTable.java @@ -1,3 +1,19 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.example.bigquery; // [START bigquery_create_table] @@ -16,11 +32,11 @@ public class CreateTable { public static void runCreateTable() { // TODO(developer): Replace these variables before running the sample. - String datasetName = "my-dataset-name"; - String tableName = "my_table_name"; + String datasetName = "MY_DATASET_NAME"; + String tableName = "MY_TABLE_NAME"; Schema schema = Schema.of( - // LegacySQLTypeName will be updated to StandardSQLTypeName once release rolls out + // INFO: LegacySQLTypeName will be updated to StandardSQLTypeName in release 1.103.0 Field.of("stringField", LegacySQLTypeName.STRING), Field.of("booleanField", LegacySQLTypeName.BOOLEAN)); createTable(datasetName, tableName, schema); diff --git a/samples/src/main/java/com/example/bigquery/DeleteDataset.java b/samples/src/main/java/com/example/bigquery/DeleteDataset.java index f0f8ac96a..ec8cfd307 100644 --- a/samples/src/main/java/com/example/bigquery/DeleteDataset.java +++ b/samples/src/main/java/com/example/bigquery/DeleteDataset.java @@ -26,8 +26,8 @@ public class DeleteDataset { public static void runDeleteDataset() { // TODO(developer): Replace these variables before running the sample.\ - String projectId = "my-project-id"; - String datasetName = "my-dataset-name"; + String projectId = "MY_PROJECT_ID"; + String datasetName = "MY_DATASET_NAME"; deleteDataset(projectId, datasetName); } diff --git a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java index a9b4a9abe..01148d088 100644 --- a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java +++ b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java @@ -1,29 +1,84 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.example.bigquery; // [START bigquery_extract_table] +import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.Job; +import com.google.cloud.bigquery.LegacySQLTypeName; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardTableDefinition; import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableDefinition; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; public class ExtractTableToJSON { public static void runExtractTableToJSON() { // TODO(developer): Replace these variables before running the sample. - Table table = null; + String datasetName = "MY_DATASET_NAME"; String format = "CSV"; String bucketName = "my-bucket"; - String gcsFileName = "gs://" + bucketName + "/extractTest.csv"; + String gcsFileName = "gs://" + bucketName + "/path/to/file"; + // Create a new table to extract to GCS as CSV + String tableName = "MY_TABLE_NAME"; + Schema schema = + Schema.of( + Field.of("stringField", LegacySQLTypeName.STRING), + Field.of("booleanField", LegacySQLTypeName.BOOLEAN)); + Table table = createTableHelper(datasetName, tableName, schema); + + //Extract table extractTableToJSON(table, format, gcsFileName); } // Exports my-dataset-name:my_table to gcs://my-bucket/my-file as raw CSV public static void extractTableToJSON(Table table, String format, String gcsFileName) { - try { - table.extract(format, gcsFileName); - System.out.println("Table extraction job completed successfully"); + Job job = table.extract(format, gcsFileName); + if(job != null && job.getStatus().getError() == null) + System.out.println("Table extraction job completed successfully. Check in GCS bucket for the CSV file."); + else + System.out.println("Table extraction job failed"); } catch (BigQueryException e) { System.out.println("Table extraction job was interrupted. \n" + e.toString()); } } + + private static Table createTableHelper(String datasetName, String tableName, Schema schema) { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + + TableId tableId = TableId.of(datasetName, tableName); + TableDefinition tableDefinition = StandardTableDefinition.of(schema); + TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build(); + + try { + Table table = bigquery.create(tableInfo); + return table; + } catch (BigQueryException e) { + System.out.println("Table was not created. \n" + e.toString()); + return null; + } + } } // [END bigquery_extract_table] diff --git a/samples/src/main/java/com/example/bigquery/ListDatasets.java b/samples/src/main/java/com/example/bigquery/ListDatasets.java index 69f8b91af..394a464e7 100644 --- a/samples/src/main/java/com/example/bigquery/ListDatasets.java +++ b/samples/src/main/java/com/example/bigquery/ListDatasets.java @@ -28,7 +28,7 @@ public class ListDatasets { public static void runListDatasets() { // TODO(developer): Replace these variables before running the sample. - String projectId = "my-project-id"; + String projectId = "MY_PROJECT_ID"; listDatasets(projectId); } diff --git a/samples/src/main/java/com/example/bigquery/UpdateDatasetAccess.java b/samples/src/main/java/com/example/bigquery/UpdateDatasetAccess.java index 1166a2baa..6d39526a2 100644 --- a/samples/src/main/java/com/example/bigquery/UpdateDatasetAccess.java +++ b/samples/src/main/java/com/example/bigquery/UpdateDatasetAccess.java @@ -30,7 +30,7 @@ public class UpdateDatasetAccess { public static void runUpdateDatasetAccess() { // TODO(developer): Replace these variables before running the sample. - String datasetName = "my-dataset-name"; + String datasetName = "MY_DATASET_NAME"; updateDatasetAccess(datasetName); } diff --git a/samples/src/main/java/com/example/bigquery/UpdateDatasetDescription.java b/samples/src/main/java/com/example/bigquery/UpdateDatasetDescription.java index 8f6ab8a5c..1fe65a7ff 100644 --- a/samples/src/main/java/com/example/bigquery/UpdateDatasetDescription.java +++ b/samples/src/main/java/com/example/bigquery/UpdateDatasetDescription.java @@ -26,7 +26,7 @@ public class UpdateDatasetDescription { public static void runUpdateDatasetDescription() { // TODO(developer): Replace these variables before running the sample. - String datasetName = "my-dataset-name"; + String datasetName = "MY_DATASET_NAME"; String newDescription = "this is the new dataset description"; updateDatasetDescription(datasetName, newDescription); } diff --git a/samples/src/main/java/com/example/bigquery/UpdateDatasetExpiration.java b/samples/src/main/java/com/example/bigquery/UpdateDatasetExpiration.java index 4cb16b4f4..ef7c97549 100644 --- a/samples/src/main/java/com/example/bigquery/UpdateDatasetExpiration.java +++ b/samples/src/main/java/com/example/bigquery/UpdateDatasetExpiration.java @@ -27,7 +27,7 @@ public class UpdateDatasetExpiration { public static void runUpdateDatasetExpiration() { // TODO(developer): Replace these variables before running the sample. - String datasetName = "my-dataset-name"; + String datasetName = "MY_DATASET_NAME"; updateDatasetExpiration(datasetName); } diff --git a/samples/src/test/java/com/example/bigquery/CreateTableIT.java b/samples/src/test/java/com/example/bigquery/CreateTableIT.java index 09c490e2d..14a004c8c 100644 --- a/samples/src/test/java/com/example/bigquery/CreateTableIT.java +++ b/samples/src/test/java/com/example/bigquery/CreateTableIT.java @@ -36,7 +36,7 @@ public void testCreateTable() { CreateDataset.createDataset(generatedDatasetName); // Create an empty table with specific schema in the dataset just created - String tableName = "my_table_name"; + String tableName = "MY_TABLE_NAME"; Schema schema = Schema.of( Field.of("stringField", LegacySQLTypeName.STRING), diff --git a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java index d515d5098..e28c70ec6 100644 --- a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java +++ b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java @@ -44,7 +44,7 @@ public void testExtractTableToJSON() { CreateDataset.createDataset(generatedDatasetName); // Create a new table to extract to GCS for - String tableName = "my_table_name"; + String tableName = "MY_TABLE_NAME"; Schema schema = Schema.of( Field.of("stringField", LegacySQLTypeName.STRING), From 1cce66fe653b0baffd9627d503a9d3b9c5cfe35c Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 23 Dec 2019 11:29:55 -0500 Subject: [PATCH 08/14] updates based on comments --- .../example/bigquery/ExtractTableToJSON.java | 53 ++++++------------- .../bigquery/ExtractTableToJSONIT.java | 45 +--------------- 2 files changed, 17 insertions(+), 81 deletions(-) diff --git a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java index 01148d088..d8f0e619c 100644 --- a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java +++ b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java @@ -20,41 +20,38 @@ import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryOptions; -import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.Job; -import com.google.cloud.bigquery.LegacySQLTypeName; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.StandardTableDefinition; import com.google.cloud.bigquery.Table; -import com.google.cloud.bigquery.TableDefinition; import com.google.cloud.bigquery.TableId; -import com.google.cloud.bigquery.TableInfo; public class ExtractTableToJSON { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + private static BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + + private static String projectId = "bigquery-public-data"; + private static String datasetName = "samples"; + private static String tableName = "shakespeare"; + private static TableId tableId = TableId.of(projectId, datasetName, tableName); + private static Table table = bigquery.getTable(tableId); public static void runExtractTableToJSON() { // TODO(developer): Replace these variables before running the sample. - String datasetName = "MY_DATASET_NAME"; + // For more information on export format available see: + // https://cloud.google.com/bigquery/docs/exporting-data#export_formats_and_compression_types String format = "CSV"; String bucketName = "my-bucket"; - String gcsFileName = "gs://" + bucketName + "/path/to/file"; - // Create a new table to extract to GCS as CSV - String tableName = "MY_TABLE_NAME"; - Schema schema = - Schema.of( - Field.of("stringField", LegacySQLTypeName.STRING), - Field.of("booleanField", LegacySQLTypeName.BOOLEAN)); - Table table = createTableHelper(datasetName, tableName, schema); + String destinationUri = "gs://" + bucketName + "/path/to/file"; //Extract table - extractTableToJSON(table, format, gcsFileName); + extractTableToJSON(format, destinationUri); } // Exports my-dataset-name:my_table to gcs://my-bucket/my-file as raw CSV - public static void extractTableToJSON(Table table, String format, String gcsFileName) { + public static void extractTableToJSON(String format, String destinationUri) { + Job job = table.extract(format, destinationUri); try { - Job job = table.extract(format, gcsFileName); - if(job != null && job.getStatus().getError() == null) + if (job != null && job.getStatus().getError() == null) System.out.println("Table extraction job completed successfully. Check in GCS bucket for the CSV file."); else System.out.println("Table extraction job failed"); @@ -62,23 +59,5 @@ public static void extractTableToJSON(Table table, String format, String gcsFile System.out.println("Table extraction job was interrupted. \n" + e.toString()); } } - - private static Table createTableHelper(String datasetName, String tableName, Schema schema) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - - TableId tableId = TableId.of(datasetName, tableName); - TableDefinition tableDefinition = StandardTableDefinition.of(schema); - TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build(); - - try { - Table table = bigquery.create(tableInfo); - return table; - } catch (BigQueryException e) { - System.out.println("Table was not created. \n" + e.toString()); - return null; - } - } } // [END bigquery_extract_table] diff --git a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java index e28c70ec6..8de59115a 100644 --- a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java +++ b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java @@ -2,18 +2,6 @@ import static com.google.common.truth.Truth.assertThat; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.BigQueryOptions; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.LegacySQLTypeName; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.StandardTableDefinition; -import com.google.cloud.bigquery.Table; -import com.google.cloud.bigquery.TableDefinition; -import com.google.cloud.bigquery.TableId; -import com.google.cloud.bigquery.TableInfo; -import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import org.junit.After; @@ -38,39 +26,8 @@ public void tearDown() { @Test public void testExtractTableToJSON() { - String generatedDatasetName = RemoteBigQueryHelper.generateDatasetName(); - - // Create a new dataset to create a new table in - CreateDataset.createDataset(generatedDatasetName); - - // Create a new table to extract to GCS for - String tableName = "MY_TABLE_NAME"; - Schema schema = - Schema.of( - Field.of("stringField", LegacySQLTypeName.STRING), - Field.of("booleanField", LegacySQLTypeName.BOOLEAN)); - Table table = createTableHelper(generatedDatasetName, tableName, schema); - // Extract table content to GCS in CSV format - ExtractTableToJSON.extractTableToJSON(table, "CSV", "gs://my-bucket/extractTest.csv"); + ExtractTableToJSON.extractTableToJSON("CSV", "gs://my-bucket/extractTest.csv"); assertThat(bout.toString()).contains("Table extraction job completed successfully"); } - - private static Table createTableHelper(String datasetName, String tableName, Schema schema) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - - TableId tableId = TableId.of(datasetName, tableName); - TableDefinition tableDefinition = StandardTableDefinition.of(schema); - TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build(); - - try { - Table table = bigquery.create(tableInfo); - return table; - } catch (BigQueryException e) { - System.out.println("Table was not created. \n" + e.toString()); - return null; - } - } } From f6ba1c8958f985b281ae82c03d750de55dee88cc Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 23 Dec 2019 12:21:52 -0500 Subject: [PATCH 09/14] updates based on comments --- .../example/bigquery/ExtractTableToJSON.java | 39 ++++++++++--------- .../bigquery/ExtractTableToJSONIT.java | 3 +- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java index d8f0e619c..1dc99165c 100644 --- a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java +++ b/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java @@ -25,36 +25,37 @@ import com.google.cloud.bigquery.TableId; public class ExtractTableToJSON { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - private static BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - - private static String projectId = "bigquery-public-data"; - private static String datasetName = "samples"; - private static String tableName = "shakespeare"; - private static TableId tableId = TableId.of(projectId, datasetName, tableName); - private static Table table = bigquery.getTable(tableId); public static void runExtractTableToJSON() { // TODO(developer): Replace these variables before running the sample. - // For more information on export format available see: - // https://cloud.google.com/bigquery/docs/exporting-data#export_formats_and_compression_types - String format = "CSV"; + String projectId = "bigquery-public-data"; + String datasetName = "samples"; + String tableName = "shakespeare"; String bucketName = "my-bucket"; String destinationUri = "gs://" + bucketName + "/path/to/file"; - //Extract table - extractTableToJSON(format, destinationUri); + // Extract table + extractTableToJSON(projectId, datasetName, tableName, destinationUri); } // Exports my-dataset-name:my_table to gcs://my-bucket/my-file as raw CSV - public static void extractTableToJSON(String format, String destinationUri) { - Job job = table.extract(format, destinationUri); + public static void extractTableToJSON( + String projectId, String datasetName, String tableName, String destinationUri) { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + + TableId tableId = TableId.of(projectId, datasetName, tableName); + Table table = bigquery.getTable(tableId); + + // For more information on export format available see: + // https://cloud.google.com/bigquery/docs/exporting-data#export_formats_and_compression_types + Job job = table.extract("CSV", destinationUri); try { if (job != null && job.getStatus().getError() == null) - System.out.println("Table extraction job completed successfully. Check in GCS bucket for the CSV file."); - else - System.out.println("Table extraction job failed"); + System.out.println( + "Table extraction job completed successfully. Check in GCS bucket for the CSV file."); + else System.out.println("Table extraction job failed"); } catch (BigQueryException e) { System.out.println("Table extraction job was interrupted. \n" + e.toString()); } diff --git a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java index 8de59115a..70c6bd010 100644 --- a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java +++ b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java @@ -27,7 +27,8 @@ public void tearDown() { @Test public void testExtractTableToJSON() { // Extract table content to GCS in CSV format - ExtractTableToJSON.extractTableToJSON("CSV", "gs://my-bucket/extractTest.csv"); + ExtractTableToJSON.extractTableToJSON( + "bigquery-public-data", "samples", "shakespeare", "gs://my-bucket/extractTest.csv"); assertThat(bout.toString()).contains("Table extraction job completed successfully"); } } From 0c00dcc23465b111ffaf7310530206618369a4b1 Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 23 Dec 2019 14:34:22 -0500 Subject: [PATCH 10/14] updates based on comments --- samples/pom.xml | 8 +++ ...bleToJSON.java => ExtractTableToJson.java} | 34 ++++++---- .../com/example/bigquery/CreateTableIT.java | 16 +++++ .../bigquery/ExtractTableToJSONIT.java | 34 ---------- .../bigquery/ExtractTableToJsonIT.java | 63 +++++++++++++++++++ 5 files changed, 109 insertions(+), 46 deletions(-) rename samples/src/main/java/com/example/bigquery/{ExtractTableToJSON.java => ExtractTableToJson.java} (66%) delete mode 100644 samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java create mode 100644 samples/src/test/java/com/example/bigquery/ExtractTableToJsonIT.java diff --git a/samples/pom.xml b/samples/pom.xml index f5b000e9f..18ba550f5 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -86,6 +86,14 @@ 1.0 test + + com.google.cloud + google-cloud-storage + + + com.google.cloud + google-cloud-storage + diff --git a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java b/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java similarity index 66% rename from samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java rename to samples/src/main/java/com/example/bigquery/ExtractTableToJson.java index 1dc99165c..3f53cd169 100644 --- a/samples/src/main/java/com/example/bigquery/ExtractTableToJSON.java +++ b/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java @@ -18,28 +18,25 @@ // [START bigquery_extract_table] import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableId; -public class ExtractTableToJSON { +public class ExtractTableToJson { - public static void runExtractTableToJSON() { + public static void runextracttabletojson() { // TODO(developer): Replace these variables before running the sample. String projectId = "bigquery-public-data"; String datasetName = "samples"; String tableName = "shakespeare"; String bucketName = "my-bucket"; String destinationUri = "gs://" + bucketName + "/path/to/file"; - - // Extract table - extractTableToJSON(projectId, datasetName, tableName, destinationUri); + extractTableToJson(projectId, datasetName, tableName, destinationUri); } - // Exports my-dataset-name:my_table to gcs://my-bucket/my-file as raw CSV - public static void extractTableToJSON( + // Exports datasetName:tableName to destinationUri as raw CSV + public static void extractTableToJson( String projectId, String datasetName, String tableName, String destinationUri) { // Initialize client that will be used to send requests. This client only needs to be created // once, and can be reused for multiple requests. @@ -50,13 +47,26 @@ public static void extractTableToJSON( // For more information on export format available see: // https://cloud.google.com/bigquery/docs/exporting-data#export_formats_and_compression_types + // For more information on Job see: + // https://googleapis.dev/java/google-cloud-clients/latest/index.html?com/google/cloud/bigquery/package-summary.html Job job = table.extract("CSV", destinationUri); try { - if (job != null && job.getStatus().getError() == null) + Job completedJob = job.waitFor(); + if (completedJob == null) { + // Job no longer exists + System.out.println("Job no longer exists"); + return; + } else if (completedJob.getStatus().getError() != null) { + // Job failed, handle error System.out.println( - "Table extraction job completed successfully. Check in GCS bucket for the CSV file."); - else System.out.println("Table extraction job failed"); - } catch (BigQueryException e) { + "BigQuery was unable to extract due to an error: \n" + job.getStatus().getError()); + return; + } else { + // Job completed successfully + System.out.println("Table export successful. Check in GCS bucket for the CSV file."); + } + } catch (InterruptedException e) { + // Handle interrupted wait System.out.println("Table extraction job was interrupted. \n" + e.toString()); } } diff --git a/samples/src/test/java/com/example/bigquery/CreateTableIT.java b/samples/src/test/java/com/example/bigquery/CreateTableIT.java index 14a004c8c..2572f6069 100644 --- a/samples/src/test/java/com/example/bigquery/CreateTableIT.java +++ b/samples/src/test/java/com/example/bigquery/CreateTableIT.java @@ -1,3 +1,19 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.example.bigquery; import static com.google.common.truth.Truth.assertThat; diff --git a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java b/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java deleted file mode 100644 index 70c6bd010..000000000 --- a/samples/src/test/java/com/example/bigquery/ExtractTableToJSONIT.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.example.bigquery; - -import static com.google.common.truth.Truth.assertThat; - -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class ExtractTableToJSONIT { - private ByteArrayOutputStream bout; - private PrintStream out; - - @Before - public void setUp() throws Exception { - bout = new ByteArrayOutputStream(); - out = new PrintStream(bout); - System.setOut(out); - } - - @After - public void tearDown() { - System.setOut(null); - } - - @Test - public void testExtractTableToJSON() { - // Extract table content to GCS in CSV format - ExtractTableToJSON.extractTableToJSON( - "bigquery-public-data", "samples", "shakespeare", "gs://my-bucket/extractTest.csv"); - assertThat(bout.toString()).contains("Table extraction job completed successfully"); - } -} diff --git a/samples/src/test/java/com/example/bigquery/ExtractTableToJsonIT.java b/samples/src/test/java/com/example/bigquery/ExtractTableToJsonIT.java new file mode 100644 index 000000000..524fe640b --- /dev/null +++ b/samples/src/test/java/com/example/bigquery/ExtractTableToJsonIT.java @@ -0,0 +1,63 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigquery; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.testing.RemoteStorageHelper; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class ExtractTableToJsonIT { + private ByteArrayOutputStream bout; + private PrintStream out; + + @Before + public void setUp() throws Exception { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() { + System.setOut(null); + } + + @Test + public void testExtractTableToJson() { + String projectId = "bigquery-public-data"; + String datasetName = "samples"; + String tableName = "shakespeare"; + String bucketName = RemoteStorageHelper.generateBucketName(); + String destinationUri = "gs://" + bucketName + "/extractTest.csv"; + + // Create GCS bucket to store extracted file + Storage storage = RemoteStorageHelper.create().getOptions().getService(); + storage.create(BucketInfo.of(bucketName)); + + // Extract table content to GCS in CSV format + ExtractTableToJson.extractTableToJson(projectId, datasetName, tableName, destinationUri); + assertThat(bout.toString()) + .contains("Table export successful. Check in GCS bucket for the CSV file."); + } +} From 672d2b11faf2654b4254f4f03364509336d3df93 Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 23 Dec 2019 18:18:01 -0500 Subject: [PATCH 11/14] updates based on comments --- .../example/bigquery/ExtractTableToJson.java | 17 +++++------- .../bigquery/ExtractTableToJsonIT.java | 26 ++++++++++++------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java b/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java index 3f53cd169..19bc5960a 100644 --- a/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java +++ b/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java @@ -18,6 +18,7 @@ // [START bigquery_extract_table] import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.Table; @@ -25,7 +26,7 @@ public class ExtractTableToJson { - public static void runextracttabletojson() { + public static void runExtractTableToJson() { // TODO(developer): Replace these variables before running the sample. String projectId = "bigquery-public-data"; String datasetName = "samples"; @@ -45,28 +46,24 @@ public static void extractTableToJson( TableId tableId = TableId.of(projectId, datasetName, tableName); Table table = bigquery.getTable(tableId); - // For more information on export format available see: + // For more information on export formats available see: // https://cloud.google.com/bigquery/docs/exporting-data#export_formats_and_compression_types // For more information on Job see: // https://googleapis.dev/java/google-cloud-clients/latest/index.html?com/google/cloud/bigquery/package-summary.html Job job = table.extract("CSV", destinationUri); try { + // Blocks until this job completes its execution, either failing or succeeding. Job completedJob = job.waitFor(); if (completedJob == null) { - // Job no longer exists - System.out.println("Job no longer exists"); + System.out.println("Job not executed since it no longer exists."); return; } else if (completedJob.getStatus().getError() != null) { - // Job failed, handle error System.out.println( "BigQuery was unable to extract due to an error: \n" + job.getStatus().getError()); return; - } else { - // Job completed successfully - System.out.println("Table export successful. Check in GCS bucket for the CSV file."); } - } catch (InterruptedException e) { - // Handle interrupted wait + System.out.println("Table export successful. Check in GCS bucket for the CSV file."); + } catch (BigQueryException | InterruptedException e) { System.out.println("Table extraction job was interrupted. \n" + e.toString()); } } diff --git a/samples/src/test/java/com/example/bigquery/ExtractTableToJsonIT.java b/samples/src/test/java/com/example/bigquery/ExtractTableToJsonIT.java index 524fe640b..5d2a2a0a0 100644 --- a/samples/src/test/java/com/example/bigquery/ExtractTableToJsonIT.java +++ b/samples/src/test/java/com/example/bigquery/ExtractTableToJsonIT.java @@ -17,20 +17,32 @@ package com.example.bigquery; import static com.google.common.truth.Truth.assertThat; +import static junit.framework.TestCase.assertNotNull; -import com.google.cloud.storage.BucketInfo; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.testing.RemoteStorageHelper; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import org.junit.After; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; public class ExtractTableToJsonIT { private ByteArrayOutputStream bout; private PrintStream out; + private static final String GCS_BUCKET = System.getenv("GCS_BUCKET"); + + private static void requireEnvVar(String varName) { + assertNotNull( + "Environment variable '%s' is required to perform these tests.".format(varName), + System.getenv(varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnvVar("GCS_BUCKET"); + } + @Before public void setUp() throws Exception { bout = new ByteArrayOutputStream(); @@ -48,12 +60,8 @@ public void testExtractTableToJson() { String projectId = "bigquery-public-data"; String datasetName = "samples"; String tableName = "shakespeare"; - String bucketName = RemoteStorageHelper.generateBucketName(); - String destinationUri = "gs://" + bucketName + "/extractTest.csv"; - - // Create GCS bucket to store extracted file - Storage storage = RemoteStorageHelper.create().getOptions().getService(); - storage.create(BucketInfo.of(bucketName)); + String destinationUri = "gs://" + GCS_BUCKET + "/extractTest.csv"; + System.out.println(destinationUri); // Extract table content to GCS in CSV format ExtractTableToJson.extractTableToJson(projectId, datasetName, tableName, destinationUri); From 0ba4c34e87a357a6b5eda78eec01d04962d1e72d Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 23 Dec 2019 18:23:43 -0500 Subject: [PATCH 12/14] remove unused storage deps --- samples/pom.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/samples/pom.xml b/samples/pom.xml index 18ba550f5..f5b000e9f 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -86,14 +86,6 @@ 1.0 test - - com.google.cloud - google-cloud-storage - - - com.google.cloud - google-cloud-storage - From 07fba2fa9342417305df725203330913e256a4c3 Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 23 Dec 2019 21:38:42 -0500 Subject: [PATCH 13/14] move function calls into try/catch block where appropriate --- .../example/bigquery/ExtractTableToJson.java | 17 +++++++++-------- .../example/bigquery/UpdateDatasetAccess.java | 19 +++++++++---------- .../bigquery/UpdateDatasetDescription.java | 4 +--- .../bigquery/UpdateDatasetExpiration.java | 3 +-- 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java b/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java index 19bc5960a..5bc2e7d2f 100644 --- a/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java +++ b/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java @@ -43,15 +43,16 @@ public static void extractTableToJson( // once, and can be reused for multiple requests. BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - TableId tableId = TableId.of(projectId, datasetName, tableName); - Table table = bigquery.getTable(tableId); - - // For more information on export formats available see: - // https://cloud.google.com/bigquery/docs/exporting-data#export_formats_and_compression_types - // For more information on Job see: - // https://googleapis.dev/java/google-cloud-clients/latest/index.html?com/google/cloud/bigquery/package-summary.html - Job job = table.extract("CSV", destinationUri); try { + TableId tableId = TableId.of(projectId, datasetName, tableName); + Table table = bigquery.getTable(tableId); + + // For more information on export formats available see: + // https://cloud.google.com/bigquery/docs/exporting-data#export_formats_and_compression_types + // For more information on Job see: + // https://googleapis.dev/java/google-cloud-clients/latest/index.html?com/google/cloud/bigquery/package-summary.html + Job job = table.extract("CSV", destinationUri); + // Blocks until this job completes its execution, either failing or succeeding. Job completedJob = job.waitFor(); if (completedJob == null) { diff --git a/samples/src/main/java/com/example/bigquery/UpdateDatasetAccess.java b/samples/src/main/java/com/example/bigquery/UpdateDatasetAccess.java index 6d39526a2..85e0d7c6e 100644 --- a/samples/src/main/java/com/example/bigquery/UpdateDatasetAccess.java +++ b/samples/src/main/java/com/example/bigquery/UpdateDatasetAccess.java @@ -39,19 +39,18 @@ public static void updateDatasetAccess(String datasetName) { // once, and can be reused for multiple requests. BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - Dataset dataset = bigquery.getDataset(datasetName); + try { + Dataset dataset = bigquery.getDataset(datasetName); - // Create a new ACL granting the READER role to "sample.bigquery.dev@gmail.com" - // For more information on the types of ACLs available see: - // https://cloud.google.com/storage/docs/access-control/lists - Acl newEntry = Acl.of(new User("sample.bigquery.dev@gmail.com"), Role.READER); + // Create a new ACL granting the READER role to "sample.bigquery.dev@gmail.com" + // For more information on the types of ACLs available see: + // https://cloud.google.com/storage/docs/access-control/lists + Acl newEntry = Acl.of(new User("sample.bigquery.dev@gmail.com"), Role.READER); - // Get a copy of the ACLs list from the dataset and append the new entry - ArrayList acls = new ArrayList<>(dataset.getAcl()); - acls.add(newEntry); + // Get a copy of the ACLs list from the dataset and append the new entry + ArrayList acls = new ArrayList<>(dataset.getAcl()); + acls.add(newEntry); - // Update the dataset to use the new ACLs - try { bigquery.update(dataset.toBuilder().setAcl(acls).build()); System.out.println("Dataset Access Control updated successfully"); } catch (BigQueryException e) { diff --git a/samples/src/main/java/com/example/bigquery/UpdateDatasetDescription.java b/samples/src/main/java/com/example/bigquery/UpdateDatasetDescription.java index 1fe65a7ff..b02e53d66 100644 --- a/samples/src/main/java/com/example/bigquery/UpdateDatasetDescription.java +++ b/samples/src/main/java/com/example/bigquery/UpdateDatasetDescription.java @@ -36,10 +36,8 @@ public static void updateDatasetDescription(String datasetName, String newDescri // once, and can be reused for multiple requests. BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - Dataset dataset = bigquery.getDataset(datasetName); - - // Update dataset description try { + Dataset dataset = bigquery.getDataset(datasetName); bigquery.update(dataset.toBuilder().setDescription(newDescription).build()); System.out.println("Dataset description updated successfully to " + newDescription); } catch (BigQueryException e) { diff --git a/samples/src/main/java/com/example/bigquery/UpdateDatasetExpiration.java b/samples/src/main/java/com/example/bigquery/UpdateDatasetExpiration.java index ef7c97549..820f396b3 100644 --- a/samples/src/main/java/com/example/bigquery/UpdateDatasetExpiration.java +++ b/samples/src/main/java/com/example/bigquery/UpdateDatasetExpiration.java @@ -36,11 +36,10 @@ public static void updateDatasetExpiration(String datasetName) { // once, and can be reused for multiple requests. BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - Dataset dataset = bigquery.getDataset(datasetName); - // Update dataset expiration to one day Long newExpiration = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS); try { + Dataset dataset = bigquery.getDataset(datasetName); bigquery.update(dataset.toBuilder().setDefaultTableLifetime(newExpiration).build()); System.out.println("Dataset description updated successfully to " + newExpiration); } catch (BigQueryException e) { From 3441bbf857013a2cab5fc013005567170a26be9f Mon Sep 17 00:00:00 2001 From: stephwang Date: Mon, 23 Dec 2019 21:54:24 -0500 Subject: [PATCH 14/14] move BQ client construction into try/catch block --- .../com/example/bigquery/CreateDataset.java | 11 +++++---- .../com/example/bigquery/CreateTable.java | 14 +++++------ .../com/example/bigquery/DeleteDataset.java | 23 +++++++++++-------- .../example/bigquery/ExtractTableToJson.java | 8 +++---- .../com/example/bigquery/ListDatasets.java | 9 ++++---- .../example/bigquery/UpdateDatasetAccess.java | 8 +++---- .../bigquery/UpdateDatasetDescription.java | 8 +++---- .../bigquery/UpdateDatasetExpiration.java | 13 ++++++----- 8 files changed, 50 insertions(+), 44 deletions(-) diff --git a/samples/src/main/java/com/example/bigquery/CreateDataset.java b/samples/src/main/java/com/example/bigquery/CreateDataset.java index ca6c944d8..3e2fe0f42 100644 --- a/samples/src/main/java/com/example/bigquery/CreateDataset.java +++ b/samples/src/main/java/com/example/bigquery/CreateDataset.java @@ -32,12 +32,13 @@ public static void runCreateDataset() { } public static void createDataset(String datasetName) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - - DatasetInfo datasetInfo = DatasetInfo.newBuilder(datasetName).build(); try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + + DatasetInfo datasetInfo = DatasetInfo.newBuilder(datasetName).build(); + Dataset newDataset = bigquery.create(datasetInfo); String newDatasetName = newDataset.getDatasetId().getDataset(); System.out.println(newDatasetName + " created successfully"); diff --git a/samples/src/main/java/com/example/bigquery/CreateTable.java b/samples/src/main/java/com/example/bigquery/CreateTable.java index 47f80d0c0..d00000e55 100644 --- a/samples/src/main/java/com/example/bigquery/CreateTable.java +++ b/samples/src/main/java/com/example/bigquery/CreateTable.java @@ -43,15 +43,15 @@ public static void runCreateTable() { } public static void createTable(String datasetName, String tableName, Schema schema) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - TableId tableId = TableId.of(datasetName, tableName); - TableDefinition tableDefinition = StandardTableDefinition.of(schema); - TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build(); + TableId tableId = TableId.of(datasetName, tableName); + TableDefinition tableDefinition = StandardTableDefinition.of(schema); + TableInfo tableInfo = TableInfo.newBuilder(tableId, tableDefinition).build(); - try { bigquery.create(tableInfo); System.out.println("Table created successfully"); } catch (BigQueryException e) { diff --git a/samples/src/main/java/com/example/bigquery/DeleteDataset.java b/samples/src/main/java/com/example/bigquery/DeleteDataset.java index ec8cfd307..7018d0397 100644 --- a/samples/src/main/java/com/example/bigquery/DeleteDataset.java +++ b/samples/src/main/java/com/example/bigquery/DeleteDataset.java @@ -19,6 +19,7 @@ // [START bigquery_delete_dataset] import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; +import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.DatasetId; @@ -32,16 +33,20 @@ public static void runDeleteDataset() { } public static void deleteDataset(String projectId, String datasetName) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - DatasetId datasetId = DatasetId.of(projectId, datasetName); - boolean success = bigquery.delete(datasetId, DatasetDeleteOption.deleteContents()); - if (success) { - System.out.println("Dataset deleted successfully"); - } else { - System.out.println("Dataset was not found"); + DatasetId datasetId = DatasetId.of(projectId, datasetName); + boolean success = bigquery.delete(datasetId, DatasetDeleteOption.deleteContents()); + if (success) { + System.out.println("Dataset deleted successfully"); + } else { + System.out.println("Dataset was not found"); + } + } catch (BigQueryException e) { + System.out.println("Dataset was not deleted. \n" + e.toString()); } } } diff --git a/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java b/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java index 5bc2e7d2f..20e29135e 100644 --- a/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java +++ b/samples/src/main/java/com/example/bigquery/ExtractTableToJson.java @@ -39,11 +39,11 @@ public static void runExtractTableToJson() { // Exports datasetName:tableName to destinationUri as raw CSV public static void extractTableToJson( String projectId, String datasetName, String tableName, String destinationUri) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + TableId tableId = TableId.of(projectId, datasetName, tableName); Table table = bigquery.getTable(tableId); diff --git a/samples/src/main/java/com/example/bigquery/ListDatasets.java b/samples/src/main/java/com/example/bigquery/ListDatasets.java index 394a464e7..2c86c71aa 100644 --- a/samples/src/main/java/com/example/bigquery/ListDatasets.java +++ b/samples/src/main/java/com/example/bigquery/ListDatasets.java @@ -33,12 +33,11 @@ public static void runListDatasets() { } public static void listDatasets(String projectId) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - - // List datasets in a specified project try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + Page datasets = bigquery.listDatasets(projectId, DatasetListOption.pageSize(100)); for (Dataset dataset : datasets.iterateAll()) { System.out.println(dataset.getDatasetId() + " dataset in project listed successfully"); diff --git a/samples/src/main/java/com/example/bigquery/UpdateDatasetAccess.java b/samples/src/main/java/com/example/bigquery/UpdateDatasetAccess.java index 85e0d7c6e..5719dcffa 100644 --- a/samples/src/main/java/com/example/bigquery/UpdateDatasetAccess.java +++ b/samples/src/main/java/com/example/bigquery/UpdateDatasetAccess.java @@ -35,11 +35,11 @@ public static void runUpdateDatasetAccess() { } public static void updateDatasetAccess(String datasetName) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + Dataset dataset = bigquery.getDataset(datasetName); // Create a new ACL granting the READER role to "sample.bigquery.dev@gmail.com" diff --git a/samples/src/main/java/com/example/bigquery/UpdateDatasetDescription.java b/samples/src/main/java/com/example/bigquery/UpdateDatasetDescription.java index b02e53d66..5738ec0e5 100644 --- a/samples/src/main/java/com/example/bigquery/UpdateDatasetDescription.java +++ b/samples/src/main/java/com/example/bigquery/UpdateDatasetDescription.java @@ -32,11 +32,11 @@ public static void runUpdateDatasetDescription() { } public static void updateDatasetDescription(String datasetName, String newDescription) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + Dataset dataset = bigquery.getDataset(datasetName); bigquery.update(dataset.toBuilder().setDescription(newDescription).build()); System.out.println("Dataset description updated successfully to " + newDescription); diff --git a/samples/src/main/java/com/example/bigquery/UpdateDatasetExpiration.java b/samples/src/main/java/com/example/bigquery/UpdateDatasetExpiration.java index 820f396b3..bea27624f 100644 --- a/samples/src/main/java/com/example/bigquery/UpdateDatasetExpiration.java +++ b/samples/src/main/java/com/example/bigquery/UpdateDatasetExpiration.java @@ -32,13 +32,14 @@ public static void runUpdateDatasetExpiration() { } public static void updateDatasetExpiration(String datasetName) { - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); - - // Update dataset expiration to one day - Long newExpiration = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS); try { + // Initialize client that will be used to send requests. This client only needs to be created + // once, and can be reused for multiple requests. + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + + // Update dataset expiration to one day + Long newExpiration = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS); + Dataset dataset = bigquery.getDataset(datasetName); bigquery.update(dataset.toBuilder().setDefaultTableLifetime(newExpiration).build()); System.out.println("Dataset description updated successfully to " + newExpiration);