Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
docs(samples): create and query Amazon s3 data using external table (#…
…835) * docs(samples): add omni query on external table aws * docs(samples): update code * docs(samples): add user id and external id * docs(samples): fix connection name * docs(samples): update create external table AWS sample * update create dataset AWS sample * nit clean up * update query external table sample * fix checkstyle errors * update based on comments Co-authored-by: Praful Makani <praful@qlogic.io>
- Loading branch information
1 parent
36b5f06
commit 53a56be
Showing
5 changed files
with
203 additions
and
51 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
68 changes: 68 additions & 0 deletions
68
samples/snippets/src/main/java/com/example/bigquery/QueryExternalTableAws.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| /* | ||
| * Copyright 2020 Google LLC | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.example.bigquery; | ||
|
|
||
| // [START bigquery_omni_query_external_aws_s3] | ||
| import com.google.cloud.bigquery.BigQuery; | ||
| import com.google.cloud.bigquery.BigQueryException; | ||
| import com.google.cloud.bigquery.BigQueryOptions; | ||
| import com.google.cloud.bigquery.CsvOptions; | ||
| import com.google.cloud.bigquery.DatasetId; | ||
| import com.google.cloud.bigquery.ExternalTableDefinition; | ||
| import com.google.cloud.bigquery.Field; | ||
| import com.google.cloud.bigquery.QueryJobConfiguration; | ||
| import com.google.cloud.bigquery.Schema; | ||
| import com.google.cloud.bigquery.StandardSQLTypeName; | ||
| import com.google.cloud.bigquery.TableId; | ||
| import com.google.cloud.bigquery.TableInfo; | ||
| import com.google.cloud.bigquery.TableResult; | ||
|
|
||
| // Sample to queries an external data source aws s3 using a permanent table | ||
| public class QueryExternalTableAws { | ||
|
|
||
| public static void main(String[] args) throws InterruptedException { | ||
| // TODO(developer): Replace these variables before running the sample. | ||
| String projectId = "MY_PROJECT_ID"; | ||
| String datasetName = "MY_DATASET_NAME"; | ||
| String externalTableName = "MY_EXTERNAL_TABLE_NAME"; | ||
| // Query to find states starting with 'W' | ||
| String query = | ||
| String.format( | ||
| "SELECT * FROM s%.%s.%s WHERE name LIKE 'W%%'", | ||
| projectId, datasetName, externalTableName); | ||
| queryExternalTableAws(query); | ||
| } | ||
|
|
||
| public static void queryExternalTableAws(String query) throws InterruptedException { | ||
| try { | ||
| // Initialize client that will be used to send requests. This client only needs to be created | ||
| // once, and can be reused for multiple requests. | ||
| BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); | ||
|
|
||
| TableResult results = bigquery.query(QueryJobConfiguration.of(query)); | ||
|
|
||
| results | ||
| .iterateAll() | ||
| .forEach(row -> row.forEach(val -> System.out.printf("%s,", val.toString()))); | ||
|
|
||
| System.out.println("Query on aws external permanent table performed successfully."); | ||
| } catch (BigQueryException e) { | ||
| System.out.println("Query not performed \n" + e.toString()); | ||
| } | ||
| } | ||
| } | ||
| // [END bigquery_omni_query_external_aws_s3] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
97 changes: 97 additions & 0 deletions
97
samples/snippets/src/test/java/com/example/bigquery/QueryExternalTableAwsIT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| /* | ||
| * Copyright 2020 Google LLC | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.example.bigquery; | ||
|
|
||
| import static com.google.common.truth.Truth.assertThat; | ||
| import static junit.framework.TestCase.assertNotNull; | ||
|
|
||
| import com.google.cloud.bigquery.CsvOptions; | ||
| import com.google.cloud.bigquery.ExternalTableDefinition; | ||
| import com.google.cloud.bigquery.Field; | ||
| import com.google.cloud.bigquery.Schema; | ||
| import com.google.cloud.bigquery.StandardSQLTypeName; | ||
| import com.google.cloud.bigquery.connection.v1.AwsCrossAccountRole; | ||
| import com.google.cloud.bigquery.connection.v1.AwsProperties; | ||
| import com.google.cloud.bigquery.connection.v1.Connection; | ||
| import com.google.cloud.bigquery.connection.v1.CreateConnectionRequest; | ||
| import com.google.cloud.bigquery.connection.v1.DeleteConnectionRequest; | ||
| import com.google.cloud.bigquery.connection.v1.LocationName; | ||
| import com.google.cloud.bigqueryconnection.v1.ConnectionServiceClient; | ||
| import java.io.ByteArrayOutputStream; | ||
| import java.io.IOException; | ||
| import java.io.PrintStream; | ||
| import java.util.UUID; | ||
| import java.util.logging.Level; | ||
| import java.util.logging.Logger; | ||
| import org.junit.After; | ||
| import org.junit.Before; | ||
| import org.junit.BeforeClass; | ||
| import org.junit.Test; | ||
|
|
||
| public class QueryExternalTableAwsIT { | ||
|
|
||
| private final Logger log = Logger.getLogger(this.getClass().getName()); | ||
| private ByteArrayOutputStream bout; | ||
| private PrintStream out; | ||
| private PrintStream originalPrintStream; | ||
|
|
||
| private static final String OMNI_PROJECT_ID = requireEnvVar("OMNI_PROJECT_ID"); | ||
| private static final String OMNI_DATASET_NAME = requireEnvVar("OMNI_DATASET_NAME"); | ||
| private static final String OMNI_EXTERNAL_TABLE_NAME = requireEnvVar("OMNI_EXTERNAL_TABLE_NAME"); | ||
|
|
||
| private static String requireEnvVar(String varName) { | ||
| String value = System.getenv(varName); | ||
| assertNotNull( | ||
| "Environment variable " + varName + " is required to perform these tests.", | ||
| System.getenv(varName)); | ||
| return value; | ||
| } | ||
|
|
||
| @BeforeClass | ||
| public static void checkRequirements() { | ||
| requireEnvVar("OMNI_PROJECT_ID"); | ||
| requireEnvVar("OMNI_DATASET_NAME"); | ||
| requireEnvVar("OMNI_EXTERNAL_TABLE_NAME"); | ||
| } | ||
|
|
||
| @Before | ||
| public void setUp() { | ||
| bout = new ByteArrayOutputStream(); | ||
| out = new PrintStream(bout); | ||
| originalPrintStream = System.out; | ||
| System.setOut(out); | ||
| } | ||
|
|
||
| @After | ||
| public void tearDown() { | ||
| // restores print statements in the original method | ||
| System.out.flush(); | ||
| System.setOut(originalPrintStream); | ||
| log.log(Level.INFO, bout.toString()); | ||
| } | ||
|
|
||
| @Test | ||
| public void testQueryExternalTableAws() throws InterruptedException { | ||
| String query = | ||
| String.format( | ||
| "SELECT * FROM %s.%s.%s WHERE name LIKE 'W%%'", | ||
| OMNI_PROJECT_ID, OMNI_DATASET_NAME, OMNI_EXTERNAL_TABLE_NAME); | ||
| QueryExternalTableAws.queryExternalTableAws(query); | ||
| assertThat(bout.toString()) | ||
| .contains("Query on aws external permanent table performed successfully."); | ||
| } | ||
| } |