Skip to content

Commit

Permalink
docs(samples): add query external bigtable using temp table (#763)
Browse files Browse the repository at this point in the history 
* docs(samples): add query external bigtable using temp table

* docs(samples): address feedback

* nit

* docs(samples): address comments

* update projectId

Co-authored-by: Stephanie Wang <stephaniewang526@users.noreply.github.com>
  • Loading branch information
@stephaniewang526
Praful Makani and stephaniewang526 committed Oct 6, 2020
1 parent 85ee12f commit fed28c7
Show file tree
Hide file tree
Showing 5 changed files with 322 additions and 0 deletions.
6 changes: 6 additions & 0 deletions samples/install-without-bom/pom.xml
View file
Expand Up @@ -60,6 +60,12 @@
<version>1.31.0</version>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bigtable</artifactId>
<version>1.15.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
Expand Down
6 changes: 6 additions & 0 deletions samples/snapshot/pom.xml
View file
Expand Up @@ -58,6 +58,12 @@
<version>1.31.0</version>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bigtable</artifactId>
<version>1.15.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
Expand Down
6 changes: 6 additions & 0 deletions samples/snippets/pom.xml
View file
Expand Up @@ -73,6 +73,12 @@
<!-- [END bigquery-enduser-installed-packages] -->

<!-- Test dependencies -->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-bigtable</artifactId>
<version>1.15.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
Expand Down
118 changes: 118 additions & 0 deletions samples/snippets/src/main/java/com/example/bigquery/QueryExternalBigtableTemp.java
View file
@@ -0,0 +1,118 @@
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.example.bigquery;

// [START bigquery_query_external_bigtable_temp]
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryException;
import com.google.cloud.bigquery.BigQueryOptions;
import com.google.cloud.bigquery.BigtableColumn;
import com.google.cloud.bigquery.BigtableColumnFamily;
import com.google.cloud.bigquery.BigtableOptions;
import com.google.cloud.bigquery.ExternalTableDefinition;
import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.TableResult;
import com.google.common.collect.ImmutableList;
import org.apache.commons.codec.binary.Base64;

// Sample to queries an external bigtable data source using a temporary table
public class QueryExternalBigtableTemp {

public static void main(String[] args) {
// TODO(developer): Replace these variables before running the sample.
String projectId = "MY_PROJECT_ID";
String bigtableInstanceId = "MY_INSTANCE_ID";
String bigtableTableName = "MY_BIGTABLE_NAME";
String bigqueryTableName = "MY_TABLE_NAME";
String sourceUri =
String.format(
"https://googleapis.com/bigtable/projects/%s/instances/%s/tables/%s",
projectId, bigtableInstanceId, bigtableTableName);
String query = String.format("SELECT * FROM %s ", bigqueryTableName);
queryExternalBigtableTemp(bigqueryTableName, sourceUri, query);
}

public static void queryExternalBigtableTemp(String tableName, String sourceUri, String query) {
try {
// Initialize client that will be used to send requests. This client only needs to be created
// once, and can be reused for multiple requests.
BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();

BigtableColumnFamily.Builder statsSummary = BigtableColumnFamily.newBuilder();

// Configuring Columns
BigtableColumn connectedCell =
BigtableColumn.newBuilder()
.setQualifierEncoded(Base64.encodeBase64String("connected_cell".getBytes()))
.setFieldName("connected_cell")
.setType("STRING")
.setEncoding("TEXT")
.build();
BigtableColumn connectedWifi =
BigtableColumn.newBuilder()
.setQualifierEncoded(Base64.encodeBase64String("connected_wifi".getBytes()))
.setFieldName("connected_wifi")
.setType("STRING")
.setEncoding("TEXT")
.build();
BigtableColumn osBuild =
BigtableColumn.newBuilder()
.setQualifierEncoded(Base64.encodeBase64String("os_build".getBytes()))
.setFieldName("os_build")
.setType("STRING")
.setEncoding("TEXT")
.build();

// Configuring column family and columns
statsSummary
.setColumns(ImmutableList.of(connectedCell, connectedWifi, osBuild))
.setFamilyID("stats_summary")
.setOnlyReadLatest(true)
.setEncoding("TEXT")
.setType("STRING")
.build();

// Configuring BigtableOptions is optional.
BigtableOptions options =
BigtableOptions.newBuilder()
.setIgnoreUnspecifiedColumnFamilies(true)
.setReadRowkeyAsString(true)
.setColumnFamilies(ImmutableList.of(statsSummary.build()))
.build();

// Configure the external data source and query job.
ExternalTableDefinition externalTable =
ExternalTableDefinition.newBuilder(sourceUri, options).build();
QueryJobConfiguration queryConfig =
QueryJobConfiguration.newBuilder(query)
.addTableDefinition(tableName, externalTable)
.build();

// Example query
TableResult results = bigquery.query(queryConfig);

results
.iterateAll()
.forEach(row -> row.forEach(val -> System.out.printf("%s,", val.toString())));

System.out.println("Query on external temporary table performed successfully.");
} catch (BigQueryException | InterruptedException e) {
System.out.println("Query not performed \n" + e.toString());
}
}
}
// [END bigquery_query_external_bigtable_temp]
186 changes: 186 additions & 0 deletions samples/snippets/src/test/java/com/example/bigquery/QueryExternalBigtableTempIT.java
View file
@@ -0,0 +1,186 @@
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.example.bigquery;

import static com.google.common.truth.Truth.assertThat;
import static junit.framework.TestCase.assertNotNull;

import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest;
import com.google.cloud.bigtable.data.v2.BigtableDataClient;
import com.google.cloud.bigtable.data.v2.models.BulkMutation;
import com.google.cloud.bigtable.data.v2.models.Mutation;
import com.google.protobuf.ByteString;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.util.UUID;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;

public class QueryExternalBigtableTempIT {

private final Logger log = Logger.getLogger(this.getClass().getName());
private static final String ID = UUID.randomUUID().toString().substring(0, 8);
private static final String TABLE_ID = "bigquery-samples-test" + ID;
private static final String COLUMN_FAMILY_NAME = "stats_summary";
private static final long TIMESTAMP = System.currentTimeMillis() * 1000;
private static final String CONNECTED_CELL = "connected_cell";
private static final String CONNECTED_WIFI = "connected_wifi";
private static final String OS_BUILD = "os_build";
private ByteArrayOutputStream bout;
private PrintStream out;
private PrintStream originalPrintStream;

private static final String INSTANCE = requireEnvVar("BIGTABLE_TESTING_INSTANCE");
private static final String PROJECT = requireEnvVar("SAMPLES_TESTING_PROJECT");

private static String requireEnvVar(String varName) {
String value = System.getenv(varName);
assertNotNull(
"Environment variable " + varName + " is required to perform these tests.",
System.getenv(varName));
return value;
}

@BeforeClass
public static void checkRequirements() {
requireEnvVar("GOOGLE_CLOUD_PROJECT");
requireEnvVar("BIGTABLE_TESTING_INSTANCE");
}

@Before
public void setUp() throws IOException {
bout = new ByteArrayOutputStream();
out = new PrintStream(bout);
originalPrintStream = System.out;
System.setOut(out);

// create a temporary bigtable table.
try (BigtableTableAdminClient client = BigtableTableAdminClient.create(PROJECT, INSTANCE)) {
CreateTableRequest createTableRequest =
CreateTableRequest.of(TABLE_ID).addFamily(COLUMN_FAMILY_NAME);
client.createTable(createTableRequest);
}
// inserting temporary rows.
try (BigtableDataClient client = BigtableDataClient.create(PROJECT, INSTANCE)) {
BulkMutation bulkMutation =
BulkMutation.create(TABLE_ID)
.add(
"phone#4c410523#20190501",
Mutation.create()
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
TIMESTAMP,
1)
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
TIMESTAMP,
1)
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190405.003"))
.add(
"phone#4c410523#20190502",
Mutation.create()
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
TIMESTAMP,
1)
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
TIMESTAMP,
1)
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190405.004"))
.add(
"phone#4c410523#20190505",
Mutation.create()
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
TIMESTAMP,
0)
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
TIMESTAMP,
1)
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190406.000"))
.add(
"phone#5c10102#20190501",
Mutation.create()
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
TIMESTAMP,
1)
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
TIMESTAMP,
1)
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190401.002"))
.add(
"phone#5c10102#20190502",
Mutation.create()
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
TIMESTAMP,
1)
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
TIMESTAMP,
0)
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190406.000"));

client.bulkMutateRows(bulkMutation);
}
}

@After
public void tearDown() throws IOException {
// Clean up
try (BigtableTableAdminClient client = BigtableTableAdminClient.create(PROJECT, INSTANCE)) {
client.deleteTable(TABLE_ID);
}
// restores print statements in the original method
System.out.flush();
System.setOut(originalPrintStream);
log.log(Level.INFO, bout.toString());
}

@Test
public void testQueryExternalBigtableTemp() {
String tableName = "EXTERNAL_TEMP_TABLE_FROM_BIGTABLE_TEST_" + ID;
String query = String.format("SELECT * FROM %s ", tableName);
String sourceUri =
String.format(
"https://googleapis.com/bigtable/projects/%s/instances/%s/tables/%s",
PROJECT, INSTANCE, TABLE_ID);
QueryExternalBigtableTemp.queryExternalBigtableTemp(tableName, sourceUri, query);
assertThat(bout.toString())
.contains("Query on external temporary table performed successfully.");
}
}

0 comments on commit fed28c7

Please sign in to comment.