Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
docs(samples): add query external bigtable using permanent table (#764)
* docs(samples): add query external bigtable using permanent table

* docs(samples): address feedback

* docs(samples): address comments
  • Loading branch information
Praful Makani committed Oct 7, 2020
1 parent a5573d1 commit 257edfc
Show file tree
Hide file tree
Showing 2 changed files with 312 additions and 0 deletions.
@@ -0,0 +1,120 @@
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.example.bigquery;

// [START bigquery_query_external_bigtable_perm]
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryException;
import com.google.cloud.bigquery.BigQueryOptions;
import com.google.cloud.bigquery.BigtableColumn;
import com.google.cloud.bigquery.BigtableColumnFamily;
import com.google.cloud.bigquery.BigtableOptions;
import com.google.cloud.bigquery.ExternalTableDefinition;
import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.TableId;
import com.google.cloud.bigquery.TableInfo;
import com.google.cloud.bigquery.TableResult;
import com.google.common.collect.ImmutableList;
import org.apache.commons.codec.binary.Base64;

// Sample to queries an external bigtable data source using a permanent table
public class QueryExternalBigtablePerm {

public static void main(String[] args) {
// TODO(developer): Replace these variables before running the sample.
String projectId = "MY_PROJECT_ID";
String bigtableInstanceId = "MY_INSTANCE_ID";
String bigtableTableName = "MY_BIGTABLE_NAME";
String bigqueryDatasetName = "MY_DATASET_NAME";
String bigqueryTableName = "MY_TABLE_NAME";
String sourceUri =
String.format(
"https://googleapis.com/bigtable/projects/%s/instances/%s/tables/%s",
projectId, bigtableInstanceId, bigtableTableName);
String query = String.format("SELECT * FROM %s ", bigqueryTableName);
queryExternalBigtablePerm(bigqueryDatasetName, bigqueryTableName, sourceUri, query);
}

public static void queryExternalBigtablePerm(
String datasetName, String tableName, String sourceUri, String query) {
try {
// Initialize client that will be used to send requests. This client only needs to be created
// once, and can be reused for multiple requests.
BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService();

BigtableColumnFamily.Builder statsSummary = BigtableColumnFamily.newBuilder();

// Configuring Columns
BigtableColumn connectedCell =
BigtableColumn.newBuilder()
.setQualifierEncoded(Base64.encodeBase64String("connected_cell".getBytes()))
.setFieldName("connected_cell")
.setType("STRING")
.setEncoding("TEXT")
.build();
BigtableColumn connectedWifi =
BigtableColumn.newBuilder()
.setQualifierEncoded(Base64.encodeBase64String("connected_wifi".getBytes()))
.setFieldName("connected_wifi")
.setType("STRING")
.setEncoding("TEXT")
.build();
BigtableColumn osBuild =
BigtableColumn.newBuilder()
.setQualifierEncoded(Base64.encodeBase64String("os_build".getBytes()))
.setFieldName("os_build")
.setType("STRING")
.setEncoding("TEXT")
.build();

// Configuring column family and columns
statsSummary
.setColumns(ImmutableList.of(connectedCell, connectedWifi, osBuild))
.setFamilyID("stats_summary")
.setOnlyReadLatest(true)
.setEncoding("TEXT")
.setType("STRING")
.build();

// Configuring BigtableOptions is optional.
BigtableOptions options =
BigtableOptions.newBuilder()
.setIgnoreUnspecifiedColumnFamilies(true)
.setReadRowkeyAsString(true)
.setColumnFamilies(ImmutableList.of(statsSummary.build()))
.build();

TableId tableId = TableId.of(datasetName, tableName);
// Create a permanent table linked to the Bigtable table
ExternalTableDefinition externalTable =
ExternalTableDefinition.newBuilder(sourceUri, options).build();
bigquery.create(TableInfo.of(tableId, externalTable));

// Example query
TableResult results = bigquery.query(QueryJobConfiguration.of(query));

results
.iterateAll()
.forEach(row -> row.forEach(val -> System.out.printf("%s,", val.toString())));

System.out.println("Query on external permanent table performed successfully.");
} catch (BigQueryException | InterruptedException e) {
System.out.println("Query not performed \n" + e.toString());
}
}
}
// [END bigquery_query_external_bigtable_perm]
@@ -0,0 +1,192 @@
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.example.bigquery;

import static com.google.common.truth.Truth.assertThat;
import static junit.framework.TestCase.assertNotNull;

import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest;
import com.google.cloud.bigtable.data.v2.BigtableDataClient;
import com.google.cloud.bigtable.data.v2.models.BulkMutation;
import com.google.cloud.bigtable.data.v2.models.Mutation;
import com.google.protobuf.ByteString;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.util.UUID;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;

public class QueryExternalBigtablePermIT {

private final Logger log = Logger.getLogger(this.getClass().getName());
private static final String ID = UUID.randomUUID().toString().substring(0, 8);
private static final String TABLE_ID = "bigquery-samples-test" + ID;
private static final String COLUMN_FAMILY_NAME = "stats_summary";
private static final long TIMESTAMP = System.currentTimeMillis() * 1000;
private static final String CONNECTED_CELL = "connected_cell";
private static final String CONNECTED_WIFI = "connected_wifi";
private static final String OS_BUILD = "os_build";
private String tableName;
private ByteArrayOutputStream bout;
private PrintStream out;
private PrintStream originalPrintStream;

private static final String INSTANCE = requireEnvVar("BIGTABLE_TESTING_INSTANCE");
private static final String PROJECT = requireEnvVar("SAMPLES_TESTING_PROJECT");
private static final String BIGQUERY_DATASET_NAME = requireEnvVar("BIGQUERY_DATASET_NAME");

private static String requireEnvVar(String varName) {
String value = System.getenv(varName);
assertNotNull(
"Environment variable " + varName + " is required to perform these tests.",
System.getenv(varName));
return value;
}

@BeforeClass
public static void checkRequirements() {
requireEnvVar("GOOGLE_CLOUD_PROJECT");
requireEnvVar("BIGTABLE_TESTING_INSTANCE");
requireEnvVar("BIGQUERY_DATASET_NAME");
}

@Before
public void setUp() throws IOException {
// Create a test table
tableName = "EXTERNAL_TABLE_FROM_BIGTABLE_TEST_" + ID;
bout = new ByteArrayOutputStream();
out = new PrintStream(bout);
originalPrintStream = System.out;
System.setOut(out);

// create a temporary bigtable table.
try (BigtableTableAdminClient client = BigtableTableAdminClient.create(PROJECT, INSTANCE)) {
CreateTableRequest createTableRequest =
CreateTableRequest.of(TABLE_ID).addFamily(COLUMN_FAMILY_NAME);
client.createTable(createTableRequest);
}
// inserting temporary rows.
try (BigtableDataClient client = BigtableDataClient.create(PROJECT, INSTANCE)) {
BulkMutation bulkMutation =
BulkMutation.create(TABLE_ID)
.add(
"phone#4c410523#20190501",
Mutation.create()
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
TIMESTAMP,
1)
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
TIMESTAMP,
1)
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190405.003"))
.add(
"phone#4c410523#20190502",
Mutation.create()
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
TIMESTAMP,
1)
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
TIMESTAMP,
1)
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190405.004"))
.add(
"phone#4c410523#20190505",
Mutation.create()
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
TIMESTAMP,
0)
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
TIMESTAMP,
1)
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190406.000"))
.add(
"phone#5c10102#20190501",
Mutation.create()
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
TIMESTAMP,
1)
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
TIMESTAMP,
1)
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190401.002"))
.add(
"phone#5c10102#20190502",
Mutation.create()
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_CELL.getBytes()),
TIMESTAMP,
1)
.setCell(
COLUMN_FAMILY_NAME,
ByteString.copyFrom(CONNECTED_WIFI.getBytes()),
TIMESTAMP,
0)
.setCell(COLUMN_FAMILY_NAME, OS_BUILD, TIMESTAMP, "PQ2A.190406.000"));

client.bulkMutateRows(bulkMutation);
}
}

@After
public void tearDown() throws IOException {
// Clean up
try (BigtableTableAdminClient client = BigtableTableAdminClient.create(PROJECT, INSTANCE)) {
client.deleteTable(TABLE_ID);
}
DeleteTable.deleteTable(BIGQUERY_DATASET_NAME, tableName);
// restores print statements in the original method
System.out.flush();
System.setOut(originalPrintStream);
log.log(Level.INFO, bout.toString());
}

@Test
public void testQueryExternalBigtablePerm() {
String query = String.format("SELECT * FROM %s.%s ", BIGQUERY_DATASET_NAME, tableName);
String sourceUri =
String.format(
"https://googleapis.com/bigtable/projects/%s/instances/%s/tables/%s",
PROJECT, INSTANCE, TABLE_ID);
QueryExternalBigtablePerm.queryExternalBigtablePerm(
BIGQUERY_DATASET_NAME, tableName, sourceUri, query);
assertThat(bout.toString())
.contains("Query on external permanent table performed successfully.");
}
}

0 comments on commit 257edfc

Please sign in to comment.