From 2e14e92328b7989ed9254964963db0ce6f3c396f Mon Sep 17 00:00:00 2001 From: Praful Makani Date: Mon, 24 Aug 2020 20:51:05 +0530 Subject: [PATCH 1/4] add jmh benchmark --- benchmark/README.md | 18 ++++ benchmark/pom.xml | 77 +++++++++++++++ .../QueryBenchmark.java | 96 +++++++++++++++++++ pom.xml | 6 ++ 4 files changed, 197 insertions(+) create mode 100644 benchmark/README.md create mode 100644 benchmark/pom.xml create mode 100644 benchmark/src/main/java/com.google.cloud.bigquery/QueryBenchmark.java diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 000000000..5cd2cc078 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,18 @@ +Benchmarking +============ + + + +To build all of the benchmarks: +``` +# Run from project root directory +mvn clean install +``` + + +To run a benchmark jar, run the following command +``` +# Run from project root directory + java -jar target/benchmark.jar +``` + diff --git a/benchmark/pom.xml b/benchmark/pom.xml new file mode 100644 index 000000000..21c378a23 --- /dev/null +++ b/benchmark/pom.xml @@ -0,0 +1,77 @@ + + + 4.0.0 + benchmark + + google-cloud-bigquery-parent + com.google.cloud + 1.116.11-SNAPSHOT + + + + UTF-8 + 1.23 + benchmarks + + + + + com.google.cloud + google-cloud-bigquery + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + package + + shade + + + ${uberjar.name} + + + org.openjdk.jmh.Main + + + + + + *:* + + module-info.class + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + diff --git a/benchmark/src/main/java/com.google.cloud.bigquery/QueryBenchmark.java b/benchmark/src/main/java/com.google.cloud.bigquery/QueryBenchmark.java new file mode 100644 index 000000000..499672045 --- /dev/null +++ b/benchmark/src/main/java/com.google.cloud.bigquery/QueryBenchmark.java @@ -0,0 +1,96 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery; + +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +@Fork(value = 1) +@BenchmarkMode(Mode.Throughput) +@Warmup(iterations = 10) +@Measurement(iterations = 20) +@State(Scope.Benchmark) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +public class QueryBenchmark { + + private static final String NYCYELLOWLIMIT1K = "SELECT * FROM `nyc-tlc.yellow.trips` LIMIT 10000"; + private static final String NYCYELLOWLIMIT10K = + "SELECT * FROM `nyc-tlc.yellow.trips` LIMIT 100000"; + private static final String NYCYELLOWLIMIT100K = + "SELECT * FROM `nyc-tlc.yellow.trips` LIMIT 1000000"; + private static final String WIKISAMPLESORDEREDLIMIT1K = + "SELECT title FROM `bigquery-public-data.samples.wikipedia` ORDER BY title LIMIT 1000"; + private static final String CURRENTTIMESTAMP = "SELECT CURRENT_TIMESTAMP() as ts"; + private static final String SESSIONUSER = "SELECT SESSION_USER() as ts"; + private static final String LITERALS = "SELECT 1 as i, 3.14 as pi"; + private static final String INVALIDQUERY = + "CREATE OR REPLACE SELECT * FROM UPDATE TABLE SET `nyc-tlc.yellow.trips`"; + + private BigQuery bigquery; + + @Setup + public void setUp() { + this.bigquery = BigQueryOptions.getDefaultInstance().getService(); + } + + @State(Scope.Benchmark) + public static class QueryParams { + + @Param({ + NYCYELLOWLIMIT1K, + NYCYELLOWLIMIT10K, + NYCYELLOWLIMIT100K, + WIKISAMPLESORDEREDLIMIT1K, + CURRENTTIMESTAMP, + SESSIONUSER, + LITERALS, + INVALIDQUERY + }) + public String queries; + } + + private void queryPerform(String queries, Blackhole blackhole) throws Exception { + TableResult result = + bigquery.query(QueryJobConfiguration.newBuilder(queries).setUseLegacySql(false).build()); + for (List row : result.iterateAll()) { + blackhole.consume(row.size()); + } + while (result.hasNextPage()) { + result = result.getNextPage(); + for (List row : result.iterateAll()) { + blackhole.consume(row.size()); + } + } + } + + @Benchmark + public void query(QueryParams queryParams, Blackhole blackhole) throws Exception { + queryPerform(queryParams.queries, blackhole); + } +} diff --git a/pom.xml b/pom.xml index f1a136c17..f831cbd5a 100644 --- a/pom.xml +++ b/pom.xml @@ -215,5 +215,11 @@ samples + + benchmark + + benchmark + + From 6c9d05e77e4722c09166e8358963031b8d8e3c93 Mon Sep 17 00:00:00 2001 From: Praful Makani Date: Tue, 25 Aug 2020 11:58:56 +0530 Subject: [PATCH 2/4] fix typo for jar name --- benchmark/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/pom.xml b/benchmark/pom.xml index 21c378a23..da9cd1863 100644 --- a/benchmark/pom.xml +++ b/benchmark/pom.xml @@ -12,7 +12,7 @@ UTF-8 1.23 - benchmarks + benchmark From 093940a98ee25ca7d11d250f81f9886b8041b4e2 Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 25 Aug 2020 11:08:56 -0400 Subject: [PATCH 3/4] update README.md with additional instructions to run Benchmarks --- benchmark/README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 5cd2cc078..c29bf0781 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -1,18 +1,20 @@ Benchmarking ============ - - To build all of the benchmarks: ``` -# Run from project root directory +# Run from benchmark directory +cd benchmark mvn clean install ``` +Set `GOOGLE_APPLICATION_CREDENTIALS`: +``` +export GOOGLE_APPLICATION_CREDENTIALS=path/to/service_account.json +``` To run a benchmark jar, run the following command ``` # Run from project root directory java -jar target/benchmark.jar ``` - From fc4e84d427514e2812a8b0d0e676bc2ccaecde2b Mon Sep 17 00:00:00 2001 From: stephwang Date: Tue, 25 Aug 2020 12:28:50 -0400 Subject: [PATCH 4/4] update benchmarkMode to AverageTime and reduce iterations --- .../main/java/com.google.cloud.bigquery/QueryBenchmark.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmark/src/main/java/com.google.cloud.bigquery/QueryBenchmark.java b/benchmark/src/main/java/com.google.cloud.bigquery/QueryBenchmark.java index 499672045..79aa484d8 100644 --- a/benchmark/src/main/java/com.google.cloud.bigquery/QueryBenchmark.java +++ b/benchmark/src/main/java/com.google.cloud.bigquery/QueryBenchmark.java @@ -32,9 +32,9 @@ import org.openjdk.jmh.infra.Blackhole; @Fork(value = 1) -@BenchmarkMode(Mode.Throughput) -@Warmup(iterations = 10) -@Measurement(iterations = 20) +@BenchmarkMode(Mode.AverageTime) +@Warmup(iterations = 5) +@Measurement(iterations = 5) @State(Scope.Benchmark) @OutputTimeUnit(TimeUnit.MILLISECONDS) public class QueryBenchmark {