From baed5b9aecb0976dc27b6264a5d40a947859cb79 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 25 Jan 2021 18:38:57 +0100 Subject: [PATCH 1/6] test(deps): update dependency com.google.truth:truth to v1.1.2 (#18) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1308d636..bcb015ed 100644 --- a/pom.xml +++ b/pom.xml @@ -130,7 +130,7 @@ com.google.truth truth - 1.1 + 1.1.2 test From d731c4161987c15ac476b5d46a8324a8d4a77fdb Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 25 Jan 2021 10:40:34 -0800 Subject: [PATCH 2/6] chore: delete presubmit/java7.cfg (#20) --- .kokoro/presubmit/java7.cfg | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 .kokoro/presubmit/java7.cfg diff --git a/.kokoro/presubmit/java7.cfg b/.kokoro/presubmit/java7.cfg deleted file mode 100644 index cb24f44e..00000000 --- a/.kokoro/presubmit/java7.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/java7" -} From 90f89e0ec169be938fa3a6b3ed910438507dd820 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Mon, 25 Jan 2021 19:41:11 +0100 Subject: [PATCH 3/6] test(deps): update dependency org.mockito:mockito-core to v3.7.7 (#19) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bcb015ed..d34eefe6 100644 --- a/pom.xml +++ b/pom.xml @@ -137,7 +137,7 @@ org.mockito mockito-core test - 3.7.0 + 3.7.7 com.google.guava From 6b2d3f80ff3485f747cc28be07383c4f25cbf958 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Mon, 25 Jan 2021 11:59:08 -0800 Subject: [PATCH 4/6] chore: create versons.txt (#21) * chore: create release-please.yml * add versions.txt * Update versions.txt Co-authored-by: Jeff Ching --- versions.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 versions.txt diff --git a/versions.txt b/versions.txt new file mode 100644 index 00000000..cba17482 --- /dev/null +++ b/versions.txt @@ -0,0 +1,4 @@ +# Format: +# module:released-version:current-version + +pubsublite-spark:0.0.0:0.0.1-SNAPSHOT From 64cad24dab014ae9bd64abc833c8f744b039e95a Mon Sep 17 00:00:00 2001 From: jiangmichaellll <40044148+jiangmichaellll@users.noreply.github.com> Date: Tue, 26 Jan 2021 17:42:35 -0500 Subject: [PATCH 5/6] feat: use gson instead of jackson (#25) --- pom.xml | 10 +---- .../pubsublite/spark/SparkSourceOffset.java | 39 ++++++------------- 2 files changed, 14 insertions(+), 35 deletions(-) diff --git a/pom.xml b/pom.xml index d34eefe6..b753bd77 100644 --- a/pom.xml +++ b/pom.xml @@ -99,20 +99,14 @@ api-common - com.fasterxml.jackson.core - jackson-core - 2.12.1 + com.google.code.gson + gson com.github.ben-manes.caffeine caffeine 2.8.8 - - com.fasterxml.jackson.core - jackson-databind - 2.12.1 - org.scala-lang scala-library diff --git a/src/main/java/com/google/cloud/pubsublite/spark/SparkSourceOffset.java b/src/main/java/com/google/cloud/pubsublite/spark/SparkSourceOffset.java index 4dcfc87f..98a37b2c 100644 --- a/src/main/java/com/google/cloud/pubsublite/spark/SparkSourceOffset.java +++ b/src/main/java/com/google/cloud/pubsublite/spark/SparkSourceOffset.java @@ -18,23 +18,21 @@ import static com.google.common.base.Preconditions.checkArgument; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import com.google.cloud.pubsublite.Partition; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import java.io.IOException; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Objects; +import java.util.TreeMap; import java.util.stream.Collectors; public final class SparkSourceOffset extends org.apache.spark.sql.sources.v2.reader.streaming.Offset { - private static final ObjectMapper objectMapper = - new ObjectMapper().configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); + private static final Gson gson = new Gson(); // Using a map to ensure unique partitions. private final ImmutableMap partitionOffsetMap; @@ -79,26 +77,17 @@ public static SparkSourceOffset merge(SparkPartitionOffset[] offsets) { return new SparkSourceOffset(map); } - @SuppressWarnings("unchecked") public static SparkSourceOffset fromJson(String json) { - Map map; - try { - // TODO: Use TypeReference instead of Map.class, currently TypeReference breaks spark with - // java.lang.LinkageError: loader constraint violation: loader previously initiated loading - // for a different type. - map = objectMapper.readValue(json, Map.class); - } catch (IOException e) { - throw new IllegalStateException("Unable to deserialize PslSourceOffset.", e); - } + Map map = gson.fromJson(json, new TypeToken>() {}.getType()); Map partitionOffsetMap = map.entrySet().stream() .collect( Collectors.toMap( - e -> Partition.of(Long.parseLong(e.getKey())), + e -> Partition.of(e.getKey()), e -> SparkPartitionOffset.builder() - .partition(Partition.of(Long.parseLong(e.getKey()))) - .offset(e.getValue().longValue()) + .partition(Partition.of(e.getKey())) + .offset(e.getValue()) .build())); return new SparkSourceOffset(partitionOffsetMap); } @@ -109,13 +98,9 @@ public Map getPartitionOffsetMap() { @Override public String json() { - try { - Map map = - partitionOffsetMap.entrySet().stream() - .collect(Collectors.toMap(e -> e.getKey().value(), e -> e.getValue().offset())); - return objectMapper.writeValueAsString(map); - } catch (JsonProcessingException e) { - throw new IllegalStateException("Unable to serialize PslSourceOffset.", e); - } + Map map = + partitionOffsetMap.entrySet().stream() + .collect(Collectors.toMap(e -> e.getKey().value(), e -> e.getValue().offset())); + return gson.toJson(new TreeMap<>(map)); } } From 6aa40afe53a54c6ef797a229ef6a12c64df2dc9f Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Tue, 26 Jan 2021 16:06:13 -0800 Subject: [PATCH 6/6] chore: scaffold samples dir (#24) * changes without context autosynth cannot find the source of changes triggered by earlier changes in this repository, or by version upgrades to tools such as linters. * scaffold samples dir * download docfx doclet resource * Update samples/snapshot/pom.xml Co-authored-by: Jeff Ching * update version number * use artifact id pubsublite-spark-sql-streaming Co-authored-by: Tianzi Cai Co-authored-by: Jeff Ching --- .kokoro/nightly/java7.cfg | 7 +++ .kokoro/presubmit/java7.cfg | 7 +++ .kokoro/release/publish_javadoc.cfg | 3 ++ pom.xml | 2 +- samples/pom.xml | 55 +++++++++++++++++++ samples/snapshot/pom.xml | 83 +++++++++++++++++++++++++++++ samples/snippets/pom.xml | 51 ++++++++++++++++++ synth.metadata | 2 +- synth.py | 5 +- versions.txt | 2 +- 10 files changed, 213 insertions(+), 4 deletions(-) create mode 100644 .kokoro/nightly/java7.cfg create mode 100644 .kokoro/presubmit/java7.cfg create mode 100644 samples/pom.xml create mode 100644 samples/snapshot/pom.xml create mode 100644 samples/snippets/pom.xml diff --git a/.kokoro/nightly/java7.cfg b/.kokoro/nightly/java7.cfg new file mode 100644 index 00000000..cb24f44e --- /dev/null +++ b/.kokoro/nightly/java7.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/java7" +} diff --git a/.kokoro/presubmit/java7.cfg b/.kokoro/presubmit/java7.cfg new file mode 100644 index 00000000..cb24f44e --- /dev/null +++ b/.kokoro/presubmit/java7.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/java7" +} diff --git a/.kokoro/release/publish_javadoc.cfg b/.kokoro/release/publish_javadoc.cfg index dba40856..cfa99b96 100644 --- a/.kokoro/release/publish_javadoc.cfg +++ b/.kokoro/release/publish_javadoc.cfg @@ -27,3 +27,6 @@ before_action { } } } + +# Downloads docfx doclet resource. This will be in ${KOKORO_GFILE_DIR}/ +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/docfx" \ No newline at end of file diff --git a/pom.xml b/pom.xml index b753bd77..c76571fc 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ 4.0.0 com.google.cloud pubsublite-spark-sql-streaming - 0.1.0-SNAPSHOT + 0.0.1-SNAPSHOT jar Pub/Sub Lite Spark SQL Streaming https://github.com/googleapis/java-pubsublite-spark diff --git a/samples/pom.xml b/samples/pom.xml new file mode 100644 index 00000000..47761966 --- /dev/null +++ b/samples/pom.xml @@ -0,0 +1,55 @@ + + + 4.0.0 + com.google.cloud + google-cloud-pubsublite-spark-samples + 0.0.1-SNAPSHOT + pom + Google Pub/Sub Lite Spark Connector Samples Parent + https://github.com/googleapis/java-pubsublite-spark + + Java idiomatic client for Google Cloud Platform services. + + + + + com.google.cloud.samples + shared-configuration + 1.0.18 + + + + 1.8 + 1.8 + UTF-8 + + + + snapshot + snippets + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.8.2 + + true + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.8 + + true + + + + + diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml new file mode 100644 index 00000000..d1b20fae --- /dev/null +++ b/samples/snapshot/pom.xml @@ -0,0 +1,83 @@ + + + 4.0.0 + com.google.cloud + pubsublite-spark-snapshot + jar + Google Pub/Sub Lite Spark Connector Snapshot Samples + https://github.com/googleapis/java-pubsublite-spark + + + + com.google.cloud.samples + shared-configuration + 1.0.12 + + + + 1.8 + 1.8 + UTF-8 + + + + + + com.google.cloud + pubsublite-spark-sql-streaming + 0.0.1-SNAPSHOT + + + + + junit + junit + 4.13 + test + + + com.google.truth + truth + 1.0.1 + test + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.1.0 + + + add-snippets-source + + add-source + + + + ../snippets/src/main/java + + + + + add-snippets-tests + + add-test-source + + + + ../snippets/src/test/java + + + + + + + + diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml new file mode 100644 index 00000000..75ee5bb3 --- /dev/null +++ b/samples/snippets/pom.xml @@ -0,0 +1,51 @@ + + + 4.0.0 + com.google.cloud + pubsublite-spark-snippets + jar + Google Pub/Sub Lite Spark Connector Snippets + https://github.com/googleapis/java-pubsublite-spark + + + + com.google.cloud.samples + shared-configuration + 1.0.12 + + + + 1.8 + 1.8 + UTF-8 + + + + + + + + com.google.cloud + pubsublite-spark-sql-streaming + 0.0.0 + + + + + + junit + junit + 4.13 + test + + + com.google.truth + truth + 1.0.1 + test + + + diff --git a/synth.metadata b/synth.metadata index ea977b9c..26a76f0e 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,7 +4,7 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/java-pubsublite-spark.git", - "sha": "fd0572c74e21fd119dc8c34d1377e4c2a53228e2" + "sha": "6b2d3f80ff3485f747cc28be07383c4f25cbf958" } }, { diff --git a/synth.py b/synth.py index 768b934f..7005963e 100644 --- a/synth.py +++ b/synth.py @@ -16,4 +16,7 @@ import synthtool.languages.java as java -java.common_templates() +java.common_templates(excludes=[ + # TODO: allow when pubsublite-spark is available in libraries-bom + 'samples/install-without-bom/*', +]) diff --git a/versions.txt b/versions.txt index cba17482..788083cc 100644 --- a/versions.txt +++ b/versions.txt @@ -1,4 +1,4 @@ # Format: # module:released-version:current-version -pubsublite-spark:0.0.0:0.0.1-SNAPSHOT +pubsublite-spark-sql-streaming:0.0.0:0.0.1-SNAPSHOT