From 8fbb80eccdfafe8ffd5ff24fe04132878d09d9ae Mon Sep 17 00:00:00 2001 From: Yiru Tang Date: Sun, 15 Mar 2020 06:07:05 -0700 Subject: [PATCH] feat: proto converter library (#100) * ProtoConverter library modified: google-cloud-bigquerystorage/pom.xml new file: google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverter.java new file: google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverterTest.java new file: google-cloud-bigquerystorage/src/test/proto/test.proto modified: pom.xml * Adjust dependency ordering --- google-cloud-bigquerystorage/pom.xml | 32 +++- .../v1alpha2/ProtoSchemaConverter.java | 78 +++++++++ .../v1alpha2/ProtoSchemaConverterTest.java | 162 ++++++++++++++++++ .../src/test/proto/test.proto | 56 ++++++ pom.xml | 19 ++ 5 files changed, 346 insertions(+), 1 deletion(-) create mode 100644 google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverter.java create mode 100644 google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverterTest.java create mode 100644 google-cloud-bigquerystorage/src/test/proto/test.proto diff --git a/google-cloud-bigquerystorage/pom.xml b/google-cloud-bigquerystorage/pom.xml index 20447a23a5..362746db40 100644 --- a/google-cloud-bigquerystorage/pom.xml +++ b/google-cloud-bigquerystorage/pom.xml @@ -16,6 +16,32 @@ google-cloud-bigquerystorage + + + + kr.motd.maven + os-maven-plugin + 1.6.2 + + + + + com.google.protobuf.tools + maven-protoc-plugin + 0.4.2 + + com.google.protobuf:protoc:${project.protobuf-java.version}:exe:${os.detected.classifier} + + + + + test-compile + + + + + + io.grpc @@ -125,6 +151,10 @@ google-cloud-core ${google.core.version} + + com.google.protobuf + protobuf-java + com.google.api.grpc @@ -169,4 +199,4 @@ - \ No newline at end of file + diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverter.java new file mode 100644 index 0000000000..81fa5fd419 --- /dev/null +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverter.java @@ -0,0 +1,78 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigquery.storage.v1alpha2; + +import com.google.api.gax.grpc.GrpcStatusCode; +import com.google.api.gax.rpc.InvalidArgumentException; +import com.google.cloud.bigquery.storage.v1alpha2.ProtoBufProto.ProtoSchema; +import com.google.protobuf.DescriptorProtos.DescriptorProto; +import com.google.protobuf.DescriptorProtos.FieldDescriptorProto; +import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.Descriptors.FieldDescriptor; +import io.grpc.Status; +import java.util.*; + +// A Converter class that turns a native protobuf::DescriptorProto to a self contained +// protobuf::DescriptorProto +// that can be reconstructed by the backend. +public class ProtoSchemaConverter { + private static class StructName { + public String getName() { + return "__S" + (count++); + } + + private int count = 0; + } + + private static ProtoSchema convertInternal( + Descriptor input, List visitedTypes, StructName structName) { + DescriptorProto.Builder resultProto = DescriptorProto.newBuilder(); + resultProto.setName(structName.getName()); + visitedTypes.add(input.getFullName()); + for (int i = 0; i < input.getFields().size(); i++) { + FieldDescriptor inputField = input.getFields().get(i); + FieldDescriptorProto.Builder resultField = inputField.toProto().toBuilder(); + if (inputField.getType() == FieldDescriptor.Type.GROUP + || inputField.getType() == FieldDescriptor.Type.MESSAGE) { + if (visitedTypes.contains(inputField.getMessageType().getFullName())) { + throw new InvalidArgumentException( + "Recursive type is not supported:" + inputField.getMessageType().getFullName(), + null, + GrpcStatusCode.of(Status.Code.INVALID_ARGUMENT), + false); + } + resultProto.addNestedType( + convertInternal(inputField.getMessageType(), visitedTypes, structName) + .getProtoDescriptor()); + visitedTypes.remove(inputField.getMessageType().getFullName()); + resultField.setTypeName( + resultProto.getNestedType(resultProto.getNestedTypeCount() - 1).getName()); + } + if (inputField.getType() == FieldDescriptor.Type.ENUM) { + resultProto.addEnumType(inputField.getEnumType().toProto()); + resultField.setTypeName( + resultProto.getEnumType(resultProto.getEnumTypeCount() - 1).getName()); + } + resultProto.addField(resultField); + } + return ProtoSchema.newBuilder().setProtoDescriptor(resultProto.build()).build(); + } + + public static ProtoSchema convert(Descriptor descriptor) { + ArrayList visitedTypes = new ArrayList(); + return convertInternal(descriptor, visitedTypes, new StructName()); + } +} diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverterTest.java new file mode 100644 index 0000000000..de1e7f4888 --- /dev/null +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverterTest.java @@ -0,0 +1,162 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigquery.storage.v1alpha2; + +import com.google.api.gax.rpc.InvalidArgumentException; +import com.google.cloud.bigquery.storage.test.Test.*; +import org.junit.*; + +public class ProtoSchemaConverterTest { + @Test + public void convertSimple() { + AllSupportedTypes testProto = AllSupportedTypes.newBuilder().setStringValue("abc").build(); + ProtoBufProto.ProtoSchema protoSchema = + ProtoSchemaConverter.convert(testProto.getDescriptorForType()); + Assert.assertEquals( + "name: \"__S0\"\n" + + "field {\n" + + " name: \"int32_value\"\n" + + " number: 1\n" + + " label: LABEL_OPTIONAL\n" + + " type: TYPE_INT32\n" + + "}\n" + + "field {\n" + + " name: \"int64_value\"\n" + + " number: 2\n" + + " label: LABEL_OPTIONAL\n" + + " type: TYPE_INT64\n" + + "}\n" + + "field {\n" + + " name: \"uint32_value\"\n" + + " number: 3\n" + + " label: LABEL_OPTIONAL\n" + + " type: TYPE_UINT32\n" + + "}\n" + + "field {\n" + + " name: \"uint64_value\"\n" + + " number: 4\n" + + " label: LABEL_OPTIONAL\n" + + " type: TYPE_UINT64\n" + + "}\n" + + "field {\n" + + " name: \"float_value\"\n" + + " number: 5\n" + + " label: LABEL_OPTIONAL\n" + + " type: TYPE_FLOAT\n" + + "}\n" + + "field {\n" + + " name: \"double_value\"\n" + + " number: 6\n" + + " label: LABEL_OPTIONAL\n" + + " type: TYPE_DOUBLE\n" + + "}\n" + + "field {\n" + + " name: \"bool_value\"\n" + + " number: 7\n" + + " label: LABEL_OPTIONAL\n" + + " type: TYPE_BOOL\n" + + "}\n" + + "field {\n" + + " name: \"enum_value\"\n" + + " number: 8\n" + + " label: LABEL_OPTIONAL\n" + + " type: TYPE_ENUM\n" + + " type_name: \"TestEnum\"\n" + + "}\n" + + "field {\n" + + " name: \"string_value\"\n" + + " number: 9\n" + + " label: LABEL_REQUIRED\n" + + " type: TYPE_STRING\n" + + "}\n" + + "enum_type {\n" + + " name: \"TestEnum\"\n" + + " value {\n" + + " name: \"TestEnum0\"\n" + + " number: 0\n" + + " }\n" + + " value {\n" + + " name: \"TestEnum1\"\n" + + " number: 1\n" + + " }\n" + + "}\n", + protoSchema.getProtoDescriptor().toString()); + } + + @Test + public void convertNested() { + ComplicateType testProto = ComplicateType.newBuilder().build(); + ProtoBufProto.ProtoSchema protoSchema = + ProtoSchemaConverter.convert(testProto.getDescriptorForType()); + Assert.assertEquals( + "name: \"__S0\"\n" + + "field {\n" + + " name: \"nested_repeated_type\"\n" + + " number: 1\n" + + " label: LABEL_REPEATED\n" + + " type: TYPE_MESSAGE\n" + + " type_name: \"__S1\"\n" + + "}\n" + + "field {\n" + + " name: \"inner_type\"\n" + + " number: 2\n" + + " label: LABEL_OPTIONAL\n" + + " type: TYPE_MESSAGE\n" + + " type_name: \"__S3\"\n" + + "}\n" + + "nested_type {\n" + + " name: \"__S1\"\n" + + " field {\n" + + " name: \"inner_type\"\n" + + " number: 1\n" + + " label: LABEL_REPEATED\n" + + " type: TYPE_MESSAGE\n" + + " type_name: \"__S2\"\n" + + " }\n" + + " nested_type {\n" + + " name: \"__S2\"\n" + + " field {\n" + + " name: \"value\"\n" + + " number: 1\n" + + " label: LABEL_REPEATED\n" + + " type: TYPE_STRING\n" + + " }\n" + + " }\n" + + "}\n" + + "nested_type {\n" + + " name: \"__S3\"\n" + + " field {\n" + + " name: \"value\"\n" + + " number: 1\n" + + " label: LABEL_REPEATED\n" + + " type: TYPE_STRING\n" + + " }\n" + + "}\n", + protoSchema.getProtoDescriptor().toString()); + } + + @Test + public void convertRecursive() { + try { + RecursiveType testProto = RecursiveType.newBuilder().build(); + ProtoBufProto.ProtoSchema protoSchema = + ProtoSchemaConverter.convert(testProto.getDescriptorForType()); + Assert.fail("No exception raised"); + } catch (InvalidArgumentException e) { + // Expected exception + } + } +} diff --git a/google-cloud-bigquerystorage/src/test/proto/test.proto b/google-cloud-bigquerystorage/src/test/proto/test.proto new file mode 100644 index 0000000000..2b1a988ea6 --- /dev/null +++ b/google-cloud-bigquerystorage/src/test/proto/test.proto @@ -0,0 +1,56 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto2"; + +package com.google.cloud.bigquery.storage.test; + +import "google/protobuf/descriptor.proto"; + +enum TestEnum { + TestEnum0 = 0; + TestEnum1 = 1; +} + +message AllSupportedTypes { + optional int32 int32_value = 1; + optional int64 int64_value = 2; + optional uint32 uint32_value = 3; + optional uint64 uint64_value = 4; + optional float float_value = 5; + optional double double_value = 6; + optional bool bool_value = 7; + optional TestEnum enum_value = 8; + required string string_value = 9; +} + +message InnerType { + repeated string value = 1; +} +message NestedType { + repeated InnerType inner_type = 1; +} + +message ComplicateType { + repeated NestedType nested_repeated_type = 1; + optional InnerType inner_type = 2; +} + +message ContainsRecursive { + optional RecursiveType field = 1; +} +message RecursiveType { + optional ContainsRecursive field = 2; +} diff --git a/pom.xml b/pom.xml index eb58c09728..04ad6291a2 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,7 @@ UTF-8 UTF-8 + 3.11.4 github google-cloud-bigquerystorage-parent 1.93.2 @@ -78,6 +79,24 @@ 1.18 + + + + never + + + false + + central + Central Repository + https://repo.maven.apache.org/maven2 + + + protoc-plugin + https://dl.bintray.com/sergei-ivanov/maven/ + + +