feat: proto converter library

google-cloud-bigquerystorage/pom.xml

@@ -16,6 +16,32 @@
   <properties>
     <site.installationModule>google-cloud-bigquerystorage</site.installationModule>
   </properties>
+  <build>
+    <extensions>
+      <extension>
+        <groupId>kr.motd.maven</groupId>
+        <artifactId>os-maven-plugin</artifactId>
+        <version>1.6.2</version>
+      </extension>
+    </extensions>
+    <plugins>
+      <plugin>
+        <groupId>com.google.protobuf.tools</groupId>
+        <artifactId>maven-protoc-plugin</artifactId>
+        <version>0.4.2</version>
+        <configuration>
+          <protocArtifact>com.google.protobuf:protoc:${project.protobuf-java.version}:exe:${os.detected.classifier}</protocArtifact>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-compile</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
   <dependencies>
     <dependency>
       <groupId>io.grpc</groupId>
@@ -125,6 +151,10 @@
       <artifactId>google-cloud-core</artifactId>
       <version>${google.core.version}</version>
     </dependency>
+    <dependency>
+      <groupId>com.google.protobuf</groupId>
+      <artifactId>protobuf-java</artifactId>
+    </dependency>
 
     <dependency>
       <groupId>com.google.api.grpc</groupId>
@@ -169,4 +199,4 @@
       </dependencies>
     </profile>
   </profiles>
-</project>
+</project>

google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverter.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigquery.storage.v1alpha2;
+
+import com.google.api.gax.grpc.GrpcStatusCode;
+import com.google.api.gax.rpc.InvalidArgumentException;
+import com.google.cloud.bigquery.storage.v1alpha2.ProtoBufProto.ProtoSchema;
+import com.google.protobuf.DescriptorProtos.DescriptorProto;
+import com.google.protobuf.DescriptorProtos.FieldDescriptorProto;
+import com.google.protobuf.Descriptors.Descriptor;
+import com.google.protobuf.Descriptors.FieldDescriptor;
+import io.grpc.Status;
+import java.util.*;
+
+// A Converter class that turns a native protobuf::DescriptorProto to a self contained
+// protobuf::DescriptorProto
+// that can be reconstructed by the backend.
+public class ProtoSchemaConverter {
+  private static class StructName {
+    public String getName() {
+      return "__S" + (count++);
+    }
+
+    private int count = 0;
+  }
+
+  private static ProtoSchema convertInternal(
+      Descriptor input, List<String> visitedTypes, StructName structName) {
+    DescriptorProto.Builder resultProto = DescriptorProto.newBuilder();
+    resultProto.setName(structName.getName());
+    visitedTypes.add(input.getFullName());
+    for (int i = 0; i < input.getFields().size(); i++) {
+      FieldDescriptor inputField = input.getFields().get(i);
+      FieldDescriptorProto.Builder resultField = inputField.toProto().toBuilder();
+      if (inputField.getType() == FieldDescriptor.Type.GROUP
+          || inputField.getType() == FieldDescriptor.Type.MESSAGE) {
+        if (visitedTypes.contains(inputField.getMessageType().getFullName())) {
+          throw new InvalidArgumentException(
+              "Recursive type is not supported:" + inputField.getMessageType().getFullName(),
+              null,
+              GrpcStatusCode.of(Status.Code.INVALID_ARGUMENT),
+              false);
+        }
+        resultProto.addNestedType(
+            convertInternal(inputField.getMessageType(), visitedTypes, structName)
+                .getProtoDescriptor());
+        visitedTypes.remove(inputField.getMessageType().getFullName());
+        resultField.setTypeName(
+            resultProto.getNestedType(resultProto.getNestedTypeCount() - 1).getName());
+      }
+      if (inputField.getType() == FieldDescriptor.Type.ENUM) {
+        resultProto.addEnumType(inputField.getEnumType().toProto());
+        resultField.setTypeName(
+            resultProto.getEnumType(resultProto.getEnumTypeCount() - 1).getName());
+      }
+      resultProto.addField(resultField);
+    }
+    return ProtoSchema.newBuilder().setProtoDescriptor(resultProto.build()).build();
+  }
+
+  public static ProtoSchema convert(Descriptor descriptor) {
+    ArrayList<String> visitedTypes = new ArrayList<String>();
+    return convertInternal(descriptor, visitedTypes, new StructName());
+  }
+}

google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/ProtoSchemaConverterTest.java

@@ -0,0 +1,162 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigquery.storage.v1alpha2;
+
+import com.google.api.gax.rpc.InvalidArgumentException;
+import com.google.cloud.bigquery.storage.test.Test.*;
+import org.junit.*;
+
+public class ProtoSchemaConverterTest {
+  @Test
+  public void convertSimple() {
+    AllSupportedTypes testProto = AllSupportedTypes.newBuilder().setStringValue("abc").build();
+    ProtoBufProto.ProtoSchema protoSchema =
+        ProtoSchemaConverter.convert(testProto.getDescriptorForType());
+    Assert.assertEquals(
+        "name: \"__S0\"\n"
+            + "field {\n"
+            + "  name: \"int32_value\"\n"
+            + "  number: 1\n"
+            + "  label: LABEL_OPTIONAL\n"
+            + "  type: TYPE_INT32\n"
+            + "}\n"
+            + "field {\n"
+            + "  name: \"int64_value\"\n"
+            + "  number: 2\n"
+            + "  label: LABEL_OPTIONAL\n"
+            + "  type: TYPE_INT64\n"
+            + "}\n"
+            + "field {\n"
+            + "  name: \"uint32_value\"\n"
+            + "  number: 3\n"
+            + "  label: LABEL_OPTIONAL\n"
+            + "  type: TYPE_UINT32\n"
+            + "}\n"
+            + "field {\n"
+            + "  name: \"uint64_value\"\n"
+            + "  number: 4\n"
+            + "  label: LABEL_OPTIONAL\n"
+            + "  type: TYPE_UINT64\n"
+            + "}\n"
+            + "field {\n"
+            + "  name: \"float_value\"\n"
+            + "  number: 5\n"
+            + "  label: LABEL_OPTIONAL\n"
+            + "  type: TYPE_FLOAT\n"
+            + "}\n"
+            + "field {\n"
+            + "  name: \"double_value\"\n"
+            + "  number: 6\n"
+            + "  label: LABEL_OPTIONAL\n"
+            + "  type: TYPE_DOUBLE\n"
+            + "}\n"
+            + "field {\n"
+            + "  name: \"bool_value\"\n"
+            + "  number: 7\n"
+            + "  label: LABEL_OPTIONAL\n"
+            + "  type: TYPE_BOOL\n"
+            + "}\n"
+            + "field {\n"
+            + "  name: \"enum_value\"\n"
+            + "  number: 8\n"
+            + "  label: LABEL_OPTIONAL\n"
+            + "  type: TYPE_ENUM\n"
+            + "  type_name: \"TestEnum\"\n"
+            + "}\n"
+            + "field {\n"
+            + "  name: \"string_value\"\n"
+            + "  number: 9\n"
+            + "  label: LABEL_REQUIRED\n"
+            + "  type: TYPE_STRING\n"
+            + "}\n"
+            + "enum_type {\n"
+            + "  name: \"TestEnum\"\n"
+            + "  value {\n"
+            + "    name: \"TestEnum0\"\n"
+            + "    number: 0\n"
+            + "  }\n"
+            + "  value {\n"
+            + "    name: \"TestEnum1\"\n"
+            + "    number: 1\n"
+            + "  }\n"
+            + "}\n",
+        protoSchema.getProtoDescriptor().toString());
+  }
+
+  @Test
+  public void convertNested() {
+    ComplicateType testProto = ComplicateType.newBuilder().build();
+    ProtoBufProto.ProtoSchema protoSchema =
+        ProtoSchemaConverter.convert(testProto.getDescriptorForType());
+    Assert.assertEquals(
+        "name: \"__S0\"\n"
+            + "field {\n"
+            + "  name: \"nested_repeated_type\"\n"
+            + "  number: 1\n"
+            + "  label: LABEL_REPEATED\n"
+            + "  type: TYPE_MESSAGE\n"
+            + "  type_name: \"__S1\"\n"
+            + "}\n"
+            + "field {\n"
+            + "  name: \"inner_type\"\n"
+            + "  number: 2\n"
+            + "  label: LABEL_OPTIONAL\n"
+            + "  type: TYPE_MESSAGE\n"
+            + "  type_name: \"__S3\"\n"
+            + "}\n"
+            + "nested_type {\n"
+            + "  name: \"__S1\"\n"
+            + "  field {\n"
+            + "    name: \"inner_type\"\n"
+            + "    number: 1\n"
+            + "    label: LABEL_REPEATED\n"
+            + "    type: TYPE_MESSAGE\n"
+            + "    type_name: \"__S2\"\n"
+            + "  }\n"
+            + "  nested_type {\n"
+            + "    name: \"__S2\"\n"
+            + "    field {\n"
+            + "      name: \"value\"\n"
+            + "      number: 1\n"
+            + "      label: LABEL_REPEATED\n"
+            + "      type: TYPE_STRING\n"
+            + "    }\n"
+            + "  }\n"
+            + "}\n"
+            + "nested_type {\n"
+            + "  name: \"__S3\"\n"
+            + "  field {\n"
+            + "    name: \"value\"\n"
+            + "    number: 1\n"
+            + "    label: LABEL_REPEATED\n"
+            + "    type: TYPE_STRING\n"
+            + "  }\n"
+            + "}\n",
+        protoSchema.getProtoDescriptor().toString());
+  }
+
+  @Test
+  public void convertRecursive() {
+    try {
+      RecursiveType testProto = RecursiveType.newBuilder().build();
+      ProtoBufProto.ProtoSchema protoSchema =
+          ProtoSchemaConverter.convert(testProto.getDescriptorForType());
+      Assert.fail("No exception raised");
+    } catch (InvalidArgumentException e) {
+      // Expected exception
+    }
+  }
+}

google-cloud-bigquerystorage/src/test/proto/test.proto

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+syntax = "proto2";
+
+package com.google.cloud.bigquery.storage.test;
+
+import "google/protobuf/descriptor.proto";
+
+enum TestEnum {
+  TestEnum0 = 0;
+  TestEnum1 = 1;
+}
+
+message AllSupportedTypes {
+  optional int32 int32_value = 1;
+  optional int64 int64_value = 2;
+  optional uint32 uint32_value = 3;
+  optional uint64 uint64_value = 4;
+  optional float float_value = 5;
+  optional double double_value = 6;
+  optional bool bool_value = 7;
+  optional TestEnum enum_value = 8;
+  required string string_value = 9;
+}
+
+message InnerType {
+  repeated string value = 1;
+}
+message NestedType {
+  repeated InnerType inner_type = 1;
+}
+
+message ComplicateType {
+  repeated NestedType nested_repeated_type = 1;
+  optional InnerType inner_type = 2;
+}
+
+message ContainsRecursive {
+  optional RecursiveType field = 1;
+}
+message RecursiveType {
+  optional ContainsRecursive field = 2;
+}

pom.xml

@@ -61,6 +61,7 @@
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+    <project.protobuf-java.version>3.11.4</project.protobuf-java.version>
     <github.global.server>github</github.global.server>
     <site.installationModule>google-cloud-bigquerystorage-parent</site.installationModule>
     <google.core.version>1.92.5</google.core.version>
@@ -78,6 +79,24 @@
     <animal-sniffer.version>1.18</animal-sniffer.version>
   </properties>
 
+  <pluginRepositories>
+    <pluginRepository>
+      <releases>
+        <updatePolicy>never</updatePolicy>
+      </releases>
+      <snapshots>
+        <enabled>false</enabled>
+      </snapshots>
+      <id>central</id>
+      <name>Central Repository</name>
+      <url>https://repo.maven.apache.org/maven2</url>
+    </pluginRepository>
+    <pluginRepository>
+      <id>protoc-plugin</id>
+      <url>https://dl.bintray.com/sergei-ivanov/maven/</url>
+    </pluginRepository>
+  </pluginRepositories>
+
   <dependencyManagement>
     <dependencies>
       <dependency>