From 46729398eee0f728c3de731f803580342fbe787f Mon Sep 17 00:00:00 2001 From: allenc3 <31832826+allenc3@users.noreply.github.com> Date: Thu, 16 Jul 2020 11:49:06 -0500 Subject: [PATCH] fix: BQTableSchemaToProtobufDescriptor will now only generate lower-cased fieldnames in the protobuf descriptor (#415) * fix: all protofields in the generated descriptor will be lowercased * Remove debug output * Test commits * Reverted test commit * No-op --- .../BQTableSchemaToProtoDescriptor.java | 7 +- .../BQTableSchemaToProtoDescriptorTest.java | 84 ++++++++++++++++++- .../src/test/proto/jsonTest.proto | 16 +++- 3 files changed, 99 insertions(+), 8 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/BQTableSchemaToProtoDescriptor.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/BQTableSchemaToProtoDescriptor.java index 2dbdbb442e..96f84fa960 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/BQTableSchemaToProtoDescriptor.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/BQTableSchemaToProtoDescriptor.java @@ -28,8 +28,9 @@ import java.util.List; /** - * Converts a BQ table schema to protobuf descriptor. The mapping between field types and field - * modes are shown in the ImmutableMaps below. + * Converts a BQ table schema to protobuf descriptor. All field names will be converted to lowercase + * when constructing the protobuf descriptor. The mapping between field types and field modes are + * shown in the ImmutableMaps below. */ public class BQTableSchemaToProtoDescriptor { private static ImmutableMap @@ -130,7 +131,7 @@ private static Descriptor convertBQTableSchemaToProtoDescriptorImpl( private static FieldDescriptorProto convertBQTableFieldToProtoField( Table.TableFieldSchema BQTableField, int index, String scope) { Table.TableFieldSchema.Mode mode = BQTableField.getMode(); - String fieldName = BQTableField.getName(); + String fieldName = BQTableField.getName().toLowerCase(); if (BQTableField.getType() == Table.TableFieldSchema.Type.STRUCT) { return FieldDescriptorProto.newBuilder() .setName(fieldName) diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/BQTableSchemaToProtoDescriptorTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/BQTableSchemaToProtoDescriptorTest.java index b8dce2bdd1..fcca818afc 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/BQTableSchemaToProtoDescriptorTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/BQTableSchemaToProtoDescriptorTest.java @@ -171,7 +171,7 @@ public void testStructComplex() throws Exception { .setType(Table.TableFieldSchema.Type.STRUCT) .setMode(Table.TableFieldSchema.Mode.REQUIRED) .addFields(0, test_int) - .setName("complexLvl2") + .setName("complex_lvl2") .build(); final Table.TableFieldSchema ComplexLvl1 = Table.TableFieldSchema.newBuilder() @@ -179,7 +179,7 @@ public void testStructComplex() throws Exception { .setMode(Table.TableFieldSchema.Mode.REQUIRED) .addFields(0, test_int) .addFields(1, ComplexLvl2) - .setName("complexLvl1") + .setName("complex_lvl1") .build(); final Table.TableSchema tableSchema = Table.TableSchema.newBuilder() @@ -197,6 +197,86 @@ public void testStructComplex() throws Exception { isDescriptorEqual(descriptor, ComplexRoot.getDescriptor()); } + @Test + public void testCasingComplexStruct() throws Exception { + final Table.TableFieldSchema required = + Table.TableFieldSchema.newBuilder() + .setType(Table.TableFieldSchema.Type.INT64) + .setMode(Table.TableFieldSchema.Mode.REQUIRED) + .setName("tEsT_ReQuIrEd") + .build(); + final Table.TableFieldSchema repeated = + Table.TableFieldSchema.newBuilder() + .setType(Table.TableFieldSchema.Type.INT64) + .setMode(Table.TableFieldSchema.Mode.REPEATED) + .setName("tESt_repEATed") + .build(); + final Table.TableFieldSchema optional = + Table.TableFieldSchema.newBuilder() + .setType(Table.TableFieldSchema.Type.INT64) + .setMode(Table.TableFieldSchema.Mode.NULLABLE) + .setName("test_opTIONal") + .build(); + final Table.TableFieldSchema test_int = + Table.TableFieldSchema.newBuilder() + .setType(Table.TableFieldSchema.Type.INT64) + .setMode(Table.TableFieldSchema.Mode.NULLABLE) + .setName("TEST_INT") + .build(); + final Table.TableFieldSchema test_string = + Table.TableFieldSchema.newBuilder() + .setType(Table.TableFieldSchema.Type.STRING) + .setMode(Table.TableFieldSchema.Mode.REPEATED) + .setName("TEST_STRING") + .build(); + final Table.TableFieldSchema test_bytes = + Table.TableFieldSchema.newBuilder() + .setType(Table.TableFieldSchema.Type.BYTES) + .setMode(Table.TableFieldSchema.Mode.REQUIRED) + .setName("TEST_BYTES") + .build(); + final Table.TableFieldSchema test_bool = + Table.TableFieldSchema.newBuilder() + .setType(Table.TableFieldSchema.Type.BOOL) + .setMode(Table.TableFieldSchema.Mode.NULLABLE) + .setName("TEST_BOOL") + .build(); + final Table.TableFieldSchema test_double = + Table.TableFieldSchema.newBuilder() + .setType(Table.TableFieldSchema.Type.DOUBLE) + .setMode(Table.TableFieldSchema.Mode.REPEATED) + .setName("TEST_DOUBLE") + .build(); + final Table.TableFieldSchema test_date = + Table.TableFieldSchema.newBuilder() + .setType(Table.TableFieldSchema.Type.DATE) + .setMode(Table.TableFieldSchema.Mode.REQUIRED) + .setName("TEST_DATE") + .build(); + final Table.TableFieldSchema option_test = + Table.TableFieldSchema.newBuilder() + .setType(Table.TableFieldSchema.Type.STRUCT) + .setMode(Table.TableFieldSchema.Mode.REQUIRED) + .addFields(0, required) + .addFields(1, repeated) + .addFields(2, optional) + .setName("option_test") + .build(); + final Table.TableSchema tableSchema = + Table.TableSchema.newBuilder() + .addFields(0, test_int) + .addFields(1, test_string) + .addFields(2, test_bytes) + .addFields(3, test_bool) + .addFields(4, test_double) + .addFields(5, test_date) + .addFields(6, option_test) + .build(); + final Descriptor descriptor = + BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema); + isDescriptorEqual(descriptor, CasingComplex.getDescriptor()); + } + @Test public void testOptions() throws Exception { final Table.TableFieldSchema required = diff --git a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto index a0a0333fd8..26a3499c23 100644 --- a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto +++ b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto @@ -9,13 +9,23 @@ message ComplexRoot { optional bool test_bool = 4; repeated double test_double = 5; required int32 test_date = 6; - required ComplexLvl1 complexLvl1 = 7; - required ComplexLvl2 complexLvl2 = 8; + required ComplexLvl1 complex_lvl1 = 7; + required ComplexLvl2 complex_lvl2 = 8; +} + +message CasingComplex { + optional int64 test_int = 1; + repeated string test_string = 2; + required bytes test_bytes = 3; + optional bool test_bool = 4; + repeated double test_double = 5; + required int32 test_date = 6; + required OptionTest option_test = 7; } message ComplexLvl1 { optional int64 test_int = 1; - required ComplexLvl2 complexLvl2 = 2; + required ComplexLvl2 complex_lvl2 = 2; } message ComplexLvl2 {