Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: BQTableSchemaToProtobufDescriptor will now only generate lower-cased fieldnames in the protobuf descriptor #415

Merged
merged 5 commits into from Jul 16, 2020
Merged

fix: BQTableSchemaToProtobufDescriptor will now only generate lower-cased fieldnames in the protobuf descriptor #415

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
dbe32e6
fix: all protofields in the generated descriptor will be lowercased
allenc3 Jul 14, 2020
80a73f6
Remove debug output
allenc3 Jul 15, 2020
87c7769
Test commits
allenc3 Jul 15, 2020
3033a94
Reverted test commit
allenc3 Jul 15, 2020
2a09ee5
No-op
allenc3 Jul 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
7 changes: 4 additions & 3 deletions .../main/java/com/google/cloud/bigquery/storage/v1alpha2/BQTableSchemaToProtoDescriptor.java
View file
Open in desktop
Expand Up @@ -28,8 +28,9 @@
import java.util.List;

/**
* Converts a BQ table schema to protobuf descriptor. The mapping between field types and field
* modes are shown in the ImmutableMaps below.
* Converts a BQ table schema to protobuf descriptor. All field names will be converted to lowercase
* when constructing the protobuf descriptor. The mapping between field types and field modes are
* shown in the ImmutableMaps below.
*/
public class BQTableSchemaToProtoDescriptor {
private static ImmutableMap<Table.TableFieldSchema.Mode, FieldDescriptorProto.Label>
Expand Down Expand Up @@ -130,7 +131,7 @@ private static Descriptor convertBQTableSchemaToProtoDescriptorImpl(
private static FieldDescriptorProto convertBQTableFieldToProtoField(
Table.TableFieldSchema BQTableField, int index, String scope) {
Table.TableFieldSchema.Mode mode = BQTableField.getMode();
String fieldName = BQTableField.getName();
String fieldName = BQTableField.getName().toLowerCase();
if (BQTableField.getType() == Table.TableFieldSchema.Type.STRUCT) {
return FieldDescriptorProto.newBuilder()
.setName(fieldName)
Expand Down
84 changes: 82 additions & 2 deletions ...t/java/com/google/cloud/bigquery/storage/v1alpha2/BQTableSchemaToProtoDescriptorTest.java
View file
Open in desktop
Expand Up @@ -171,15 +171,15 @@ public void testStructComplex() throws Exception {
.setType(Table.TableFieldSchema.Type.STRUCT)
.setMode(Table.TableFieldSchema.Mode.REQUIRED)
.addFields(0, test_int)
.setName("complexLvl2")
.setName("complex_lvl2")
.build();
final Table.TableFieldSchema ComplexLvl1 =
Table.TableFieldSchema.newBuilder()
.setType(Table.TableFieldSchema.Type.STRUCT)
.setMode(Table.TableFieldSchema.Mode.REQUIRED)
.addFields(0, test_int)
.addFields(1, ComplexLvl2)
.setName("complexLvl1")
.setName("complex_lvl1")
.build();
final Table.TableSchema tableSchema =
Table.TableSchema.newBuilder()
Expand All @@ -197,6 +197,86 @@ public void testStructComplex() throws Exception {
isDescriptorEqual(descriptor, ComplexRoot.getDescriptor());
}

@Test
public void testCasingComplexStruct() throws Exception {
final Table.TableFieldSchema required =
Table.TableFieldSchema.newBuilder()
.setType(Table.TableFieldSchema.Type.INT64)
.setMode(Table.TableFieldSchema.Mode.REQUIRED)
.setName("tEsT_ReQuIrEd")
.build();
final Table.TableFieldSchema repeated =
Table.TableFieldSchema.newBuilder()
.setType(Table.TableFieldSchema.Type.INT64)
.setMode(Table.TableFieldSchema.Mode.REPEATED)
.setName("tESt_repEATed")
.build();
final Table.TableFieldSchema optional =
Table.TableFieldSchema.newBuilder()
.setType(Table.TableFieldSchema.Type.INT64)
.setMode(Table.TableFieldSchema.Mode.NULLABLE)
.setName("test_opTIONal")
.build();
final Table.TableFieldSchema test_int =
Table.TableFieldSchema.newBuilder()
.setType(Table.TableFieldSchema.Type.INT64)
.setMode(Table.TableFieldSchema.Mode.NULLABLE)
.setName("TEST_INT")
.build();
final Table.TableFieldSchema test_string =
Table.TableFieldSchema.newBuilder()
.setType(Table.TableFieldSchema.Type.STRING)
.setMode(Table.TableFieldSchema.Mode.REPEATED)
.setName("TEST_STRING")
.build();
final Table.TableFieldSchema test_bytes =
Table.TableFieldSchema.newBuilder()
.setType(Table.TableFieldSchema.Type.BYTES)
.setMode(Table.TableFieldSchema.Mode.REQUIRED)
.setName("TEST_BYTES")
.build();
final Table.TableFieldSchema test_bool =
Table.TableFieldSchema.newBuilder()
.setType(Table.TableFieldSchema.Type.BOOL)
.setMode(Table.TableFieldSchema.Mode.NULLABLE)
.setName("TEST_BOOL")
.build();
final Table.TableFieldSchema test_double =
Table.TableFieldSchema.newBuilder()
.setType(Table.TableFieldSchema.Type.DOUBLE)
.setMode(Table.TableFieldSchema.Mode.REPEATED)
.setName("TEST_DOUBLE")
.build();
final Table.TableFieldSchema test_date =
Table.TableFieldSchema.newBuilder()
.setType(Table.TableFieldSchema.Type.DATE)
.setMode(Table.TableFieldSchema.Mode.REQUIRED)
.setName("TEST_DATE")
.build();
final Table.TableFieldSchema option_test =
Table.TableFieldSchema.newBuilder()
.setType(Table.TableFieldSchema.Type.STRUCT)
.setMode(Table.TableFieldSchema.Mode.REQUIRED)
.addFields(0, required)
.addFields(1, repeated)
.addFields(2, optional)
.setName("option_test")
.build();
final Table.TableSchema tableSchema =
Table.TableSchema.newBuilder()
.addFields(0, test_int)
.addFields(1, test_string)
.addFields(2, test_bytes)
.addFields(3, test_bool)
.addFields(4, test_double)
.addFields(5, test_date)
.addFields(6, option_test)
.build();
final Descriptor descriptor =
BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema);
isDescriptorEqual(descriptor, CasingComplex.getDescriptor());
}

@Test
public void testOptions() throws Exception {
final Table.TableFieldSchema required =
Expand Down
16 changes: 13 additions & 3 deletions google-cloud-bigquerystorage/src/test/proto/jsonTest.proto
View file
Open in desktop
Expand Up @@ -9,13 +9,23 @@ message ComplexRoot {
optional bool test_bool = 4;
repeated double test_double = 5;
required int32 test_date = 6;
required ComplexLvl1 complexLvl1 = 7;
required ComplexLvl2 complexLvl2 = 8;
required ComplexLvl1 complex_lvl1 = 7;
required ComplexLvl2 complex_lvl2 = 8;
}

message CasingComplex {
optional int64 test_int = 1;
repeated string test_string = 2;
required bytes test_bytes = 3;
optional bool test_bool = 4;
repeated double test_double = 5;
required int32 test_date = 6;
required OptionTest option_test = 7;
}

message ComplexLvl1 {
optional int64 test_int = 1;
required ComplexLvl2 complexLvl2 = 2;
required ComplexLvl2 complex_lvl2 = 2;
}

message ComplexLvl2 {
Expand Down