Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
feat: Add INTERVAL and JSON type support to JsonStreamWriter (#1351)
* fix: update code comment to reflect max size change

* feat: Add JSON and INTERVAL type support to JsonStreamWriter
  • Loading branch information
yirutang committed Oct 12, 2021
1 parent ada370a commit f2121a7
Show file tree
Hide file tree
Showing 7 changed files with 70 additions and 0 deletions.
Expand Up @@ -56,6 +56,8 @@ public class BQTableSchemaToProtoDescriptor {
.put(TableFieldSchema.Type.STRUCT, FieldDescriptorProto.Type.TYPE_MESSAGE)
.put(TableFieldSchema.Type.TIME, FieldDescriptorProto.Type.TYPE_INT64)
.put(TableFieldSchema.Type.TIMESTAMP, FieldDescriptorProto.Type.TYPE_INT64)
.put(TableFieldSchema.Type.JSON, FieldDescriptorProto.Type.TYPE_STRING)
.put(TableFieldSchema.Type.INTERVAL, FieldDescriptorProto.Type.TYPE_STRING)
.build();

/**
Expand Down
Expand Up @@ -56,6 +56,8 @@ public class BQTableSchemaToProtoDescriptor {
.put(TableFieldSchema.Type.STRUCT, FieldDescriptorProto.Type.TYPE_MESSAGE)
.put(TableFieldSchema.Type.TIME, FieldDescriptorProto.Type.TYPE_INT64)
.put(TableFieldSchema.Type.TIMESTAMP, FieldDescriptorProto.Type.TYPE_INT64)
.put(TableFieldSchema.Type.JSON, FieldDescriptorProto.Type.TYPE_STRING)
.put(TableFieldSchema.Type.INTERVAL, FieldDescriptorProto.Type.TYPE_STRING)
.build();

/**
Expand Down
Expand Up @@ -245,6 +245,18 @@ public void testStructComplex() throws Exception {
.setMode(TableFieldSchema.Mode.REPEATED)
.setName("test_bignumeric_str")
.build();
final TableFieldSchema TEST_INTERVAL =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.INTERVAL)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_interval")
.build();
final TableFieldSchema TEST_JSON =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.JSON)
.setMode(TableFieldSchema.Mode.REPEATED)
.setName("test_json")
.build();
final TableSchema tableSchema =
TableSchema.newBuilder()
.addFields(0, test_int)
Expand All @@ -266,6 +278,8 @@ public void testStructComplex() throws Exception {
.addFields(16, TEST_NUMERIC_STR)
.addFields(17, TEST_BIGNUMERIC)
.addFields(18, TEST_BIGNUMERIC_STR)
.addFields(19, TEST_INTERVAL)
.addFields(20, TEST_JSON)
.build();
final Descriptor descriptor =
BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema);
Expand Down
Expand Up @@ -394,6 +394,18 @@ public class JsonToProtoMessageTest {
.setMode(TableFieldSchema.Mode.REPEATED)
.setName("test_bignumeric_str")
.build();
final TableFieldSchema TEST_INTERVAL =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.INTERVAL)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_interval")
.build();
final TableFieldSchema TEST_JSON =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.JSON)
.setMode(TableFieldSchema.Mode.REPEATED)
.setName("test_json")
.build();
private final TableSchema COMPLEX_TABLE_SCHEMA =
TableSchema.newBuilder()
.addFields(0, TEST_INT)
Expand All @@ -415,6 +427,8 @@ public class JsonToProtoMessageTest {
.addFields(16, TEST_NUMERIC_STR)
.addFields(17, TEST_BIGNUMERIC)
.addFields(18, TEST_BIGNUMERIC_STR)
.addFields(19, TEST_INTERVAL)
.addFields(20, TEST_JSON)
.build();

@Test
Expand Down Expand Up @@ -743,6 +757,8 @@ public void testStructComplex() throws Exception {
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("2.3")))
.addTestBignumericStr(
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23")))
.setTestInterval("0-0 0 0:0:0.000005")
.addTestJson("{'a':'b'}")
.build();
JSONObject complex_lvl2 = new JSONObject();
complex_lvl2.put("test_int", 3);
Expand Down Expand Up @@ -789,6 +805,8 @@ public void testStructComplex() throws Exception {
"test_bignumeric",
BigDecimalByteStringEncoder.encodeToNumericByteString(BigDecimal.valueOf(2.3)));
json.put("test_bignumeric_str", new JSONArray(new String[] {"1.23"}));
json.put("test_interval", "0-0 0 0:0:0.000005");
json.put("test_json", new JSONArray(new String[] {"{'a':'b'}"}));
DynamicMessage protoMsg =
JsonToProtoMessage.convertJsonToProtoMessage(
ComplexRoot.getDescriptor(), COMPLEX_TABLE_SCHEMA, json);
Expand Down
Expand Up @@ -245,6 +245,18 @@ public void testStructComplex() throws Exception {
.setMode(TableFieldSchema.Mode.REPEATED)
.setName("test_bignumeric_str")
.build();
final TableFieldSchema TEST_INTERVAL =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.INTERVAL)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_interval")
.build();
final TableFieldSchema TEST_JSON =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.JSON)
.setMode(TableFieldSchema.Mode.REPEATED)
.setName("test_json")
.build();
final TableSchema tableSchema =
TableSchema.newBuilder()
.addFields(0, test_int)
Expand All @@ -266,6 +278,8 @@ public void testStructComplex() throws Exception {
.addFields(16, TEST_NUMERIC_STR)
.addFields(17, TEST_BIGNUMERIC)
.addFields(18, TEST_BIGNUMERIC_STR)
.addFields(19, TEST_INTERVAL)
.addFields(20, TEST_JSON)
.build();
final Descriptor descriptor =
BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema);
Expand Down
Expand Up @@ -395,6 +395,18 @@ public class JsonToProtoMessageTest {
.setMode(TableFieldSchema.Mode.REPEATED)
.setName("test_bignumeric_str")
.build();
private final TableFieldSchema TEST_INTERVAL =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.INTERVAL)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_interval")
.build();
private final TableFieldSchema TEST_JSON =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.JSON)
.setMode(TableFieldSchema.Mode.REPEATED)
.setName("test_json")
.build();
private final TableSchema COMPLEX_TABLE_SCHEMA =
TableSchema.newBuilder()
.addFields(0, TEST_INT)
Expand All @@ -416,6 +428,8 @@ public class JsonToProtoMessageTest {
.addFields(16, TEST_NUMERIC_STR)
.addFields(17, TEST_BIGNUMERIC)
.addFields(18, TEST_BIGNUMERIC_STR)
.addFields(19, TEST_INTERVAL)
.addFields(20, TEST_JSON)
.build();

@Test
Expand Down Expand Up @@ -744,6 +758,8 @@ public void testStructComplex() throws Exception {
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("2.3")))
.addTestBignumericStr(
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23")))
.setTestInterval("0-0 0 0:0:0.000005")
.addTestJson("{'a':'b'}")
.build();
JSONObject complex_lvl2 = new JSONObject();
complex_lvl2.put("test_int", 3);
Expand Down Expand Up @@ -790,6 +806,8 @@ public void testStructComplex() throws Exception {
"test_bignumeric",
BigDecimalByteStringEncoder.encodeToNumericByteString(BigDecimal.valueOf(2.3)));
json.put("test_bignumeric_str", new JSONArray(new String[] {"1.23"}));
json.put("test_interval", "0-0 0 0:0:0.000005");
json.put("test_json", new JSONArray(new String[] {"{'a':'b'}"}));
DynamicMessage protoMsg =
JsonToProtoMessage.convertJsonToProtoMessage(
ComplexRoot.getDescriptor(), COMPLEX_TABLE_SCHEMA, json);
Expand Down
2 changes: 2 additions & 0 deletions google-cloud-bigquerystorage/src/test/proto/jsonTest.proto
Expand Up @@ -23,6 +23,8 @@ message ComplexRoot {
optional bytes test_numeric_str = 17;
optional bytes test_bignumeric = 18;
repeated bytes test_bignumeric_str = 19;
optional string test_interval = 20;
repeated string test_json = 21;
}

message CasingComplex {
Expand Down

0 comments on commit f2121a7

Please sign in to comment.