From f2121a796fa04d153124d87bd0d1041a401c0a42 Mon Sep 17 00:00:00 2001 From: Yiru Tang Date: Tue, 12 Oct 2021 12:14:58 -0700 Subject: [PATCH] feat: Add INTERVAL and JSON type support to JsonStreamWriter (#1351) * fix: update code comment to reflect max size change * feat: Add JSON and INTERVAL type support to JsonStreamWriter --- .../v1/BQTableSchemaToProtoDescriptor.java | 2 ++ .../BQTableSchemaToProtoDescriptor.java | 2 ++ .../v1/BQTableSchemaToProtoDescriptorTest.java | 14 ++++++++++++++ .../storage/v1/JsonToProtoMessageTest.java | 18 ++++++++++++++++++ .../BQTableSchemaToProtoDescriptorTest.java | 14 ++++++++++++++ .../v1beta2/JsonToProtoMessageTest.java | 18 ++++++++++++++++++ .../src/test/proto/jsonTest.proto | 2 ++ 7 files changed, 70 insertions(+) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java index cfb34a3f8e..6c2a3dfebc 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java @@ -56,6 +56,8 @@ public class BQTableSchemaToProtoDescriptor { .put(TableFieldSchema.Type.STRUCT, FieldDescriptorProto.Type.TYPE_MESSAGE) .put(TableFieldSchema.Type.TIME, FieldDescriptorProto.Type.TYPE_INT64) .put(TableFieldSchema.Type.TIMESTAMP, FieldDescriptorProto.Type.TYPE_INT64) + .put(TableFieldSchema.Type.JSON, FieldDescriptorProto.Type.TYPE_STRING) + .put(TableFieldSchema.Type.INTERVAL, FieldDescriptorProto.Type.TYPE_STRING) .build(); /** diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptor.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptor.java index ec9083fefa..99128101a5 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptor.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptor.java @@ -56,6 +56,8 @@ public class BQTableSchemaToProtoDescriptor { .put(TableFieldSchema.Type.STRUCT, FieldDescriptorProto.Type.TYPE_MESSAGE) .put(TableFieldSchema.Type.TIME, FieldDescriptorProto.Type.TYPE_INT64) .put(TableFieldSchema.Type.TIMESTAMP, FieldDescriptorProto.Type.TYPE_INT64) + .put(TableFieldSchema.Type.JSON, FieldDescriptorProto.Type.TYPE_STRING) + .put(TableFieldSchema.Type.INTERVAL, FieldDescriptorProto.Type.TYPE_STRING) .build(); /** diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java index 3ef5f87d02..35b9944735 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java @@ -245,6 +245,18 @@ public void testStructComplex() throws Exception { .setMode(TableFieldSchema.Mode.REPEATED) .setName("test_bignumeric_str") .build(); + final TableFieldSchema TEST_INTERVAL = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.INTERVAL) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_interval") + .build(); + final TableFieldSchema TEST_JSON = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.JSON) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_json") + .build(); final TableSchema tableSchema = TableSchema.newBuilder() .addFields(0, test_int) @@ -266,6 +278,8 @@ public void testStructComplex() throws Exception { .addFields(16, TEST_NUMERIC_STR) .addFields(17, TEST_BIGNUMERIC) .addFields(18, TEST_BIGNUMERIC_STR) + .addFields(19, TEST_INTERVAL) + .addFields(20, TEST_JSON) .build(); final Descriptor descriptor = BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema); diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java index db5b14f73d..47285f8e0f 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java @@ -394,6 +394,18 @@ public class JsonToProtoMessageTest { .setMode(TableFieldSchema.Mode.REPEATED) .setName("test_bignumeric_str") .build(); + final TableFieldSchema TEST_INTERVAL = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.INTERVAL) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_interval") + .build(); + final TableFieldSchema TEST_JSON = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.JSON) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_json") + .build(); private final TableSchema COMPLEX_TABLE_SCHEMA = TableSchema.newBuilder() .addFields(0, TEST_INT) @@ -415,6 +427,8 @@ public class JsonToProtoMessageTest { .addFields(16, TEST_NUMERIC_STR) .addFields(17, TEST_BIGNUMERIC) .addFields(18, TEST_BIGNUMERIC_STR) + .addFields(19, TEST_INTERVAL) + .addFields(20, TEST_JSON) .build(); @Test @@ -743,6 +757,8 @@ public void testStructComplex() throws Exception { BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("2.3"))) .addTestBignumericStr( BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23"))) + .setTestInterval("0-0 0 0:0:0.000005") + .addTestJson("{'a':'b'}") .build(); JSONObject complex_lvl2 = new JSONObject(); complex_lvl2.put("test_int", 3); @@ -789,6 +805,8 @@ public void testStructComplex() throws Exception { "test_bignumeric", BigDecimalByteStringEncoder.encodeToNumericByteString(BigDecimal.valueOf(2.3))); json.put("test_bignumeric_str", new JSONArray(new String[] {"1.23"})); + json.put("test_interval", "0-0 0 0:0:0.000005"); + json.put("test_json", new JSONArray(new String[] {"{'a':'b'}"})); DynamicMessage protoMsg = JsonToProtoMessage.convertJsonToProtoMessage( ComplexRoot.getDescriptor(), COMPLEX_TABLE_SCHEMA, json); diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java index c6e0973d58..bedba1ebb8 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java @@ -245,6 +245,18 @@ public void testStructComplex() throws Exception { .setMode(TableFieldSchema.Mode.REPEATED) .setName("test_bignumeric_str") .build(); + final TableFieldSchema TEST_INTERVAL = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.INTERVAL) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_interval") + .build(); + final TableFieldSchema TEST_JSON = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.JSON) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_json") + .build(); final TableSchema tableSchema = TableSchema.newBuilder() .addFields(0, test_int) @@ -266,6 +278,8 @@ public void testStructComplex() throws Exception { .addFields(16, TEST_NUMERIC_STR) .addFields(17, TEST_BIGNUMERIC) .addFields(18, TEST_BIGNUMERIC_STR) + .addFields(19, TEST_INTERVAL) + .addFields(20, TEST_JSON) .build(); final Descriptor descriptor = BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema); diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessageTest.java index 89848b0e61..5a84b5da47 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessageTest.java @@ -395,6 +395,18 @@ public class JsonToProtoMessageTest { .setMode(TableFieldSchema.Mode.REPEATED) .setName("test_bignumeric_str") .build(); + private final TableFieldSchema TEST_INTERVAL = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.INTERVAL) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_interval") + .build(); + private final TableFieldSchema TEST_JSON = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.JSON) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_json") + .build(); private final TableSchema COMPLEX_TABLE_SCHEMA = TableSchema.newBuilder() .addFields(0, TEST_INT) @@ -416,6 +428,8 @@ public class JsonToProtoMessageTest { .addFields(16, TEST_NUMERIC_STR) .addFields(17, TEST_BIGNUMERIC) .addFields(18, TEST_BIGNUMERIC_STR) + .addFields(19, TEST_INTERVAL) + .addFields(20, TEST_JSON) .build(); @Test @@ -744,6 +758,8 @@ public void testStructComplex() throws Exception { BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("2.3"))) .addTestBignumericStr( BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23"))) + .setTestInterval("0-0 0 0:0:0.000005") + .addTestJson("{'a':'b'}") .build(); JSONObject complex_lvl2 = new JSONObject(); complex_lvl2.put("test_int", 3); @@ -790,6 +806,8 @@ public void testStructComplex() throws Exception { "test_bignumeric", BigDecimalByteStringEncoder.encodeToNumericByteString(BigDecimal.valueOf(2.3))); json.put("test_bignumeric_str", new JSONArray(new String[] {"1.23"})); + json.put("test_interval", "0-0 0 0:0:0.000005"); + json.put("test_json", new JSONArray(new String[] {"{'a':'b'}"})); DynamicMessage protoMsg = JsonToProtoMessage.convertJsonToProtoMessage( ComplexRoot.getDescriptor(), COMPLEX_TABLE_SCHEMA, json); diff --git a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto index b929604004..c08dd7648f 100644 --- a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto +++ b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto @@ -23,6 +23,8 @@ message ComplexRoot { optional bytes test_numeric_str = 17; optional bytes test_bignumeric = 18; repeated bytes test_bignumeric_str = 19; + optional string test_interval = 20; + repeated string test_json = 21; } message CasingComplex {