From 691f078f09e32fa7784d4afeeee0e8071f78d6fd Mon Sep 17 00:00:00 2001 From: Yiru Tang Date: Fri, 1 Oct 2021 12:09:13 -0700 Subject: [PATCH] fix: JsonWriter accepts string input for DATETIME, TIME, NUMERIC, BIGNUMERIC field (#1339) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: update code comment to reflect max size change * fix: JsonWriter support string DateTime, Time, Numeric, BigNumeric * . * . * fix format * remove a test that is covered by JsonToProtoMessageTest * . * . * . * remove v1 test that is failing due to test proto update, test coverage will be added when the additional type support is ported to v1 * . * . * . * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- .../storage/v1beta2/JsonStreamWriter.java | 4 +- .../storage/v1beta2/JsonToProtoMessage.java | 197 +++++++++--- .../BQTableSchemaToProtoDescriptorTest.java | 56 +++- .../storage/v1/JsonStreamWriterTest.java | 199 ------------- .../BQTableSchemaToProtoDescriptorTest.java | 56 +++- .../storage/v1beta2/JsonStreamWriterTest.java | 249 ++++------------ .../v1beta2/JsonToProtoMessageTest.java | 280 +++++++++++++++++- .../src/test/proto/jsonTest.proto | 37 ++- 8 files changed, 624 insertions(+), 454 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriter.java index 64687f8732..5837e5c4a5 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriter.java @@ -75,6 +75,7 @@ private JsonStreamWriter(Builder builder) builder.traceId); this.streamWriter = streamWriterBuilder.build(); this.streamName = builder.streamName; + this.tableSchema = builder.tableSchema; } /** @@ -105,7 +106,8 @@ public ApiFuture append(JSONArray jsonArr, long offset) { // of JSON data. for (int i = 0; i < jsonArr.length(); i++) { JSONObject json = jsonArr.getJSONObject(i); - Message protoMessage = JsonToProtoMessage.convertJsonToProtoMessage(this.descriptor, json); + Message protoMessage = + JsonToProtoMessage.convertJsonToProtoMessage(this.descriptor, this.tableSchema, json); rowsBuilder.addSerializedRows(protoMessage.toByteString()); } // Need to make sure refreshAppendAndSetDescriptor finish first before this can run diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessage.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessage.java index 5077d7e7d2..5913306b08 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessage.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessage.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigquery.storage.v1beta2; +import com.google.api.pathtemplate.ValidationException; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.protobuf.ByteString; @@ -23,10 +24,14 @@ import com.google.protobuf.DynamicMessage; import com.google.protobuf.Message; import com.google.protobuf.UninitializedMessageException; +import java.math.BigDecimal; +import java.util.List; import java.util.logging.Logger; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import org.threeten.bp.LocalDateTime; +import org.threeten.bp.LocalTime; /** * Converts Json data to protocol buffer messages given the protocol buffer descriptor. The protobuf @@ -58,7 +63,28 @@ public static DynamicMessage convertJsonToProtoMessage(Descriptor protoSchema, J Preconditions.checkNotNull(protoSchema, "Protobuf descriptor is null."); Preconditions.checkState(json.length() != 0, "JSONObject is empty."); - return convertJsonToProtoMessageImpl(protoSchema, json, "root", /*topLevel=*/ true); + return convertJsonToProtoMessageImpl(protoSchema, null, json, "root", /*topLevel=*/ true); + } + + /** + * Converts Json data to protocol buffer messages given the protocol buffer descriptor. + * + * @param protoSchema + * @param tableSchema bigquery table schema is needed for type conversion of DATETIME, TIME, + * NUMERIC, BIGNUMERIC + * @param json + * @throws IllegalArgumentException when JSON data is not compatible with proto descriptor. + */ + public static DynamicMessage convertJsonToProtoMessage( + Descriptor protoSchema, TableSchema tableSchema, JSONObject json) + throws IllegalArgumentException { + Preconditions.checkNotNull(json, "JSONObject is null."); + Preconditions.checkNotNull(protoSchema, "Protobuf descriptor is null."); + Preconditions.checkNotNull(tableSchema, "TableSchema is null."); + Preconditions.checkState(json.length() != 0, "JSONObject is empty."); + + return convertJsonToProtoMessageImpl( + protoSchema, tableSchema.getFieldsList(), json, "root", /*topLevel=*/ true); } /** @@ -71,9 +97,12 @@ public static DynamicMessage convertJsonToProtoMessage(Descriptor protoSchema, J * @throws IllegalArgumentException when JSON data is not compatible with proto descriptor. */ private static DynamicMessage convertJsonToProtoMessageImpl( - Descriptor protoSchema, JSONObject json, String jsonScope, boolean topLevel) + Descriptor protoSchema, + List tableSchema, + JSONObject json, + String jsonScope, + boolean topLevel) throws IllegalArgumentException { - DynamicMessage.Builder protoMsg = DynamicMessage.newBuilder(protoSchema); String[] jsonNames = JSONObject.getNames(json); if (jsonNames == null) { @@ -90,10 +119,25 @@ private static DynamicMessage convertJsonToProtoMessageImpl( throw new IllegalArgumentException( String.format("JSONObject has fields unknown to BigQuery: %s.", currentScope)); } + TableFieldSchema fieldSchema = null; + if (tableSchema != null) { + // protoSchema is generated from tableSchema so their field ordering should match. + fieldSchema = tableSchema.get(field.getIndex()); + if (!fieldSchema.getName().equals(field.getName())) { + throw new ValidationException( + "Field at index " + + field.getIndex() + + " has mismatch names (" + + fieldSchema.getName() + + ") (" + + field.getName() + + ")"); + } + } if (!field.isRepeated()) { - fillField(protoMsg, field, json, jsonName, currentScope); + fillField(protoMsg, field, fieldSchema, json, jsonName, currentScope); } else { - fillRepeatedField(protoMsg, field, json, jsonName, currentScope); + fillRepeatedField(protoMsg, field, fieldSchema, json, jsonName, currentScope); } } @@ -127,6 +171,7 @@ private static DynamicMessage convertJsonToProtoMessageImpl( private static void fillField( DynamicMessage.Builder protoMsg, FieldDescriptor fieldDescriptor, + TableFieldSchema fieldSchema, JSONObject json, String exactJsonKeyName, String currentScope) @@ -144,6 +189,25 @@ private static void fillField( } break; case BYTES: + if (fieldSchema != null) { + if (fieldSchema.getType() == TableFieldSchema.Type.NUMERIC) { + if (val instanceof String) { + protoMsg.setField( + fieldDescriptor, + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal((String) val))); + return; + } + } else if (fieldSchema.getType() == TableFieldSchema.Type.BIGNUMERIC) { + if (val instanceof String) { + protoMsg.setField( + fieldDescriptor, + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal((String) val))); + return; + } + } + } if (val instanceof ByteString) { protoMsg.setField(fieldDescriptor, ((ByteString) val).toByteArray()); return; @@ -170,6 +234,29 @@ private static void fillField( } break; case INT64: + if (fieldSchema != null) { + if (fieldSchema.getType() == TableFieldSchema.Type.DATETIME) { + if (val instanceof String) { + protoMsg.setField( + fieldDescriptor, + CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.parse((String) val))); + return; + } else if (val instanceof Long) { + protoMsg.setField(fieldDescriptor, (Long) val); + return; + } + } else if (fieldSchema.getType() == TableFieldSchema.Type.TIME) { + if (val instanceof String) { + protoMsg.setField( + fieldDescriptor, + CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.parse((String) val))); + return; + } else if (val instanceof Long) { + protoMsg.setField(fieldDescriptor, (Long) val); + return; + } + } + } if (val instanceof Integer) { protoMsg.setField(fieldDescriptor, new Long((Integer) val)); return; @@ -206,6 +293,7 @@ private static void fillField( fieldDescriptor, convertJsonToProtoMessageImpl( fieldDescriptor.getMessageType(), + fieldSchema == null ? null : fieldSchema.getFieldsList(), json.getJSONObject(exactJsonKeyName), currentScope, /*topLevel =*/ false)); @@ -232,6 +320,7 @@ private static void fillField( private static void fillRepeatedField( DynamicMessage.Builder protoMsg, FieldDescriptor fieldDescriptor, + TableFieldSchema fieldSchema, JSONObject json, String exactJsonKeyName, String currentScope) @@ -259,40 +348,81 @@ private static void fillRepeatedField( } break; case BYTES: - if (val instanceof JSONArray) { - try { - byte[] bytes = new byte[((JSONArray) val).length()]; - for (int j = 0; j < ((JSONArray) val).length(); j++) { - bytes[j] = (byte) ((JSONArray) val).getInt(j); - if (bytes[j] != ((JSONArray) val).getInt(j)) { - throw new IllegalArgumentException( - String.format( - "Error: " - + currentScope - + "[" - + index - + "] could not be converted to byte[].")); + Boolean added = false; + if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.NUMERIC) { + if (val instanceof String) { + protoMsg.addRepeatedField( + fieldDescriptor, + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal((String) val))); + added = true; + } + } else if (fieldSchema != null + && fieldSchema.getType() == TableFieldSchema.Type.BIGNUMERIC) { + if (val instanceof String) { + protoMsg.addRepeatedField( + fieldDescriptor, + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal((String) val))); + added = true; + } + } + if (!added) { + if (val instanceof JSONArray) { + try { + byte[] bytes = new byte[((JSONArray) val).length()]; + for (int j = 0; j < ((JSONArray) val).length(); j++) { + bytes[j] = (byte) ((JSONArray) val).getInt(j); + if (bytes[j] != ((JSONArray) val).getInt(j)) { + throw new IllegalArgumentException( + String.format( + "Error: " + + currentScope + + "[" + + index + + "] could not be converted to byte[].")); + } } + protoMsg.addRepeatedField(fieldDescriptor, bytes); + } catch (JSONException e) { + throw new IllegalArgumentException( + String.format( + "Error: " + + currentScope + + "[" + + index + + "] could not be converted to byte[].")); } - protoMsg.addRepeatedField(fieldDescriptor, bytes); - } catch (JSONException e) { - throw new IllegalArgumentException( - String.format( - "Error: " - + currentScope - + "[" - + index - + "] could not be converted to byte[].")); + } else if (val instanceof ByteString) { + protoMsg.addRepeatedField(fieldDescriptor, ((ByteString) val).toByteArray()); + return; + } else { + fail = true; } - } else if (val instanceof ByteString) { - protoMsg.addRepeatedField(fieldDescriptor, ((ByteString) val).toByteArray()); - return; - } else { - fail = true; } break; case INT64: - if (val instanceof Integer) { + if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.DATETIME) { + if (val instanceof String) { + protoMsg.addRepeatedField( + fieldDescriptor, + CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.parse((String) val))); + } else if (val instanceof Long) { + protoMsg.addRepeatedField(fieldDescriptor, (Long) val); + } else { + fail = true; + } + } else if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.TIME) { + if (val instanceof String) { + protoMsg.addRepeatedField( + fieldDescriptor, + CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.parse((String) val))); + } else if (val instanceof Long) { + protoMsg.addRepeatedField(fieldDescriptor, (Long) val); + } else { + fail = true; + } + } else if (val instanceof Integer) { protoMsg.addRepeatedField(fieldDescriptor, new Long((Integer) val)); } else if (val instanceof Long) { protoMsg.addRepeatedField(fieldDescriptor, (Long) val); @@ -330,6 +460,7 @@ private static void fillRepeatedField( fieldDescriptor, convertJsonToProtoMessageImpl( fieldDescriptor.getMessageType(), + fieldSchema == null ? null : fieldSchema.getFieldsList(), jsonArray.getJSONObject(i), currentScope, /*topLevel =*/ false)); diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java index 07cb1c8657..3ef5f87d02 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java @@ -164,6 +164,18 @@ public void testStructComplex() throws Exception { .setMode(TableFieldSchema.Mode.REQUIRED) .setName("test_date") .build(); + final TableFieldSchema test_datetime = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.DATETIME) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_datetime") + .build(); + final TableFieldSchema test_datetime_str = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.DATETIME) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_datetime_str") + .build(); final TableFieldSchema ComplexLvl2 = TableFieldSchema.newBuilder() .setType(TableFieldSchema.Type.STRUCT) @@ -203,12 +215,36 @@ public void testStructComplex() throws Exception { .setMode(TableFieldSchema.Mode.NULLABLE) .setName("test_time") .build(); + final TableFieldSchema TEST_TIME_STR = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.TIME) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_time_str") + .build(); final TableFieldSchema TEST_NUMERIC_REPEATED = TableFieldSchema.newBuilder() .setType(TableFieldSchema.Type.NUMERIC) .setMode(TableFieldSchema.Mode.REPEATED) .setName("test_numeric_repeated") .build(); + final TableFieldSchema TEST_NUMERIC_STR = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_numeric_str") + .build(); + final TableFieldSchema TEST_BIGNUMERIC = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_bignumeric") + .build(); + final TableFieldSchema TEST_BIGNUMERIC_STR = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_bignumeric_str") + .build(); final TableSchema tableSchema = TableSchema.newBuilder() .addFields(0, test_int) @@ -217,13 +253,19 @@ public void testStructComplex() throws Exception { .addFields(3, test_bool) .addFields(4, test_double) .addFields(5, test_date) - .addFields(6, ComplexLvl1) - .addFields(7, ComplexLvl2) - .addFields(8, TEST_NUMERIC) - .addFields(9, TEST_GEO) - .addFields(10, TEST_TIMESTAMP) - .addFields(11, TEST_TIME) - .addFields(12, TEST_NUMERIC_REPEATED) + .addFields(6, test_datetime) + .addFields(7, test_datetime_str) + .addFields(8, ComplexLvl1) + .addFields(9, ComplexLvl2) + .addFields(10, TEST_NUMERIC) + .addFields(11, TEST_GEO) + .addFields(12, TEST_TIMESTAMP) + .addFields(13, TEST_TIME) + .addFields(14, TEST_TIME_STR) + .addFields(15, TEST_NUMERIC_REPEATED) + .addFields(16, TEST_NUMERIC_STR) + .addFields(17, TEST_BIGNUMERIC) + .addFields(18, TEST_BIGNUMERIC_STR) .build(); final Descriptor descriptor = BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema); diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java index ea9e7e6f4c..b688b7542a 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java @@ -24,14 +24,11 @@ import com.google.api.gax.grpc.testing.LocalChannelProvider; import com.google.api.gax.grpc.testing.MockGrpcService; import com.google.api.gax.grpc.testing.MockServiceHelper; -import com.google.cloud.bigquery.storage.test.JsonTest.ComplexRoot; import com.google.cloud.bigquery.storage.test.Test.FooType; -import com.google.protobuf.ByteString; import com.google.protobuf.Descriptors.DescriptorValidationException; import com.google.protobuf.Int64Value; import com.google.protobuf.Timestamp; import java.io.IOException; -import java.math.BigDecimal; import java.util.Arrays; import java.util.UUID; import java.util.concurrent.ExecutionException; @@ -45,7 +42,6 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; import org.threeten.bp.Instant; -import org.threeten.bp.LocalTime; @RunWith(JUnit4.class) public class JsonStreamWriterTest { @@ -96,91 +92,6 @@ public class JsonStreamWriterTest { .setMode(TableFieldSchema.Mode.REPEATED) .setName("test_string") .build(); - private final TableFieldSchema TEST_BYTES = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.BYTES) - .setMode(TableFieldSchema.Mode.REQUIRED) - .setName("test_bytes") - .build(); - private final TableFieldSchema TEST_BOOL = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.BOOL) - .setMode(TableFieldSchema.Mode.NULLABLE) - .setName("test_bool") - .build(); - private final TableFieldSchema TEST_DOUBLE = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.DOUBLE) - .setMode(TableFieldSchema.Mode.REPEATED) - .setName("test_double") - .build(); - private final TableFieldSchema TEST_DATE = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.DATE) - .setMode(TableFieldSchema.Mode.REQUIRED) - .setName("test_date") - .build(); - private final TableFieldSchema COMPLEXLVL2 = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.STRUCT) - .setMode(TableFieldSchema.Mode.REQUIRED) - .addFields(0, TEST_INT) - .setName("complex_lvl2") - .build(); - private final TableFieldSchema COMPLEXLVL1 = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.STRUCT) - .setMode(TableFieldSchema.Mode.REQUIRED) - .addFields(0, TEST_INT) - .addFields(1, COMPLEXLVL2) - .setName("complex_lvl1") - .build(); - private final TableFieldSchema TEST_NUMERIC = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.NUMERIC) - .setMode(TableFieldSchema.Mode.NULLABLE) - .setName("test_numeric") - .build(); - private final TableFieldSchema TEST_NUMERIC_REPEATED = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.NUMERIC) - .setMode(TableFieldSchema.Mode.REPEATED) - .setName("test_numeric_repeated") - .build(); - private final TableFieldSchema TEST_GEO = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.GEOGRAPHY) - .setMode(TableFieldSchema.Mode.NULLABLE) - .setName("test_geo") - .build(); - private final TableFieldSchema TEST_TIMESTAMP = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.TIMESTAMP) - .setMode(TableFieldSchema.Mode.NULLABLE) - .setName("test_timestamp") - .build(); - private final TableFieldSchema TEST_TIME = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.TIME) - .setMode(TableFieldSchema.Mode.NULLABLE) - .setName("test_time") - .build(); - private final TableSchema COMPLEX_TABLE_SCHEMA = - TableSchema.newBuilder() - .addFields(0, TEST_INT) - .addFields(1, TEST_STRING) - .addFields(2, TEST_BYTES) - .addFields(3, TEST_BOOL) - .addFields(4, TEST_DOUBLE) - .addFields(5, TEST_DATE) - .addFields(6, COMPLEXLVL1) - .addFields(7, COMPLEXLVL2) - .addFields(8, TEST_NUMERIC) - .addFields(9, TEST_GEO) - .addFields(10, TEST_TIMESTAMP) - .addFields(11, TEST_TIME) - .addFields(12, TEST_NUMERIC_REPEATED) - .build(); @Before public void setUp() throws Exception { @@ -391,116 +302,6 @@ public void testMultipleAppendSimpleJson() throws Exception { } } - @Test - public void testSingleAppendComplexJson() throws Exception { - ComplexRoot expectedProto = - ComplexRoot.newBuilder() - .setTestInt(1) - .addTestString("a") - .addTestString("b") - .addTestString("c") - .setTestBytes(ByteString.copyFrom("hello".getBytes())) - .setTestBool(true) - .addTestDouble(1.1) - .addTestDouble(2.2) - .addTestDouble(3.3) - .addTestDouble(4.4) - .setTestDate(1) - .setComplexLvl1( - com.google.cloud.bigquery.storage.test.JsonTest.ComplexLvl1.newBuilder() - .setTestInt(2) - .setComplexLvl2( - com.google.cloud.bigquery.storage.test.JsonTest.ComplexLvl2.newBuilder() - .setTestInt(3) - .build()) - .build()) - .setComplexLvl2( - com.google.cloud.bigquery.storage.test.JsonTest.ComplexLvl2.newBuilder() - .setTestInt(3) - .build()) - .setTestNumeric( - BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23456"))) - .setTestGeo("POINT(1,1)") - .setTestTimestamp(12345678) - .setTestTime(CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1))) - .addTestNumericRepeated( - BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0"))) - .addTestNumericRepeated( - BigDecimalByteStringEncoder.encodeToNumericByteString( - new BigDecimal("99999999999999999999999999999.999999999"))) - .addTestNumericRepeated( - BigDecimalByteStringEncoder.encodeToNumericByteString( - new BigDecimal("-99999999999999999999999999999.999999999"))) - .build(); - JSONObject complex_lvl2 = new JSONObject(); - complex_lvl2.put("test_int", 3); - - JSONObject complex_lvl1 = new JSONObject(); - complex_lvl1.put("test_int", 2); - complex_lvl1.put("complex_lvl2", complex_lvl2); - - JSONObject json = new JSONObject(); - json.put("test_int", 1); - json.put("test_string", new JSONArray(new String[] {"a", "b", "c"})); - json.put("test_bytes", ByteString.copyFrom("hello".getBytes())); - json.put("test_bool", true); - json.put("test_DOUBLe", new JSONArray(new Double[] {1.1, 2.2, 3.3, 4.4})); - json.put("test_date", 1); - json.put("complex_lvl1", complex_lvl1); - json.put("complex_lvl2", complex_lvl2); - json.put( - "test_numeric", - BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23456"))); - json.put( - "test_numeric_repeated", - new JSONArray( - new byte[][] { - BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0")) - .toByteArray(), - BigDecimalByteStringEncoder.encodeToNumericByteString( - new BigDecimal("99999999999999999999999999999.999999999")) - .toByteArray(), - BigDecimalByteStringEncoder.encodeToNumericByteString( - new BigDecimal("-99999999999999999999999999999.999999999")) - .toByteArray(), - })); - json.put("test_geo", "POINT(1,1)"); - json.put("test_timestamp", 12345678); - json.put("test_time", CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1))); - JSONArray jsonArr = new JSONArray(); - jsonArr.put(json); - - try (JsonStreamWriter writer = - getTestJsonStreamWriterBuilder(TEST_STREAM, COMPLEX_TABLE_SCHEMA).build()) { - testBigQueryWrite.addResponse( - AppendRowsResponse.newBuilder() - .setAppendResult( - AppendRowsResponse.AppendResult.newBuilder().setOffset(Int64Value.of(0)).build()) - .build()); - - ApiFuture appendFuture = writer.append(jsonArr); - - assertEquals(0L, appendFuture.get().getAppendResult().getOffset().getValue()); - appendFuture.get(); - assertEquals( - 1, - testBigQueryWrite - .getAppendRequests() - .get(0) - .getProtoRows() - .getRows() - .getSerializedRowsCount()); - assertEquals( - testBigQueryWrite - .getAppendRequests() - .get(0) - .getProtoRows() - .getRows() - .getSerializedRows(0), - expectedProto.toByteString()); - } - } - @Test public void testAppendOutOfRangeException() throws Exception { try (JsonStreamWriter writer = diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java index 29b065f44b..c6e0973d58 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java @@ -164,6 +164,18 @@ public void testStructComplex() throws Exception { .setMode(TableFieldSchema.Mode.REQUIRED) .setName("test_date") .build(); + final TableFieldSchema test_datetime = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.DATETIME) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_datetime") + .build(); + final TableFieldSchema test_datetime_str = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.DATETIME) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_datetime_str") + .build(); final TableFieldSchema ComplexLvl2 = TableFieldSchema.newBuilder() .setType(TableFieldSchema.Type.STRUCT) @@ -203,12 +215,36 @@ public void testStructComplex() throws Exception { .setMode(TableFieldSchema.Mode.NULLABLE) .setName("test_time") .build(); + final TableFieldSchema TEST_TIME_STR = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.TIME) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_time_str") + .build(); final TableFieldSchema TEST_NUMERIC_REPEATED = TableFieldSchema.newBuilder() .setType(TableFieldSchema.Type.NUMERIC) .setMode(TableFieldSchema.Mode.REPEATED) .setName("test_numeric_repeated") .build(); + final TableFieldSchema TEST_NUMERIC_STR = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_numeric_str") + .build(); + final TableFieldSchema TEST_BIGNUMERIC = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_bignumeric") + .build(); + final TableFieldSchema TEST_BIGNUMERIC_STR = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_bignumeric_str") + .build(); final TableSchema tableSchema = TableSchema.newBuilder() .addFields(0, test_int) @@ -217,13 +253,19 @@ public void testStructComplex() throws Exception { .addFields(3, test_bool) .addFields(4, test_double) .addFields(5, test_date) - .addFields(6, ComplexLvl1) - .addFields(7, ComplexLvl2) - .addFields(8, TEST_NUMERIC) - .addFields(9, TEST_GEO) - .addFields(10, TEST_TIMESTAMP) - .addFields(11, TEST_TIME) - .addFields(12, TEST_NUMERIC_REPEATED) + .addFields(6, test_datetime) + .addFields(7, test_datetime_str) + .addFields(8, ComplexLvl1) + .addFields(9, ComplexLvl2) + .addFields(10, TEST_NUMERIC) + .addFields(11, TEST_GEO) + .addFields(12, TEST_TIMESTAMP) + .addFields(13, TEST_TIME) + .addFields(14, TEST_TIME_STR) + .addFields(15, TEST_NUMERIC_REPEATED) + .addFields(16, TEST_NUMERIC_STR) + .addFields(17, TEST_BIGNUMERIC) + .addFields(18, TEST_BIGNUMERIC_STR) .build(); final Descriptor descriptor = BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema); diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriterTest.java index dbc7460163..85c83dbf25 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriterTest.java @@ -24,14 +24,12 @@ import com.google.api.gax.grpc.testing.LocalChannelProvider; import com.google.api.gax.grpc.testing.MockGrpcService; import com.google.api.gax.grpc.testing.MockServiceHelper; -import com.google.cloud.bigquery.storage.test.JsonTest.ComplexRoot; +import com.google.cloud.bigquery.storage.test.JsonTest; import com.google.cloud.bigquery.storage.test.Test.FooType; -import com.google.protobuf.ByteString; import com.google.protobuf.Descriptors.DescriptorValidationException; import com.google.protobuf.Int64Value; import com.google.protobuf.Timestamp; import java.io.IOException; -import java.math.BigDecimal; import java.util.Arrays; import java.util.UUID; import java.util.concurrent.ExecutionException; @@ -96,91 +94,6 @@ public class JsonStreamWriterTest { .setMode(TableFieldSchema.Mode.REPEATED) .setName("test_string") .build(); - private final TableFieldSchema TEST_BYTES = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.BYTES) - .setMode(TableFieldSchema.Mode.REQUIRED) - .setName("test_bytes") - .build(); - private final TableFieldSchema TEST_BOOL = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.BOOL) - .setMode(TableFieldSchema.Mode.NULLABLE) - .setName("test_bool") - .build(); - private final TableFieldSchema TEST_DOUBLE = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.DOUBLE) - .setMode(TableFieldSchema.Mode.REPEATED) - .setName("test_double") - .build(); - private final TableFieldSchema TEST_DATE = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.DATE) - .setMode(TableFieldSchema.Mode.REQUIRED) - .setName("test_date") - .build(); - private final TableFieldSchema COMPLEXLVL2 = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.STRUCT) - .setMode(TableFieldSchema.Mode.REQUIRED) - .addFields(0, TEST_INT) - .setName("complex_lvl2") - .build(); - private final TableFieldSchema COMPLEXLVL1 = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.STRUCT) - .setMode(TableFieldSchema.Mode.REQUIRED) - .addFields(0, TEST_INT) - .addFields(1, COMPLEXLVL2) - .setName("complex_lvl1") - .build(); - private final TableFieldSchema TEST_NUMERIC = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.NUMERIC) - .setMode(TableFieldSchema.Mode.NULLABLE) - .setName("test_numeric") - .build(); - private final TableFieldSchema TEST_NUMERIC_REPEATED = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.NUMERIC) - .setMode(TableFieldSchema.Mode.REPEATED) - .setName("test_numeric_repeated") - .build(); - private final TableFieldSchema TEST_GEO = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.GEOGRAPHY) - .setMode(TableFieldSchema.Mode.NULLABLE) - .setName("test_geo") - .build(); - private final TableFieldSchema TEST_TIMESTAMP = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.TIMESTAMP) - .setMode(TableFieldSchema.Mode.NULLABLE) - .setName("test_timestamp") - .build(); - private final TableFieldSchema TEST_TIME = - TableFieldSchema.newBuilder() - .setType(TableFieldSchema.Type.TIME) - .setMode(TableFieldSchema.Mode.NULLABLE) - .setName("test_time") - .build(); - private final TableSchema COMPLEX_TABLE_SCHEMA = - TableSchema.newBuilder() - .addFields(0, TEST_INT) - .addFields(1, TEST_STRING) - .addFields(2, TEST_BYTES) - .addFields(3, TEST_BOOL) - .addFields(4, TEST_DOUBLE) - .addFields(5, TEST_DATE) - .addFields(6, COMPLEXLVL1) - .addFields(7, COMPLEXLVL2) - .addFields(8, TEST_NUMERIC) - .addFields(9, TEST_GEO) - .addFields(10, TEST_TIMESTAMP) - .addFields(11, TEST_TIME) - .addFields(12, TEST_NUMERIC_REPEATED) - .build(); @Before public void setUp() throws Exception { @@ -284,6 +197,56 @@ public void testSingleAppendSimpleJson() throws Exception { } } + @Test + public void testSpecialTypeAppend() throws Exception { + TableFieldSchema field = + TableFieldSchema.newBuilder() + .setName("time") + .setType(TableFieldSchema.Type.TIME) + .setMode(TableFieldSchema.Mode.REPEATED) + .build(); + TableSchema tableSchema = TableSchema.newBuilder().addFields(field).build(); + + JsonTest.TestTime expectedProto = + JsonTest.TestTime.newBuilder() + .addTime(CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1))) + .build(); + JSONObject foo = new JSONObject(); + foo.put("time", new JSONArray(new String[] {"01:00:01"})); + JSONArray jsonArr = new JSONArray(); + jsonArr.put(foo); + + try (JsonStreamWriter writer = + getTestJsonStreamWriterBuilder(TEST_STREAM, tableSchema).build()) { + + testBigQueryWrite.addResponse( + AppendRowsResponse.newBuilder() + .setAppendResult( + AppendRowsResponse.AppendResult.newBuilder().setOffset(Int64Value.of(0)).build()) + .build()); + + ApiFuture appendFuture = writer.append(jsonArr); + assertEquals(0L, appendFuture.get().getAppendResult().getOffset().getValue()); + appendFuture.get(); + assertEquals( + 1, + testBigQueryWrite + .getAppendRequests() + .get(0) + .getProtoRows() + .getRows() + .getSerializedRowsCount()); + assertEquals( + testBigQueryWrite + .getAppendRequests() + .get(0) + .getProtoRows() + .getRows() + .getSerializedRows(0), + expectedProto.toByteString()); + } + } + @Test public void testSingleAppendMultipleSimpleJson() throws Exception { FooType expectedProto = FooType.newBuilder().setFoo("allen").build(); @@ -391,116 +354,6 @@ public void testMultipleAppendSimpleJson() throws Exception { } } - @Test - public void testSingleAppendComplexJson() throws Exception { - ComplexRoot expectedProto = - ComplexRoot.newBuilder() - .setTestInt(1) - .addTestString("a") - .addTestString("b") - .addTestString("c") - .setTestBytes(ByteString.copyFrom("hello".getBytes())) - .setTestBool(true) - .addTestDouble(1.1) - .addTestDouble(2.2) - .addTestDouble(3.3) - .addTestDouble(4.4) - .setTestDate(1) - .setComplexLvl1( - com.google.cloud.bigquery.storage.test.JsonTest.ComplexLvl1.newBuilder() - .setTestInt(2) - .setComplexLvl2( - com.google.cloud.bigquery.storage.test.JsonTest.ComplexLvl2.newBuilder() - .setTestInt(3) - .build()) - .build()) - .setComplexLvl2( - com.google.cloud.bigquery.storage.test.JsonTest.ComplexLvl2.newBuilder() - .setTestInt(3) - .build()) - .setTestNumeric( - BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23456"))) - .setTestGeo("POINT(1,1)") - .setTestTimestamp(12345678) - .setTestTime(CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1))) - .addTestNumericRepeated( - BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0"))) - .addTestNumericRepeated( - BigDecimalByteStringEncoder.encodeToNumericByteString( - new BigDecimal("99999999999999999999999999999.999999999"))) - .addTestNumericRepeated( - BigDecimalByteStringEncoder.encodeToNumericByteString( - new BigDecimal("-99999999999999999999999999999.999999999"))) - .build(); - JSONObject complex_lvl2 = new JSONObject(); - complex_lvl2.put("test_int", 3); - - JSONObject complex_lvl1 = new JSONObject(); - complex_lvl1.put("test_int", 2); - complex_lvl1.put("complex_lvl2", complex_lvl2); - - JSONObject json = new JSONObject(); - json.put("test_int", 1); - json.put("test_string", new JSONArray(new String[] {"a", "b", "c"})); - json.put("test_bytes", ByteString.copyFrom("hello".getBytes())); - json.put("test_bool", true); - json.put("test_DOUBLe", new JSONArray(new Double[] {1.1, 2.2, 3.3, 4.4})); - json.put("test_date", 1); - json.put("complex_lvl1", complex_lvl1); - json.put("complex_lvl2", complex_lvl2); - json.put( - "test_numeric", - BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23456"))); - json.put( - "test_numeric_repeated", - new JSONArray( - new byte[][] { - BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0")) - .toByteArray(), - BigDecimalByteStringEncoder.encodeToNumericByteString( - new BigDecimal("99999999999999999999999999999.999999999")) - .toByteArray(), - BigDecimalByteStringEncoder.encodeToNumericByteString( - new BigDecimal("-99999999999999999999999999999.999999999")) - .toByteArray(), - })); - json.put("test_geo", "POINT(1,1)"); - json.put("test_timestamp", 12345678); - json.put("test_time", CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1))); - JSONArray jsonArr = new JSONArray(); - jsonArr.put(json); - - try (JsonStreamWriter writer = - getTestJsonStreamWriterBuilder(TEST_STREAM, COMPLEX_TABLE_SCHEMA).build()) { - testBigQueryWrite.addResponse( - AppendRowsResponse.newBuilder() - .setAppendResult( - AppendRowsResponse.AppendResult.newBuilder().setOffset(Int64Value.of(0)).build()) - .build()); - - ApiFuture appendFuture = writer.append(jsonArr); - - assertEquals(0L, appendFuture.get().getAppendResult().getOffset().getValue()); - appendFuture.get(); - assertEquals( - 1, - testBigQueryWrite - .getAppendRequests() - .get(0) - .getProtoRows() - .getRows() - .getSerializedRowsCount()); - assertEquals( - testBigQueryWrite - .getAppendRequests() - .get(0) - .getProtoRows() - .getRows() - .getSerializedRows(0), - expectedProto.toByteString()); - } - } - @Test public void testAppendOutOfRangeException() throws Exception { try (JsonStreamWriter writer = diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessageTest.java index 74097a661b..89848b0e61 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessageTest.java @@ -35,6 +35,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +import org.threeten.bp.LocalTime; @RunWith(JUnit4.class) public class JsonToProtoMessageTest { @@ -277,6 +278,146 @@ public class JsonToProtoMessageTest { })) }; + private final TableFieldSchema TEST_INT = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.INT64) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_int") + .build(); + private final TableFieldSchema TEST_STRING = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.STRING) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_string") + .build(); + private final TableFieldSchema TEST_BYTES = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.BYTES) + .setMode(TableFieldSchema.Mode.REQUIRED) + .setName("test_bytes") + .build(); + private final TableFieldSchema TEST_BOOL = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.BOOL) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_bool") + .build(); + private final TableFieldSchema TEST_DOUBLE = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.DOUBLE) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_double") + .build(); + private final TableFieldSchema TEST_DATE = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.DATE) + .setMode(TableFieldSchema.Mode.REQUIRED) + .setName("test_date") + .build(); + private final TableFieldSchema TEST_DATETIME = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.DATETIME) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_datetime") + .build(); + private final TableFieldSchema TEST_DATETIME_STR = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.DATETIME) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_datetime_str") + .build(); + private final TableFieldSchema COMPLEXLVL2 = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.STRUCT) + .setMode(TableFieldSchema.Mode.REQUIRED) + .addFields(0, TEST_INT) + .setName("complex_lvl2") + .build(); + private final TableFieldSchema COMPLEXLVL1 = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.STRUCT) + .setMode(TableFieldSchema.Mode.REQUIRED) + .addFields(0, TEST_INT) + .addFields(1, COMPLEXLVL2) + .setName("complex_lvl1") + .build(); + private final TableFieldSchema TEST_NUMERIC = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_numeric") + .build(); + private final TableFieldSchema TEST_NUMERIC_REPEATED = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_numeric_repeated") + .build(); + private final TableFieldSchema TEST_GEO = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.GEOGRAPHY) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_geo") + .build(); + private final TableFieldSchema TEST_TIMESTAMP = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.TIMESTAMP) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_timestamp") + .build(); + private final TableFieldSchema TEST_TIME = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.TIME) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_time") + .build(); + private final TableFieldSchema TEST_TIME_STR = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.TIME) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_time_str") + .build(); + private final TableFieldSchema TEST_NUMERIC_STR = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_numeric_str") + .build(); + private final TableFieldSchema TEST_BIGNUMERIC = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_bignumeric") + .build(); + private final TableFieldSchema TEST_BIGNUMERIC_STR = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_bignumeric_str") + .build(); + private final TableSchema COMPLEX_TABLE_SCHEMA = + TableSchema.newBuilder() + .addFields(0, TEST_INT) + .addFields(1, TEST_STRING) + .addFields(2, TEST_BYTES) + .addFields(3, TEST_BOOL) + .addFields(4, TEST_DOUBLE) + .addFields(5, TEST_DATE) + .addFields(6, TEST_DATETIME) + .addFields(7, TEST_DATETIME_STR) + .addFields(8, COMPLEXLVL1) + .addFields(9, COMPLEXLVL2) + .addFields(10, TEST_NUMERIC) + .addFields(11, TEST_GEO) + .addFields(12, TEST_TIMESTAMP) + .addFields(13, TEST_TIME) + .addFields(14, TEST_TIME_STR) + .addFields(15, TEST_NUMERIC_REPEATED) + .addFields(16, TEST_NUMERIC_STR) + .addFields(17, TEST_BIGNUMERIC) + .addFields(18, TEST_BIGNUMERIC_STR) + .build(); + @Test public void testDifferentNameCasing() throws Exception { TestInt64 expectedProto = @@ -333,6 +474,89 @@ public void testInt32NotMatchInt64() throws Exception { } } + @Test + public void testDateTimeMismatch() throws Exception { + TableFieldSchema field = + TableFieldSchema.newBuilder() + .setName("datetime") + .setType(TableFieldSchema.Type.DATETIME) + .setMode(TableFieldSchema.Mode.REPEATED) + .build(); + TableSchema tableSchema = TableSchema.newBuilder().addFields(field).build(); + JSONObject json = new JSONObject(); + json.put("datetime", 1.0); + try { + DynamicMessage protoMsg = + JsonToProtoMessage.convertJsonToProtoMessage( + TestDatetime.getDescriptor(), tableSchema, json); + Assert.fail("should fail"); + } catch (IllegalArgumentException e) { + assertEquals("JSONObject does not have a int64 field at root.datetime.", e.getMessage()); + } + } + + @Test + public void testTimeMismatch() throws Exception { + TableFieldSchema field = + TableFieldSchema.newBuilder() + .setName("time") + .setType(TableFieldSchema.Type.TIME) + .setMode(TableFieldSchema.Mode.REPEATED) + .build(); + TableSchema tableSchema = TableSchema.newBuilder().addFields(field).build(); + JSONObject json = new JSONObject(); + json.put("time", new JSONArray(new Double[] {1.0})); + try { + DynamicMessage protoMsg = + JsonToProtoMessage.convertJsonToProtoMessage(TestTime.getDescriptor(), tableSchema, json); + Assert.fail("should fail"); + } catch (IllegalArgumentException e) { + assertEquals("JSONObject does not have a int64 field at root.time[0].", e.getMessage()); + } + } + + @Test + public void testNumericMismatch() throws Exception { + TableFieldSchema field = + TableFieldSchema.newBuilder() + .setName("numeric") + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.NULLABLE) + .build(); + TableSchema tableSchema = TableSchema.newBuilder().addFields(field).build(); + JSONObject json = new JSONObject(); + json.put("numeric", 1.0); + try { + DynamicMessage protoMsg = + JsonToProtoMessage.convertJsonToProtoMessage( + TestNumeric.getDescriptor(), tableSchema, json); + Assert.fail("should fail"); + } catch (IllegalArgumentException e) { + assertEquals("JSONObject does not have a bytes field at root.numeric.", e.getMessage()); + } + } + + @Test + public void testBigNumericMismatch() throws Exception { + TableFieldSchema field = + TableFieldSchema.newBuilder() + .setName("bignumeric") + .setType(TableFieldSchema.Type.BIGNUMERIC) + .setMode(TableFieldSchema.Mode.REPEATED) + .build(); + TableSchema tableSchema = TableSchema.newBuilder().addFields(field).build(); + JSONObject json = new JSONObject(); + json.put("bignumeric", new JSONArray(new Double[] {1.0})); + try { + DynamicMessage protoMsg = + JsonToProtoMessage.convertJsonToProtoMessage( + TestBignumeric.getDescriptor(), tableSchema, json); + Assert.fail("should fail"); + } catch (IllegalArgumentException e) { + assertEquals("JSONObject does not have a bytes field at root.bignumeric[0].", e.getMessage()); + } + } + @Test public void testDouble() throws Exception { TestDouble expectedProto = TestDouble.newBuilder().setDouble(1.2).setFloat(3.4f).build(); @@ -491,12 +715,35 @@ public void testStructComplex() throws Exception { .addTestDouble(3.3) .addTestDouble(4.4) .setTestDate(1) + .setTestDatetime(1) + .addTestDatetimeStr(142258614586538368L) + .addTestDatetimeStr(142258525253402624L) .setComplexLvl1( ComplexLvl1.newBuilder() .setTestInt(2) .setComplexLvl2(ComplexLvl2.newBuilder().setTestInt(3).build()) .build()) .setComplexLvl2(ComplexLvl2.newBuilder().setTestInt(3).build()) + .setTestNumeric( + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23456"))) + .setTestGeo("POINT(1,1)") + .setTestTimestamp(12345678) + .setTestTime(CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1))) + .setTestTimeStr(89332507144L) + .addTestNumericRepeated( + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0"))) + .addTestNumericRepeated( + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal("99999999999999999999999999999.999999999"))) + .addTestNumericRepeated( + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal("-99999999999999999999999999999.999999999"))) + .setTestNumericStr( + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("12.4"))) + .setTestBignumeric( + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("2.3"))) + .addTestBignumericStr( + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23"))) .build(); JSONObject complex_lvl2 = new JSONObject(); complex_lvl2.put("test_int", 3); @@ -512,11 +759,40 @@ public void testStructComplex() throws Exception { json.put("test_bool", true); json.put("test_DOUBLe", new JSONArray(new Double[] {1.1, 2.2, 3.3, 4.4})); json.put("test_date", 1); + json.put("test_datetime", 1); + json.put( + "test_datetime_str", + new JSONArray(new String[] {"2021-09-27T20:51:10.752", "2021-09-27T00:00:00"})); json.put("complex_lvl1", complex_lvl1); json.put("complex_lvl2", complex_lvl2); - + json.put( + "test_numeric", + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23456"))); + json.put( + "test_numeric_repeated", + new JSONArray( + new byte[][] { + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0")) + .toByteArray(), + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal("99999999999999999999999999999.999999999")) + .toByteArray(), + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal("-99999999999999999999999999999.999999999")) + .toByteArray(), + })); + json.put("test_geo", "POINT(1,1)"); + json.put("test_timestamp", 12345678); + json.put("test_time", CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1))); + json.put("test_time_str", "20:51:10.1234"); + json.put("test_numeric_str", "12.4"); + json.put( + "test_bignumeric", + BigDecimalByteStringEncoder.encodeToNumericByteString(BigDecimal.valueOf(2.3))); + json.put("test_bignumeric_str", new JSONArray(new String[] {"1.23"})); DynamicMessage protoMsg = - JsonToProtoMessage.convertJsonToProtoMessage(ComplexRoot.getDescriptor(), json); + JsonToProtoMessage.convertJsonToProtoMessage( + ComplexRoot.getDescriptor(), COMPLEX_TABLE_SCHEMA, json); assertEquals(expectedProto, protoMsg); } diff --git a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto index ecffb2e0a8..b929604004 100644 --- a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto +++ b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto @@ -9,13 +9,20 @@ message ComplexRoot { optional bool test_bool = 4; repeated double test_double = 5; required int32 test_date = 6; - required ComplexLvl1 complex_lvl1 = 7; - required ComplexLvl2 complex_lvl2 = 8; - optional bytes test_numeric = 9; - optional string test_geo = 10; - optional int64 test_timestamp = 11; - optional int64 test_time = 12; - repeated bytes test_numeric_repeated = 13; + optional int64 test_datetime = 7; + repeated int64 test_datetime_str = 8; + required ComplexLvl1 complex_lvl1 = 9; + required + ComplexLvl2 complex_lvl2 = 10; + optional bytes test_numeric = 11; + optional string test_geo = 12; + optional int64 test_timestamp = 13; + optional int64 test_time = 14; + optional int64 test_time_str = 15; + repeated bytes test_numeric_repeated = 16; + optional bytes test_numeric_str = 17; + optional bytes test_bignumeric = 18; + repeated bytes test_bignumeric_str = 19; } message CasingComplex { @@ -131,3 +138,19 @@ message TestRepeatedIsOptional { message TopLevelMismatch { optional double mismatch_double = 1; } + +message TestDatetime { + optional int64 datetime = 1; +} + +message TestTime { + repeated int64 time = 1; +} + +message TestNumeric { + optional bytes numeric = 1; +} + +message TestBignumeric { + repeated bytes bignumeric = 1; +}