From 9d272dd711dfe8909a7b421dabdcdef30f1dc689 Mon Sep 17 00:00:00 2001 From: Yiru Tang Date: Mon, 4 Oct 2021 07:17:39 -0700 Subject: [PATCH] fix: add string to DATETIME, TIME, NUMERIC, BIGNUMERIC support in JsonStreamWriter v1 (#1345) * fix: update code comment to reflect max size change * fix: String to DATETIME, TIME, NUMERIC, BIGNUMERIC conversion in JsonWriter * . * . * . --- .../bigquery/storage/v1/JsonStreamWriter.java | 4 +- .../storage/v1/JsonToProtoMessage.java | 198 +++++++++++-- .../storage/v1/JsonStreamWriterTest.java | 52 ++++ .../storage/v1/JsonToProtoMessageTest.java | 279 +++++++++++++++++- 4 files changed, 498 insertions(+), 35 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java index 4a13b4dfcd..6b20e89bcb 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriter.java @@ -75,6 +75,7 @@ private JsonStreamWriter(Builder builder) builder.traceId); this.streamWriter = streamWriterBuilder.build(); this.streamName = builder.streamName; + this.tableSchema = builder.tableSchema; } /** @@ -105,7 +106,8 @@ public ApiFuture append(JSONArray jsonArr, long offset) { // of JSON data. for (int i = 0; i < jsonArr.length(); i++) { JSONObject json = jsonArr.getJSONObject(i); - Message protoMessage = JsonToProtoMessage.convertJsonToProtoMessage(this.descriptor, json); + Message protoMessage = + JsonToProtoMessage.convertJsonToProtoMessage(this.descriptor, this.tableSchema, json); rowsBuilder.addSerializedRows(protoMessage.toByteString()); } // Need to make sure refreshAppendAndSetDescriptor finish first before this can run diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java index d2cdbab982..df6d26ac8e 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigquery.storage.v1; +import com.google.api.pathtemplate.ValidationException; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.protobuf.ByteString; @@ -23,10 +24,14 @@ import com.google.protobuf.DynamicMessage; import com.google.protobuf.Message; import com.google.protobuf.UninitializedMessageException; +import java.math.BigDecimal; +import java.util.List; import java.util.logging.Logger; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import org.threeten.bp.LocalDateTime; +import org.threeten.bp.LocalTime; /** * Converts Json data to protocol buffer messages given the protocol buffer descriptor. The protobuf @@ -58,7 +63,28 @@ public static DynamicMessage convertJsonToProtoMessage(Descriptor protoSchema, J Preconditions.checkNotNull(protoSchema, "Protobuf descriptor is null."); Preconditions.checkState(json.length() != 0, "JSONObject is empty."); - return convertJsonToProtoMessageImpl(protoSchema, json, "root", /*topLevel=*/ true); + return convertJsonToProtoMessageImpl(protoSchema, null, json, "root", /*topLevel=*/ true); + } + + /** + * Converts Json data to protocol buffer messages given the protocol buffer descriptor. + * + * @param protoSchema + * @param tableSchema bigquery table schema is needed for type conversion of DATETIME, TIME, + * NUMERIC, BIGNUMERIC + * @param json + * @throws IllegalArgumentException when JSON data is not compatible with proto descriptor. + */ + public static DynamicMessage convertJsonToProtoMessage( + Descriptor protoSchema, TableSchema tableSchema, JSONObject json) + throws IllegalArgumentException { + Preconditions.checkNotNull(json, "JSONObject is null."); + Preconditions.checkNotNull(protoSchema, "Protobuf descriptor is null."); + Preconditions.checkNotNull(tableSchema, "TableSchema is null."); + Preconditions.checkState(json.length() != 0, "JSONObject is empty."); + + return convertJsonToProtoMessageImpl( + protoSchema, tableSchema.getFieldsList(), json, "root", /*topLevel=*/ true); } /** @@ -71,7 +97,11 @@ public static DynamicMessage convertJsonToProtoMessage(Descriptor protoSchema, J * @throws IllegalArgumentException when JSON data is not compatible with proto descriptor. */ private static DynamicMessage convertJsonToProtoMessageImpl( - Descriptor protoSchema, JSONObject json, String jsonScope, boolean topLevel) + Descriptor protoSchema, + List tableSchema, + JSONObject json, + String jsonScope, + boolean topLevel) throws IllegalArgumentException { DynamicMessage.Builder protoMsg = DynamicMessage.newBuilder(protoSchema); @@ -90,10 +120,25 @@ private static DynamicMessage convertJsonToProtoMessageImpl( throw new IllegalArgumentException( String.format("JSONObject has fields unknown to BigQuery: %s.", currentScope)); } + TableFieldSchema fieldSchema = null; + if (tableSchema != null) { + // protoSchema is generated from tableSchema so their field ordering should match. + fieldSchema = tableSchema.get(field.getIndex()); + if (!fieldSchema.getName().equals(field.getName())) { + throw new ValidationException( + "Field at index " + + field.getIndex() + + " has mismatch names (" + + fieldSchema.getName() + + ") (" + + field.getName() + + ")"); + } + } if (!field.isRepeated()) { - fillField(protoMsg, field, json, jsonName, currentScope); + fillField(protoMsg, field, fieldSchema, json, jsonName, currentScope); } else { - fillRepeatedField(protoMsg, field, json, jsonName, currentScope); + fillRepeatedField(protoMsg, field, fieldSchema, json, jsonName, currentScope); } } @@ -119,6 +164,7 @@ private static DynamicMessage convertJsonToProtoMessageImpl( * * @param protoMsg The protocol buffer message being constructed * @param fieldDescriptor + * @param fieldSchema * @param json * @param exactJsonKeyName Exact key name in JSONObject instead of lowercased version * @param currentScope Debugging purposes @@ -127,6 +173,7 @@ private static DynamicMessage convertJsonToProtoMessageImpl( private static void fillField( DynamicMessage.Builder protoMsg, FieldDescriptor fieldDescriptor, + TableFieldSchema fieldSchema, JSONObject json, String exactJsonKeyName, String currentScope) @@ -144,6 +191,25 @@ private static void fillField( } break; case BYTES: + if (fieldSchema != null) { + if (fieldSchema.getType() == TableFieldSchema.Type.NUMERIC) { + if (val instanceof String) { + protoMsg.setField( + fieldDescriptor, + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal((String) val))); + return; + } + } else if (fieldSchema.getType() == TableFieldSchema.Type.BIGNUMERIC) { + if (val instanceof String) { + protoMsg.setField( + fieldDescriptor, + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal((String) val))); + return; + } + } + } if (val instanceof ByteString) { protoMsg.setField(fieldDescriptor, ((ByteString) val).toByteArray()); return; @@ -170,6 +236,29 @@ private static void fillField( } break; case INT64: + if (fieldSchema != null) { + if (fieldSchema.getType() == TableFieldSchema.Type.DATETIME) { + if (val instanceof String) { + protoMsg.setField( + fieldDescriptor, + CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.parse((String) val))); + return; + } else if (val instanceof Long) { + protoMsg.setField(fieldDescriptor, (Long) val); + return; + } + } else if (fieldSchema.getType() == TableFieldSchema.Type.TIME) { + if (val instanceof String) { + protoMsg.setField( + fieldDescriptor, + CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.parse((String) val))); + return; + } else if (val instanceof Long) { + protoMsg.setField(fieldDescriptor, (Long) val); + return; + } + } + } if (val instanceof Integer) { protoMsg.setField(fieldDescriptor, new Long((Integer) val)); return; @@ -206,6 +295,7 @@ private static void fillField( fieldDescriptor, convertJsonToProtoMessageImpl( fieldDescriptor.getMessageType(), + fieldSchema == null ? null : fieldSchema.getFieldsList(), json.getJSONObject(exactJsonKeyName), currentScope, /*topLevel =*/ false)); @@ -224,6 +314,7 @@ private static void fillField( * * @param protoMsg The protocol buffer message being constructed * @param fieldDescriptor + * @param fieldSchema * @param json If root level has no matching fields, throws exception. * @param exactJsonKeyName Exact key name in JSONObject instead of lowercased version * @param currentScope Debugging purposes @@ -232,6 +323,7 @@ private static void fillField( private static void fillRepeatedField( DynamicMessage.Builder protoMsg, FieldDescriptor fieldDescriptor, + TableFieldSchema fieldSchema, JSONObject json, String exactJsonKeyName, String currentScope) @@ -259,40 +351,81 @@ private static void fillRepeatedField( } break; case BYTES: - if (val instanceof JSONArray) { - try { - byte[] bytes = new byte[((JSONArray) val).length()]; - for (int j = 0; j < ((JSONArray) val).length(); j++) { - bytes[j] = (byte) ((JSONArray) val).getInt(j); - if (bytes[j] != ((JSONArray) val).getInt(j)) { - throw new IllegalArgumentException( - String.format( - "Error: " - + currentScope - + "[" - + index - + "] could not be converted to byte[].")); + Boolean added = false; + if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.NUMERIC) { + if (val instanceof String) { + protoMsg.addRepeatedField( + fieldDescriptor, + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal((String) val))); + added = true; + } + } else if (fieldSchema != null + && fieldSchema.getType() == TableFieldSchema.Type.BIGNUMERIC) { + if (val instanceof String) { + protoMsg.addRepeatedField( + fieldDescriptor, + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal((String) val))); + added = true; + } + } + if (!added) { + if (val instanceof JSONArray) { + try { + byte[] bytes = new byte[((JSONArray) val).length()]; + for (int j = 0; j < ((JSONArray) val).length(); j++) { + bytes[j] = (byte) ((JSONArray) val).getInt(j); + if (bytes[j] != ((JSONArray) val).getInt(j)) { + throw new IllegalArgumentException( + String.format( + "Error: " + + currentScope + + "[" + + index + + "] could not be converted to byte[].")); + } } + protoMsg.addRepeatedField(fieldDescriptor, bytes); + } catch (JSONException e) { + throw new IllegalArgumentException( + String.format( + "Error: " + + currentScope + + "[" + + index + + "] could not be converted to byte[].")); } - protoMsg.addRepeatedField(fieldDescriptor, bytes); - } catch (JSONException e) { - throw new IllegalArgumentException( - String.format( - "Error: " - + currentScope - + "[" - + index - + "] could not be converted to byte[].")); + } else if (val instanceof ByteString) { + protoMsg.addRepeatedField(fieldDescriptor, ((ByteString) val).toByteArray()); + return; + } else { + fail = true; } - } else if (val instanceof ByteString) { - protoMsg.addRepeatedField(fieldDescriptor, ((ByteString) val).toByteArray()); - return; - } else { - fail = true; } break; case INT64: - if (val instanceof Integer) { + if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.DATETIME) { + if (val instanceof String) { + protoMsg.addRepeatedField( + fieldDescriptor, + CivilTimeEncoder.encodePacked64DatetimeMicros(LocalDateTime.parse((String) val))); + } else if (val instanceof Long) { + protoMsg.addRepeatedField(fieldDescriptor, (Long) val); + } else { + fail = true; + } + } else if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.TIME) { + if (val instanceof String) { + protoMsg.addRepeatedField( + fieldDescriptor, + CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.parse((String) val))); + } else if (val instanceof Long) { + protoMsg.addRepeatedField(fieldDescriptor, (Long) val); + } else { + fail = true; + } + } else if (val instanceof Integer) { protoMsg.addRepeatedField(fieldDescriptor, new Long((Integer) val)); } else if (val instanceof Long) { protoMsg.addRepeatedField(fieldDescriptor, (Long) val); @@ -330,6 +463,7 @@ private static void fillRepeatedField( fieldDescriptor, convertJsonToProtoMessageImpl( fieldDescriptor.getMessageType(), + fieldSchema == null ? null : fieldSchema.getFieldsList(), jsonArray.getJSONObject(i), currentScope, /*topLevel =*/ false)); diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java index b688b7542a..cebd8cc7c2 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonStreamWriterTest.java @@ -24,6 +24,7 @@ import com.google.api.gax.grpc.testing.LocalChannelProvider; import com.google.api.gax.grpc.testing.MockGrpcService; import com.google.api.gax.grpc.testing.MockServiceHelper; +import com.google.cloud.bigquery.storage.test.JsonTest; import com.google.cloud.bigquery.storage.test.Test.FooType; import com.google.protobuf.Descriptors.DescriptorValidationException; import com.google.protobuf.Int64Value; @@ -42,6 +43,7 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; import org.threeten.bp.Instant; +import org.threeten.bp.LocalTime; @RunWith(JUnit4.class) public class JsonStreamWriterTest { @@ -195,6 +197,56 @@ public void testSingleAppendSimpleJson() throws Exception { } } + @Test + public void testSpecialTypeAppend() throws Exception { + TableFieldSchema field = + TableFieldSchema.newBuilder() + .setName("time") + .setType(TableFieldSchema.Type.TIME) + .setMode(TableFieldSchema.Mode.REPEATED) + .build(); + TableSchema tableSchema = TableSchema.newBuilder().addFields(field).build(); + + JsonTest.TestTime expectedProto = + JsonTest.TestTime.newBuilder() + .addTime(CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1))) + .build(); + JSONObject foo = new JSONObject(); + foo.put("time", new JSONArray(new String[] {"01:00:01"})); + JSONArray jsonArr = new JSONArray(); + jsonArr.put(foo); + + try (JsonStreamWriter writer = + getTestJsonStreamWriterBuilder(TEST_STREAM, tableSchema).build()) { + + testBigQueryWrite.addResponse( + AppendRowsResponse.newBuilder() + .setAppendResult( + AppendRowsResponse.AppendResult.newBuilder().setOffset(Int64Value.of(0)).build()) + .build()); + + ApiFuture appendFuture = writer.append(jsonArr); + assertEquals(0L, appendFuture.get().getAppendResult().getOffset().getValue()); + appendFuture.get(); + assertEquals( + 1, + testBigQueryWrite + .getAppendRequests() + .get(0) + .getProtoRows() + .getRows() + .getSerializedRowsCount()); + assertEquals( + testBigQueryWrite + .getAppendRequests() + .get(0) + .getProtoRows() + .getRows() + .getSerializedRows(0), + expectedProto.toByteString()); + } + } + @Test public void testSingleAppendMultipleSimpleJson() throws Exception { FooType expectedProto = FooType.newBuilder().setFoo("allen").build(); diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java index b8eba3c893..db5b14f73d 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java @@ -35,6 +35,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +import org.threeten.bp.LocalTime; @RunWith(JUnit4.class) public class JsonToProtoMessageTest { @@ -276,6 +277,145 @@ public class JsonToProtoMessageTest { new JSONObject().put("test_int", 3) })) }; + private final TableFieldSchema TEST_INT = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.INT64) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_int") + .build(); + private final TableFieldSchema TEST_STRING = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.STRING) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_string") + .build(); + private final TableFieldSchema TEST_BYTES = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.BYTES) + .setMode(TableFieldSchema.Mode.REQUIRED) + .setName("test_bytes") + .build(); + private final TableFieldSchema TEST_BOOL = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.BOOL) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_bool") + .build(); + private final TableFieldSchema TEST_DOUBLE = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.DOUBLE) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_double") + .build(); + private final TableFieldSchema TEST_DATE = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.DATE) + .setMode(TableFieldSchema.Mode.REQUIRED) + .setName("test_date") + .build(); + private final TableFieldSchema TEST_DATETIME = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.DATETIME) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_datetime") + .build(); + private final TableFieldSchema TEST_DATETIME_STR = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.DATETIME) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_datetime_str") + .build(); + private final TableFieldSchema COMPLEXLVL2 = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.STRUCT) + .setMode(TableFieldSchema.Mode.REQUIRED) + .addFields(0, TEST_INT) + .setName("complex_lvl2") + .build(); + private final TableFieldSchema COMPLEXLVL1 = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.STRUCT) + .setMode(TableFieldSchema.Mode.REQUIRED) + .addFields(0, TEST_INT) + .addFields(1, COMPLEXLVL2) + .setName("complex_lvl1") + .build(); + private final TableFieldSchema TEST_NUMERIC = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_numeric") + .build(); + private final TableFieldSchema TEST_NUMERIC_REPEATED = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_numeric_repeated") + .build(); + private final TableFieldSchema TEST_GEO = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.GEOGRAPHY) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_geo") + .build(); + private final TableFieldSchema TEST_TIMESTAMP = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.TIMESTAMP) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_timestamp") + .build(); + private final TableFieldSchema TEST_TIME = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.TIME) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_time") + .build(); + private final TableFieldSchema TEST_TIME_STR = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.TIME) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_time_str") + .build(); + private final TableFieldSchema TEST_NUMERIC_STR = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_numeric_str") + .build(); + private final TableFieldSchema TEST_BIGNUMERIC = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.NULLABLE) + .setName("test_bignumeric") + .build(); + private final TableFieldSchema TEST_BIGNUMERIC_STR = + TableFieldSchema.newBuilder() + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.REPEATED) + .setName("test_bignumeric_str") + .build(); + private final TableSchema COMPLEX_TABLE_SCHEMA = + TableSchema.newBuilder() + .addFields(0, TEST_INT) + .addFields(1, TEST_STRING) + .addFields(2, TEST_BYTES) + .addFields(3, TEST_BOOL) + .addFields(4, TEST_DOUBLE) + .addFields(5, TEST_DATE) + .addFields(6, TEST_DATETIME) + .addFields(7, TEST_DATETIME_STR) + .addFields(8, COMPLEXLVL1) + .addFields(9, COMPLEXLVL2) + .addFields(10, TEST_NUMERIC) + .addFields(11, TEST_GEO) + .addFields(12, TEST_TIMESTAMP) + .addFields(13, TEST_TIME) + .addFields(14, TEST_TIME_STR) + .addFields(15, TEST_NUMERIC_REPEATED) + .addFields(16, TEST_NUMERIC_STR) + .addFields(17, TEST_BIGNUMERIC) + .addFields(18, TEST_BIGNUMERIC_STR) + .build(); @Test public void testDifferentNameCasing() throws Exception { @@ -333,6 +473,89 @@ public void testInt32NotMatchInt64() throws Exception { } } + @Test + public void testDateTimeMismatch() throws Exception { + TableFieldSchema field = + TableFieldSchema.newBuilder() + .setName("datetime") + .setType(TableFieldSchema.Type.DATETIME) + .setMode(TableFieldSchema.Mode.REPEATED) + .build(); + TableSchema tableSchema = TableSchema.newBuilder().addFields(field).build(); + JSONObject json = new JSONObject(); + json.put("datetime", 1.0); + try { + DynamicMessage protoMsg = + JsonToProtoMessage.convertJsonToProtoMessage( + TestDatetime.getDescriptor(), tableSchema, json); + Assert.fail("should fail"); + } catch (IllegalArgumentException e) { + assertEquals("JSONObject does not have a int64 field at root.datetime.", e.getMessage()); + } + } + + @Test + public void testTimeMismatch() throws Exception { + TableFieldSchema field = + TableFieldSchema.newBuilder() + .setName("time") + .setType(TableFieldSchema.Type.TIME) + .setMode(TableFieldSchema.Mode.REPEATED) + .build(); + TableSchema tableSchema = TableSchema.newBuilder().addFields(field).build(); + JSONObject json = new JSONObject(); + json.put("time", new JSONArray(new Double[] {1.0})); + try { + DynamicMessage protoMsg = + JsonToProtoMessage.convertJsonToProtoMessage(TestTime.getDescriptor(), tableSchema, json); + Assert.fail("should fail"); + } catch (IllegalArgumentException e) { + assertEquals("JSONObject does not have a int64 field at root.time[0].", e.getMessage()); + } + } + + @Test + public void testNumericMismatch() throws Exception { + TableFieldSchema field = + TableFieldSchema.newBuilder() + .setName("numeric") + .setType(TableFieldSchema.Type.NUMERIC) + .setMode(TableFieldSchema.Mode.NULLABLE) + .build(); + TableSchema tableSchema = TableSchema.newBuilder().addFields(field).build(); + JSONObject json = new JSONObject(); + json.put("numeric", 1.0); + try { + DynamicMessage protoMsg = + JsonToProtoMessage.convertJsonToProtoMessage( + TestNumeric.getDescriptor(), tableSchema, json); + Assert.fail("should fail"); + } catch (IllegalArgumentException e) { + assertEquals("JSONObject does not have a bytes field at root.numeric.", e.getMessage()); + } + } + + @Test + public void testBigNumericMismatch() throws Exception { + TableFieldSchema field = + TableFieldSchema.newBuilder() + .setName("bignumeric") + .setType(TableFieldSchema.Type.BIGNUMERIC) + .setMode(TableFieldSchema.Mode.REPEATED) + .build(); + TableSchema tableSchema = TableSchema.newBuilder().addFields(field).build(); + JSONObject json = new JSONObject(); + json.put("bignumeric", new JSONArray(new Double[] {1.0})); + try { + DynamicMessage protoMsg = + JsonToProtoMessage.convertJsonToProtoMessage( + TestBignumeric.getDescriptor(), tableSchema, json); + Assert.fail("should fail"); + } catch (IllegalArgumentException e) { + assertEquals("JSONObject does not have a bytes field at root.bignumeric[0].", e.getMessage()); + } + } + @Test public void testDouble() throws Exception { TestDouble expectedProto = TestDouble.newBuilder().setDouble(1.2).setFloat(3.4f).build(); @@ -491,12 +714,35 @@ public void testStructComplex() throws Exception { .addTestDouble(3.3) .addTestDouble(4.4) .setTestDate(1) + .setTestDatetime(1) + .addTestDatetimeStr(142258614586538368L) + .addTestDatetimeStr(142258525253402624L) .setComplexLvl1( ComplexLvl1.newBuilder() .setTestInt(2) .setComplexLvl2(ComplexLvl2.newBuilder().setTestInt(3).build()) .build()) .setComplexLvl2(ComplexLvl2.newBuilder().setTestInt(3).build()) + .setTestNumeric( + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23456"))) + .setTestGeo("POINT(1,1)") + .setTestTimestamp(12345678) + .setTestTime(CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1))) + .setTestTimeStr(89332507144L) + .addTestNumericRepeated( + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0"))) + .addTestNumericRepeated( + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal("99999999999999999999999999999.999999999"))) + .addTestNumericRepeated( + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal("-99999999999999999999999999999.999999999"))) + .setTestNumericStr( + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("12.4"))) + .setTestBignumeric( + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("2.3"))) + .addTestBignumericStr( + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23"))) .build(); JSONObject complex_lvl2 = new JSONObject(); complex_lvl2.put("test_int", 3); @@ -512,11 +758,40 @@ public void testStructComplex() throws Exception { json.put("test_bool", true); json.put("test_DOUBLe", new JSONArray(new Double[] {1.1, 2.2, 3.3, 4.4})); json.put("test_date", 1); + json.put("test_datetime", 1); + json.put( + "test_datetime_str", + new JSONArray(new String[] {"2021-09-27T20:51:10.752", "2021-09-27T00:00:00"})); json.put("complex_lvl1", complex_lvl1); json.put("complex_lvl2", complex_lvl2); - + json.put( + "test_numeric", + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23456"))); + json.put( + "test_numeric_repeated", + new JSONArray( + new byte[][] { + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0")) + .toByteArray(), + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal("99999999999999999999999999999.999999999")) + .toByteArray(), + BigDecimalByteStringEncoder.encodeToNumericByteString( + new BigDecimal("-99999999999999999999999999999.999999999")) + .toByteArray(), + })); + json.put("test_geo", "POINT(1,1)"); + json.put("test_timestamp", 12345678); + json.put("test_time", CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1))); + json.put("test_time_str", "20:51:10.1234"); + json.put("test_numeric_str", "12.4"); + json.put( + "test_bignumeric", + BigDecimalByteStringEncoder.encodeToNumericByteString(BigDecimal.valueOf(2.3))); + json.put("test_bignumeric_str", new JSONArray(new String[] {"1.23"})); DynamicMessage protoMsg = - JsonToProtoMessage.convertJsonToProtoMessage(ComplexRoot.getDescriptor(), json); + JsonToProtoMessage.convertJsonToProtoMessage( + ComplexRoot.getDescriptor(), COMPLEX_TABLE_SCHEMA, json); assertEquals(expectedProto, protoMsg); }