From 8409ed94862754ebf238a525f2a958f47aa35788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20G=C3=B6k?= Date: Wed, 3 Nov 2021 12:53:03 +0300 Subject: [PATCH] feat: DATE and DOUBLE support more input formats 1. Long values are converted into Integer for DATE fields. 2. Date literals are converted into epoch days (Integer) for DATE fields. 3. Number values are converted into Double for DOUBLE fields. --- .../storage/v1/JsonToProtoMessage.java | 34 +++++++--- .../storage/v1/JsonToProtoMessageTest.java | 65 +++++++++++++++++-- .../src/test/proto/jsonTest.proto | 9 +++ 3 files changed, 93 insertions(+), 15 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java index 91a432c258..6423fe43f7 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java @@ -25,6 +25,7 @@ import com.google.protobuf.Message; import com.google.protobuf.UninitializedMessageException; import java.math.BigDecimal; +import java.time.LocalDate; import java.util.List; import java.util.logging.Logger; import org.json.JSONArray; @@ -269,6 +270,15 @@ private static void fillField( } break; case INT32: + if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.DATE) { + if (val instanceof String) { + protoMsg.setField(fieldDescriptor, (int) LocalDate.parse((String) val).toEpochDay()); + return; + } else if (val instanceof Integer || val instanceof Long) { + protoMsg.setField(fieldDescriptor, ((Number) val).intValue()); + return; + } + } if (val instanceof Integer) { protoMsg.setField(fieldDescriptor, (Integer) val); return; @@ -281,11 +291,8 @@ private static void fillField( } break; case DOUBLE: - if (val instanceof Double) { - protoMsg.setField(fieldDescriptor, (Double) val); - return; - } else if (val instanceof Float) { - protoMsg.setField(fieldDescriptor, new Double((Float) val)); + if (val instanceof Number) { + protoMsg.setField(fieldDescriptor, ((Number) val).doubleValue()); return; } break; @@ -435,7 +442,16 @@ private static void fillRepeatedField( } break; case INT32: - if (val instanceof Integer) { + if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.DATE) { + if (val instanceof String) { + protoMsg.addRepeatedField( + fieldDescriptor, (int) LocalDate.parse((String) val).toEpochDay()); + } else if (val instanceof Integer || val instanceof Long) { + protoMsg.addRepeatedField(fieldDescriptor, ((Number) val).intValue()); + } else { + fail = true; + } + } else if (val instanceof Integer) { protoMsg.addRepeatedField(fieldDescriptor, (Integer) val); } else { fail = true; @@ -449,10 +465,8 @@ private static void fillRepeatedField( } break; case DOUBLE: - if (val instanceof Double) { - protoMsg.addRepeatedField(fieldDescriptor, (Double) val); - } else if (val instanceof Float) { - protoMsg.addRepeatedField(fieldDescriptor, new Double((float) val)); + if (val instanceof Number) { + protoMsg.addRepeatedField(fieldDescriptor, ((Number) val).doubleValue()); } else { fail = true; } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java index e71f927e6d..96b46ed8d0 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java @@ -76,7 +76,11 @@ public class JsonToProtoMessageTest { new Message[] {Int32Type.newBuilder().setTestFieldType(Integer.MAX_VALUE).build()}) .put( DoubleType.getDescriptor(), - new Message[] {DoubleType.newBuilder().setTestFieldType(1.23).build()}) + new Message[] { + DoubleType.newBuilder().setTestFieldType(Long.MAX_VALUE).build(), + DoubleType.newBuilder().setTestFieldType(Integer.MAX_VALUE).build(), + DoubleType.newBuilder().setTestFieldType(1.23).build() + }) .put( StringType.getDescriptor(), new Message[] {StringType.newBuilder().setTestFieldType("test").build()}) @@ -181,6 +185,26 @@ public class JsonToProtoMessageTest { .put( RepeatedDouble.getDescriptor(), new Message[] { + RepeatedDouble.newBuilder() + .addTestRepeated(Long.MAX_VALUE) + .addTestRepeated(Long.MIN_VALUE) + .addTestRepeated(Integer.MAX_VALUE) + .addTestRepeated(Integer.MIN_VALUE) + .addTestRepeated(Short.MAX_VALUE) + .addTestRepeated(Short.MIN_VALUE) + .addTestRepeated(Byte.MAX_VALUE) + .addTestRepeated(Byte.MIN_VALUE) + .addTestRepeated(0) + .build(), + RepeatedDouble.newBuilder() + .addTestRepeated(Integer.MAX_VALUE) + .addTestRepeated(Integer.MIN_VALUE) + .addTestRepeated(Short.MAX_VALUE) + .addTestRepeated(Short.MIN_VALUE) + .addTestRepeated(Byte.MAX_VALUE) + .addTestRepeated(Byte.MIN_VALUE) + .addTestRepeated(0) + .build(), RepeatedDouble.newBuilder() .addTestRepeated(Double.MAX_VALUE) .addTestRepeated(Double.MIN_VALUE) @@ -593,15 +617,43 @@ public void testBigNumericMismatch() throws Exception { @Test public void testDouble() throws Exception { - TestDouble expectedProto = TestDouble.newBuilder().setDouble(1.2).setFloat(3.4f).build(); + TestDouble expectedProto = + TestDouble.newBuilder() + .setDouble(1.2) + .setFloat(3.4f) + .setByte(5) + .setShort(6) + .setInt(7) + .setLong(8) + .build(); JSONObject json = new JSONObject(); json.put("double", 1.2); json.put("float", 3.4f); + json.put("byte", new Byte((byte) 5)); + json.put("short", new Short((short) 6)); + json.put("int", 7); + json.put("long", 8L); DynamicMessage protoMsg = JsonToProtoMessage.convertJsonToProtoMessage(TestDouble.getDescriptor(), json); assertEquals(expectedProto, protoMsg); } + @Test + public void testDate() throws Exception { + TableSchema tableSchema = + TableSchema.newBuilder() + .addFields(TableFieldSchema.newBuilder(TEST_DATE).setName("test_string").build()) + .addFields(TableFieldSchema.newBuilder(TEST_DATE).setName("test_long").build()) + .build(); + TestDate expectedProto = TestDate.newBuilder().setTestString(18935).setTestLong(18935).build(); + JSONObject json = new JSONObject(); + json.put("test_string", "2021-11-04"); + json.put("test_long", 18935L); + DynamicMessage protoMsg = + JsonToProtoMessage.convertJsonToProtoMessage(TestDate.getDescriptor(), tableSchema, json); + assertEquals(expectedProto, protoMsg); + } + @Test public void testAllTypes() throws Exception { for (Map.Entry entry : AllTypesToDebugMessageTest.entrySet()) { @@ -620,7 +672,9 @@ public void testAllTypes() throws Exception { e.getMessage()); } } - if (entry.getKey() == Int64Type.getDescriptor() + if (entry.getKey() == DoubleType.getDescriptor()) { + assertEquals(entry.getKey().getFullName(), 3, success); + } else if (entry.getKey() == Int64Type.getDescriptor() || entry.getKey() == BytesType.getDescriptor()) { assertEquals(entry.getKey().getFullName(), 2, success); } else { @@ -656,8 +710,9 @@ public void testAllRepeatedTypesWithLimits() throws Exception { .equals("Error: root.test_repeated[0] could not be converted to byte[].")); } } - if (entry.getKey() == RepeatedInt64.getDescriptor() - || entry.getKey() == RepeatedDouble.getDescriptor()) { + if (entry.getKey() == RepeatedDouble.getDescriptor()) { + assertEquals(entry.getKey().getFullName(), 4, success); + } else if (entry.getKey() == RepeatedInt64.getDescriptor()) { assertEquals(entry.getKey().getFullName(), 2, success); } else { assertEquals(entry.getKey().getFullName(), 1, success); diff --git a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto index 29efcdfb9c..fc76467f81 100644 --- a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto +++ b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto @@ -119,6 +119,15 @@ message TestInt32 { message TestDouble { optional double double = 1; optional double float = 2; + optional double byte = 3; + optional double short = 4; + optional double int = 5; + optional double long = 6; +} + +message TestDate { + optional int32 test_string = 1; + optional int32 test_long = 2; } message NestedRepeated {