From 94cb80d124fb8db414d1f6dd945ae0c520e150f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20G=C3=B6k?= Date: Wed, 3 Nov 2021 12:53:03 +0300 Subject: [PATCH] feat: DATE and DOUBLE support more input formats 1. Long values are converted into Integer for DATE fields. 2. Date literals are converted into epoch days (Integer) for DATE fields. 3. Number values are converted into Double for DOUBLE fields. --- .../storage/v1/JsonToProtoMessage.java | 35 +++++++++++++------ .../storage/v1/JsonToProtoMessageTest.java | 30 +++++++++++++++- .../src/test/proto/jsonTest.proto | 9 +++++ 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java index 91a432c258..4bd2fee5ac 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java @@ -25,6 +25,7 @@ import com.google.protobuf.Message; import com.google.protobuf.UninitializedMessageException; import java.math.BigDecimal; +import java.time.LocalDate; import java.util.List; import java.util.logging.Logger; import org.json.JSONArray; @@ -269,6 +270,15 @@ private static void fillField( } break; case INT32: + if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.DATE) { + if (val instanceof String) { + protoMsg.setField(fieldDescriptor, (int) LocalDate.parse((String) val).toEpochDay()); + return; + } else if (val instanceof Integer || val instanceof Long) { + protoMsg.setField(fieldDescriptor, ((Number) val).intValue()); + return; + } + } if (val instanceof Integer) { protoMsg.setField(fieldDescriptor, (Integer) val); return; @@ -281,11 +291,8 @@ private static void fillField( } break; case DOUBLE: - if (val instanceof Double) { - protoMsg.setField(fieldDescriptor, (Double) val); - return; - } else if (val instanceof Float) { - protoMsg.setField(fieldDescriptor, new Double((Float) val)); + if (val instanceof Number) { + protoMsg.setField(fieldDescriptor, ((Number) val).doubleValue()); return; } break; @@ -435,7 +442,17 @@ private static void fillRepeatedField( } break; case INT32: - if (val instanceof Integer) { + if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.DATE) { + if (val instanceof String) { + protoMsg.addRepeatedField( + fieldDescriptor, + (int) LocalDate.parse((String) val).toEpochDay()); + } else if (val instanceof Integer || val instanceof Long) { + protoMsg.addRepeatedField(fieldDescriptor, ((Number) val).intValue()); + } else { + fail = true; + } + } else if (val instanceof Integer) { protoMsg.addRepeatedField(fieldDescriptor, (Integer) val); } else { fail = true; @@ -449,10 +466,8 @@ private static void fillRepeatedField( } break; case DOUBLE: - if (val instanceof Double) { - protoMsg.addRepeatedField(fieldDescriptor, (Double) val); - } else if (val instanceof Float) { - protoMsg.addRepeatedField(fieldDescriptor, new Double((float) val)); + if (val instanceof Number) { + protoMsg.addRepeatedField(fieldDescriptor, ((Number) val).doubleValue()); } else { fail = true; } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java index e71f927e6d..4fa83804ab 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java @@ -593,15 +593,43 @@ public void testBigNumericMismatch() throws Exception { @Test public void testDouble() throws Exception { - TestDouble expectedProto = TestDouble.newBuilder().setDouble(1.2).setFloat(3.4f).build(); + TestDouble expectedProto = + TestDouble.newBuilder() + .setDouble(1.2) + .setFloat(3.4f) + .setByte(5) + .setShort(6) + .setInt(7) + .setLong(8) + .build(); JSONObject json = new JSONObject(); json.put("double", 1.2); json.put("float", 3.4f); + json.put("byte", new Byte((byte) 5)); + json.put("short", new Short((short) 6)); + json.put("int", 7); + json.put("long", 8L); DynamicMessage protoMsg = JsonToProtoMessage.convertJsonToProtoMessage(TestDouble.getDescriptor(), json); assertEquals(expectedProto, protoMsg); } + @Test + public void testDate() throws Exception { + TableSchema tableSchema = + TableSchema.newBuilder() + .addFields(TableFieldSchema.newBuilder(TEST_DATE).setName("test_string").build()) + .addFields(TableFieldSchema.newBuilder(TEST_DATE).setName("test_long").build()) + .build(); + TestDate expectedProto = TestDate.newBuilder().setTestString(18935).setTestLong(18935).build(); + JSONObject json = new JSONObject(); + json.put("test_string", "2021-11-04"); + json.put("test_long", 18935L); + DynamicMessage protoMsg = + JsonToProtoMessage.convertJsonToProtoMessage(TestDate.getDescriptor(), tableSchema, json); + assertEquals(expectedProto, protoMsg); + } + @Test public void testAllTypes() throws Exception { for (Map.Entry entry : AllTypesToDebugMessageTest.entrySet()) { diff --git a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto index 29efcdfb9c..fc76467f81 100644 --- a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto +++ b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto @@ -119,6 +119,15 @@ message TestInt32 { message TestDouble { optional double double = 1; optional double float = 2; + optional double byte = 3; + optional double short = 4; + optional double int = 5; + optional double long = 6; +} + +message TestDate { + optional int32 test_string = 1; + optional int32 test_long = 2; } message NestedRepeated {