Skip to content

Commit

Permalink
feat: DATE and DOUBLE support more input formats (#1397)
Browse files Browse the repository at this point in the history 
1. Long values are converted into Integer for DATE fields.
2. Date literals are converted into epoch days (Integer) for DATE fields.
3. Number values are converted into Double for DOUBLE fields.
  • Loading branch information
@burakgok
burakgok committed Nov 9, 2021
1 parent 75aed71 commit 9c8dc0a
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 15 deletions.
34 changes: 24 additions & 10 deletions ...igquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java
View file
Expand Up @@ -25,6 +25,7 @@
import com.google.protobuf.Message;
import com.google.protobuf.UninitializedMessageException;
import java.math.BigDecimal;
import java.time.LocalDate;
import java.util.List;
import java.util.logging.Logger;
import org.json.JSONArray;
Expand Down Expand Up @@ -269,6 +270,15 @@ private static void fillField(
}
break;
case INT32:
if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.DATE) {
if (val instanceof String) {
protoMsg.setField(fieldDescriptor, (int) LocalDate.parse((String) val).toEpochDay());
return;
} else if (val instanceof Integer || val instanceof Long) {
protoMsg.setField(fieldDescriptor, ((Number) val).intValue());
return;
}
}
if (val instanceof Integer) {
protoMsg.setField(fieldDescriptor, (Integer) val);
return;
Expand All @@ -281,11 +291,8 @@ private static void fillField(
}
break;
case DOUBLE:
if (val instanceof Double) {
protoMsg.setField(fieldDescriptor, (Double) val);
return;
} else if (val instanceof Float) {
protoMsg.setField(fieldDescriptor, new Double((Float) val));
if (val instanceof Number) {
protoMsg.setField(fieldDescriptor, ((Number) val).doubleValue());
return;
}
break;
Expand Down Expand Up @@ -435,7 +442,16 @@ private static void fillRepeatedField(
}
break;
case INT32:
if (val instanceof Integer) {
if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.DATE) {
if (val instanceof String) {
protoMsg.addRepeatedField(
fieldDescriptor, (int) LocalDate.parse((String) val).toEpochDay());
} else if (val instanceof Integer || val instanceof Long) {
protoMsg.addRepeatedField(fieldDescriptor, ((Number) val).intValue());
} else {
fail = true;
}
} else if (val instanceof Integer) {
protoMsg.addRepeatedField(fieldDescriptor, (Integer) val);
} else {
fail = true;
Expand All @@ -449,10 +465,8 @@ private static void fillRepeatedField(
}
break;
case DOUBLE:
if (val instanceof Double) {
protoMsg.addRepeatedField(fieldDescriptor, (Double) val);
} else if (val instanceof Float) {
protoMsg.addRepeatedField(fieldDescriptor, new Double((float) val));
if (val instanceof Number) {
protoMsg.addRepeatedField(fieldDescriptor, ((Number) val).doubleValue());
} else {
fail = true;
}
Expand Down
65 changes: 60 additions & 5 deletions ...erystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java
View file
Expand Up @@ -76,7 +76,11 @@ public class JsonToProtoMessageTest {
new Message[] {Int32Type.newBuilder().setTestFieldType(Integer.MAX_VALUE).build()})
.put(
DoubleType.getDescriptor(),
new Message[] {DoubleType.newBuilder().setTestFieldType(1.23).build()})
new Message[] {
DoubleType.newBuilder().setTestFieldType(Long.MAX_VALUE).build(),
DoubleType.newBuilder().setTestFieldType(Integer.MAX_VALUE).build(),
DoubleType.newBuilder().setTestFieldType(1.23).build()
})
.put(
StringType.getDescriptor(),
new Message[] {StringType.newBuilder().setTestFieldType("test").build()})
Expand Down Expand Up @@ -181,6 +185,26 @@ public class JsonToProtoMessageTest {
.put(
RepeatedDouble.getDescriptor(),
new Message[] {
RepeatedDouble.newBuilder()
.addTestRepeated(Long.MAX_VALUE)
.addTestRepeated(Long.MIN_VALUE)
.addTestRepeated(Integer.MAX_VALUE)
.addTestRepeated(Integer.MIN_VALUE)
.addTestRepeated(Short.MAX_VALUE)
.addTestRepeated(Short.MIN_VALUE)
.addTestRepeated(Byte.MAX_VALUE)
.addTestRepeated(Byte.MIN_VALUE)
.addTestRepeated(0)
.build(),
RepeatedDouble.newBuilder()
.addTestRepeated(Integer.MAX_VALUE)
.addTestRepeated(Integer.MIN_VALUE)
.addTestRepeated(Short.MAX_VALUE)
.addTestRepeated(Short.MIN_VALUE)
.addTestRepeated(Byte.MAX_VALUE)
.addTestRepeated(Byte.MIN_VALUE)
.addTestRepeated(0)
.build(),
RepeatedDouble.newBuilder()
.addTestRepeated(Double.MAX_VALUE)
.addTestRepeated(Double.MIN_VALUE)
Expand Down Expand Up @@ -593,15 +617,43 @@ public void testBigNumericMismatch() throws Exception {

@Test
public void testDouble() throws Exception {
TestDouble expectedProto = TestDouble.newBuilder().setDouble(1.2).setFloat(3.4f).build();
TestDouble expectedProto =
TestDouble.newBuilder()
.setDouble(1.2)
.setFloat(3.4f)
.setByte(5)
.setShort(6)
.setInt(7)
.setLong(8)
.build();
JSONObject json = new JSONObject();
json.put("double", 1.2);
json.put("float", 3.4f);
json.put("byte", new Byte((byte) 5));
json.put("short", new Short((short) 6));
json.put("int", 7);
json.put("long", 8L);
DynamicMessage protoMsg =
JsonToProtoMessage.convertJsonToProtoMessage(TestDouble.getDescriptor(), json);
assertEquals(expectedProto, protoMsg);
}

@Test
public void testDate() throws Exception {
TableSchema tableSchema =
TableSchema.newBuilder()
.addFields(TableFieldSchema.newBuilder(TEST_DATE).setName("test_string").build())
.addFields(TableFieldSchema.newBuilder(TEST_DATE).setName("test_long").build())
.build();
TestDate expectedProto = TestDate.newBuilder().setTestString(18935).setTestLong(18935).build();
JSONObject json = new JSONObject();
json.put("test_string", "2021-11-04");
json.put("test_long", 18935L);
DynamicMessage protoMsg =
JsonToProtoMessage.convertJsonToProtoMessage(TestDate.getDescriptor(), tableSchema, json);
assertEquals(expectedProto, protoMsg);
}

@Test
public void testAllTypes() throws Exception {
for (Map.Entry<Descriptor, String> entry : AllTypesToDebugMessageTest.entrySet()) {
Expand All @@ -620,7 +672,9 @@ public void testAllTypes() throws Exception {
e.getMessage());
}
}
if (entry.getKey() == Int64Type.getDescriptor()
if (entry.getKey() == DoubleType.getDescriptor()) {
assertEquals(entry.getKey().getFullName(), 3, success);
} else if (entry.getKey() == Int64Type.getDescriptor()
|| entry.getKey() == BytesType.getDescriptor()) {
assertEquals(entry.getKey().getFullName(), 2, success);
} else {
Expand Down Expand Up @@ -656,8 +710,9 @@ public void testAllRepeatedTypesWithLimits() throws Exception {
.equals("Error: root.test_repeated[0] could not be converted to byte[]."));
}
}
if (entry.getKey() == RepeatedInt64.getDescriptor()
|| entry.getKey() == RepeatedDouble.getDescriptor()) {
if (entry.getKey() == RepeatedDouble.getDescriptor()) {
assertEquals(entry.getKey().getFullName(), 4, success);
} else if (entry.getKey() == RepeatedInt64.getDescriptor()) {
assertEquals(entry.getKey().getFullName(), 2, success);
} else {
assertEquals(entry.getKey().getFullName(), 1, success);
Expand Down
9 changes: 9 additions & 0 deletions google-cloud-bigquerystorage/src/test/proto/jsonTest.proto
View file
Expand Up @@ -119,6 +119,15 @@ message TestInt32 {
message TestDouble {
optional double double = 1;
optional double float = 2;
optional double byte = 3;
optional double short = 4;
optional double int = 5;
optional double long = 6;
}

message TestDate {
optional int32 test_string = 1;
optional int32 test_long = 2;
}

message NestedRepeated {
Expand Down

0 comments on commit 9c8dc0a

Please sign in to comment.