Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: DATE and DOUBLE support more input formats #1397

Merged
merged 1 commit into from Nov 9, 2021
Merged

feat: DATE and DOUBLE support more input formats #1397

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
34 changes: 24 additions & 10 deletions ...igquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java
View file
Open in desktop
Expand Up @@ -25,6 +25,7 @@
import com.google.protobuf.Message;
import com.google.protobuf.UninitializedMessageException;
import java.math.BigDecimal;
import java.time.LocalDate;
import java.util.List;
import java.util.logging.Logger;
import org.json.JSONArray;
Expand Down Expand Up @@ -269,6 +270,15 @@ private static void fillField(
}
break;
case INT32:
if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.DATE) {
if (val instanceof String) {
protoMsg.setField(fieldDescriptor, (int) LocalDate.parse((String) val).toEpochDay());
return;
} else if (val instanceof Integer || val instanceof Long) {
protoMsg.setField(fieldDescriptor, ((Number) val).intValue());
return;
}
}
if (val instanceof Integer) {
protoMsg.setField(fieldDescriptor, (Integer) val);
return;
Expand All @@ -281,11 +291,8 @@ private static void fillField(
}
break;
case DOUBLE:
if (val instanceof Double) {
protoMsg.setField(fieldDescriptor, (Double) val);
return;
} else if (val instanceof Float) {
protoMsg.setField(fieldDescriptor, new Double((Float) val));
if (val instanceof Number) {
protoMsg.setField(fieldDescriptor, ((Number) val).doubleValue());
return;
}
break;
Expand Down Expand Up @@ -435,7 +442,16 @@ private static void fillRepeatedField(
}
break;
case INT32:
if (val instanceof Integer) {
if (fieldSchema != null && fieldSchema.getType() == TableFieldSchema.Type.DATE) {
if (val instanceof String) {
protoMsg.addRepeatedField(
fieldDescriptor, (int) LocalDate.parse((String) val).toEpochDay());
} else if (val instanceof Integer || val instanceof Long) {
protoMsg.addRepeatedField(fieldDescriptor, ((Number) val).intValue());
} else {
fail = true;
}
} else if (val instanceof Integer) {
protoMsg.addRepeatedField(fieldDescriptor, (Integer) val);
} else {
fail = true;
Expand All @@ -449,10 +465,8 @@ private static void fillRepeatedField(
}
break;
case DOUBLE:
if (val instanceof Double) {
protoMsg.addRepeatedField(fieldDescriptor, (Double) val);
} else if (val instanceof Float) {
protoMsg.addRepeatedField(fieldDescriptor, new Double((float) val));
if (val instanceof Number) {
protoMsg.addRepeatedField(fieldDescriptor, ((Number) val).doubleValue());
} else {
fail = true;
}
Expand Down
65 changes: 60 additions & 5 deletions ...erystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java
View file
Open in desktop
Expand Up @@ -76,7 +76,11 @@ public class JsonToProtoMessageTest {
new Message[] {Int32Type.newBuilder().setTestFieldType(Integer.MAX_VALUE).build()})
.put(
DoubleType.getDescriptor(),
new Message[] {DoubleType.newBuilder().setTestFieldType(1.23).build()})
new Message[] {
DoubleType.newBuilder().setTestFieldType(Long.MAX_VALUE).build(),
DoubleType.newBuilder().setTestFieldType(Integer.MAX_VALUE).build(),
DoubleType.newBuilder().setTestFieldType(1.23).build()
})
.put(
StringType.getDescriptor(),
new Message[] {StringType.newBuilder().setTestFieldType("test").build()})
Expand Down Expand Up @@ -181,6 +185,26 @@ public class JsonToProtoMessageTest {
.put(
RepeatedDouble.getDescriptor(),
new Message[] {
RepeatedDouble.newBuilder()
.addTestRepeated(Long.MAX_VALUE)
.addTestRepeated(Long.MIN_VALUE)
.addTestRepeated(Integer.MAX_VALUE)
.addTestRepeated(Integer.MIN_VALUE)
.addTestRepeated(Short.MAX_VALUE)
.addTestRepeated(Short.MIN_VALUE)
.addTestRepeated(Byte.MAX_VALUE)
.addTestRepeated(Byte.MIN_VALUE)
.addTestRepeated(0)
.build(),
RepeatedDouble.newBuilder()
.addTestRepeated(Integer.MAX_VALUE)
.addTestRepeated(Integer.MIN_VALUE)
.addTestRepeated(Short.MAX_VALUE)
.addTestRepeated(Short.MIN_VALUE)
.addTestRepeated(Byte.MAX_VALUE)
.addTestRepeated(Byte.MIN_VALUE)
.addTestRepeated(0)
.build(),
RepeatedDouble.newBuilder()
.addTestRepeated(Double.MAX_VALUE)
.addTestRepeated(Double.MIN_VALUE)
Expand Down Expand Up @@ -593,15 +617,43 @@ public void testBigNumericMismatch() throws Exception {

@Test
public void testDouble() throws Exception {
TestDouble expectedProto = TestDouble.newBuilder().setDouble(1.2).setFloat(3.4f).build();
TestDouble expectedProto =
TestDouble.newBuilder()
.setDouble(1.2)
.setFloat(3.4f)
.setByte(5)
.setShort(6)
.setInt(7)
.setLong(8)
.build();
JSONObject json = new JSONObject();
json.put("double", 1.2);
json.put("float", 3.4f);
json.put("byte", new Byte((byte) 5));
json.put("short", new Short((short) 6));
json.put("int", 7);
json.put("long", 8L);
DynamicMessage protoMsg =
JsonToProtoMessage.convertJsonToProtoMessage(TestDouble.getDescriptor(), json);
assertEquals(expectedProto, protoMsg);
}

@Test
public void testDate() throws Exception {
TableSchema tableSchema =
TableSchema.newBuilder()
.addFields(TableFieldSchema.newBuilder(TEST_DATE).setName("test_string").build())
.addFields(TableFieldSchema.newBuilder(TEST_DATE).setName("test_long").build())
.build();
TestDate expectedProto = TestDate.newBuilder().setTestString(18935).setTestLong(18935).build();
JSONObject json = new JSONObject();
json.put("test_string", "2021-11-04");
json.put("test_long", 18935L);
DynamicMessage protoMsg =
JsonToProtoMessage.convertJsonToProtoMessage(TestDate.getDescriptor(), tableSchema, json);
assertEquals(expectedProto, protoMsg);
}

@Test
public void testAllTypes() throws Exception {
for (Map.Entry<Descriptor, String> entry : AllTypesToDebugMessageTest.entrySet()) {
Expand All @@ -620,7 +672,9 @@ public void testAllTypes() throws Exception {
e.getMessage());
}
}
if (entry.getKey() == Int64Type.getDescriptor()
if (entry.getKey() == DoubleType.getDescriptor()) {
assertEquals(entry.getKey().getFullName(), 3, success);
} else if (entry.getKey() == Int64Type.getDescriptor()
|| entry.getKey() == BytesType.getDescriptor()) {
assertEquals(entry.getKey().getFullName(), 2, success);
} else {
Expand Down Expand Up @@ -656,8 +710,9 @@ public void testAllRepeatedTypesWithLimits() throws Exception {
.equals("Error: root.test_repeated[0] could not be converted to byte[]."));
}
}
if (entry.getKey() == RepeatedInt64.getDescriptor()
|| entry.getKey() == RepeatedDouble.getDescriptor()) {
if (entry.getKey() == RepeatedDouble.getDescriptor()) {
assertEquals(entry.getKey().getFullName(), 4, success);
} else if (entry.getKey() == RepeatedInt64.getDescriptor()) {
assertEquals(entry.getKey().getFullName(), 2, success);
} else {
assertEquals(entry.getKey().getFullName(), 1, success);
Expand Down
9 changes: 9 additions & 0 deletions google-cloud-bigquerystorage/src/test/proto/jsonTest.proto
View file
Open in desktop
Expand Up @@ -119,6 +119,15 @@ message TestInt32 {
message TestDouble {
optional double double = 1;
optional double float = 2;
optional double byte = 3;
optional double short = 4;
optional double int = 5;
optional double long = 6;
}

message TestDate {
optional int32 test_string = 1;
optional int32 test_long = 2;
}

message NestedRepeated {
Expand Down