Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: Cleanup JsonWriter bytes conversion code and add some test coverage #984

Merged
merged 2 commits into from Apr 7, 2021
Merged

fix: Cleanup JsonWriter bytes conversion code and add some test coverage #984

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b91dca5
fix: Cleanup JsonWriter bytes conversion code and add some test coverage
yirutang Apr 7, 2021
4a7e76e
fix test
yirutang Apr 7, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion .../main/java/com/google/cloud/bigquery/storage/v1alpha2/BQTableSchemaToProtoDescriptor.java
View file
Open in desktop
Expand Up @@ -54,7 +54,7 @@ public class BQTableSchemaToProtoDescriptor {
.put(Table.TableFieldSchema.Type.DOUBLE, FieldDescriptorProto.Type.TYPE_DOUBLE)
.put(Table.TableFieldSchema.Type.GEOGRAPHY, FieldDescriptorProto.Type.TYPE_STRING)
.put(Table.TableFieldSchema.Type.INT64, FieldDescriptorProto.Type.TYPE_INT64)
.put(Table.TableFieldSchema.Type.NUMERIC, FieldDescriptorProto.Type.TYPE_STRING)
.put(Table.TableFieldSchema.Type.NUMERIC, FieldDescriptorProto.Type.TYPE_BYTES)
.put(Table.TableFieldSchema.Type.STRING, FieldDescriptorProto.Type.TYPE_STRING)
.put(Table.TableFieldSchema.Type.STRUCT, FieldDescriptorProto.Type.TYPE_MESSAGE)
.put(Table.TableFieldSchema.Type.TIME, FieldDescriptorProto.Type.TYPE_STRING)
Expand Down
26 changes: 15 additions & 11 deletions ...rystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessage.java
View file
Open in desktop
Expand Up @@ -23,6 +23,7 @@
import com.google.protobuf.DynamicMessage;
import com.google.protobuf.Message;
import com.google.protobuf.UninitializedMessageException;
import java.util.logging.Logger;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
Expand All @@ -32,10 +33,11 @@
* descriptor must have all fields lowercased.
*/
public class JsonToProtoMessage {
private static final Logger LOG = Logger.getLogger(JsonToProtoMessage.class.getName());
private static ImmutableMap<FieldDescriptor.Type, String> FieldTypeToDebugMessage =
new ImmutableMap.Builder<FieldDescriptor.Type, String>()
.put(FieldDescriptor.Type.BOOL, "boolean")
.put(FieldDescriptor.Type.BYTES, "string")
.put(FieldDescriptor.Type.BYTES, "bytes")
.put(FieldDescriptor.Type.INT32, "int32")
.put(FieldDescriptor.Type.DOUBLE, "double")
.put(FieldDescriptor.Type.INT64, "int64")
Expand Down Expand Up @@ -142,9 +144,6 @@ private static void fillField(
if (val instanceof ByteString) {
protoMsg.setField(fieldDescriptor, ((ByteString) val).toByteArray());
return;
} else if (val instanceof String) {
protoMsg.setField(fieldDescriptor, ((String) val).getBytes());
return;
}
break;
case INT64:
Expand Down Expand Up @@ -237,18 +236,23 @@ private static void fillRepeatedField(
}
break;
case BYTES:
if (val instanceof String) {
// TODO(jstocklass): If string, decode it and pass in the byte array. Will need to
// update tests to ensure that strings passed in are properly encoded as well.
protoMsg.addRepeatedField(fieldDescriptor, ((String) val).getBytes());
} else if (val instanceof JSONArray) {
if (val instanceof JSONArray) {
try {
byte[] bytes = new byte[((JSONArray) val).length()];
for (int j = 0; j < ((JSONArray) val).length(); j++) {
bytes[j] = (byte) ((byte) (((JSONArray) val).get(j)) & 0xFF);
bytes[j] = (byte) ((JSONArray) val).getInt(j);
if (bytes[j] != ((JSONArray) val).getInt(j)) {
throw new IllegalArgumentException(
String.format(
"Error: "
+ currentScope
+ "["
+ index
+ "] could not be converted to byte[]."));
}
}
protoMsg.addRepeatedField(fieldDescriptor, bytes);
} catch (ClassCastException e) {
} catch (JSONException e) {
throw new IllegalArgumentException(
String.format(
"Error: "
Expand Down
9 changes: 8 additions & 1 deletion ...t/java/com/google/cloud/bigquery/storage/v1alpha2/BQTableSchemaToProtoDescriptorTest.java
View file
Open in desktop
Expand Up @@ -44,7 +44,7 @@ public class BQTableSchemaToProtoDescriptorTest {
.put(Table.TableFieldSchema.Type.DOUBLE, DoubleType.getDescriptor())
.put(Table.TableFieldSchema.Type.GEOGRAPHY, StringType.getDescriptor())
.put(Table.TableFieldSchema.Type.INT64, Int64Type.getDescriptor())
.put(Table.TableFieldSchema.Type.NUMERIC, StringType.getDescriptor())
.put(Table.TableFieldSchema.Type.NUMERIC, BytesType.getDescriptor())
.put(Table.TableFieldSchema.Type.STRING, StringType.getDescriptor())
.put(Table.TableFieldSchema.Type.TIME, StringType.getDescriptor())
.put(Table.TableFieldSchema.Type.TIMESTAMP, Int64Type.getDescriptor())
Expand Down Expand Up @@ -206,6 +206,12 @@ public void testStructComplex() throws Exception {
.setMode(Table.TableFieldSchema.Mode.NULLABLE)
.setName("test_time")
.build();
final Table.TableFieldSchema TEST_NUMERIC_REPEATED =
Table.TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.NUMERIC)
.setMode(Table.TableFieldSchema.Mode.REPEATED)
.setName("test_numeric_repeated")
.build();
final Table.TableSchema tableSchema =
Table.TableSchema.newBuilder()
.addFields(0, test_int)
Expand All @@ -220,6 +226,7 @@ public void testStructComplex() throws Exception {
.addFields(9, TEST_GEO)
.addFields(10, TEST_TIMESTAMP)
.addFields(11, TEST_TIME)
.addFields(12, TEST_NUMERIC_REPEATED)
.build();
final Descriptor descriptor =
BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema);
Expand Down
7 changes: 7 additions & 0 deletions ...st/java/com/google/cloud/bigquery/storage/v1beta2/BQTableSchemaToProtoDescriptorTest.java
View file
Open in desktop
Expand Up @@ -203,6 +203,12 @@ public void testStructComplex() throws Exception {
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_time")
.build();
final TableFieldSchema TEST_NUMERIC_REPEATED =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.NUMERIC)
.setMode(TableFieldSchema.Mode.REPEATED)
.setName("test_numeric_repeated")
.build();
final TableSchema tableSchema =
TableSchema.newBuilder()
.addFields(0, test_int)
Expand All @@ -217,6 +223,7 @@ public void testStructComplex() throws Exception {
.addFields(9, TEST_GEO)
.addFields(10, TEST_TIMESTAMP)
.addFields(11, TEST_TIME)
.addFields(12, TEST_NUMERIC_REPEATED)
.build();
final Descriptor descriptor =
BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema);
Expand Down
36 changes: 34 additions & 2 deletions ...storage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonStreamWriterTest.java
View file
Open in desktop
Expand Up @@ -37,7 +37,11 @@
import com.google.protobuf.Timestamp;
import java.io.IOException;
import java.math.BigDecimal;
import java.util.*;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.logging.Logger;
import org.json.JSONArray;
Expand Down Expand Up @@ -145,6 +149,12 @@ public class JsonStreamWriterTest {
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_numeric")
.build();
private final TableFieldSchema TEST_NUMERIC_REPEATED =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.NUMERIC)
.setMode(TableFieldSchema.Mode.REPEATED)
.setName("test_numeric_repeated")
.build();
private final TableFieldSchema TEST_GEO =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.GEOGRAPHY)
Expand Down Expand Up @@ -177,6 +187,7 @@ public class JsonStreamWriterTest {
.addFields(9, TEST_GEO)
.addFields(10, TEST_TIMESTAMP)
.addFields(11, TEST_TIME)
.addFields(12, TEST_NUMERIC_REPEATED)
.build();

@Before
Expand Down Expand Up @@ -414,6 +425,14 @@ public void testSingleAppendComplexJson() throws Exception {
.setTestGeo("POINT(1,1)")
.setTestTimestamp(12345678)
.setTestTime(CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1)))
.addTestNumericRepeated(
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0")))
.addTestNumericRepeated(
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal("99999999999999999999999999999.999999999")))
.addTestNumericRepeated(
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal("-99999999999999999999999999999.999999999")))
.build();
JSONObject complex_lvl2 = new JSONObject();
complex_lvl2.put("test_int", 3);
Expand All @@ -425,7 +444,7 @@ public void testSingleAppendComplexJson() throws Exception {
JSONObject json = new JSONObject();
json.put("test_int", 1);
json.put("test_string", new JSONArray(new String[] {"a", "b", "c"}));
json.put("test_bytes", "hello");
json.put("test_bytes", ByteString.copyFrom("hello".getBytes()));
json.put("test_bool", true);
json.put("test_DOUBLe", new JSONArray(new Double[] {1.1, 2.2, 3.3, 4.4}));
json.put("test_date", 1);
Expand All @@ -434,6 +453,19 @@ public void testSingleAppendComplexJson() throws Exception {
json.put(
"test_numeric",
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23456")));
json.put(
"test_numeric_repeated",
new JSONArray(
new byte[][] {
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("0"))
.toByteArray(),
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal("99999999999999999999999999999.999999999"))
.toByteArray(),
BigDecimalByteStringEncoder.encodeToNumericByteString(
new BigDecimal("-99999999999999999999999999999.999999999"))
.toByteArray(),
}));
json.put("test_geo", "POINT(1,1)");
json.put("test_timestamp", 12345678);
json.put("test_time", CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1)));
Expand Down
63 changes: 35 additions & 28 deletions ...orage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/JsonToProtoMessageTest.java
View file
Open in desktop
Expand Up @@ -42,7 +42,7 @@ public class JsonToProtoMessageTest {
private static ImmutableMap<Descriptor, String> AllTypesToDebugMessageTest =
new ImmutableMap.Builder<Descriptor, String>()
.put(BoolType.getDescriptor(), "boolean")
.put(BytesType.getDescriptor(), "string")
.put(BytesType.getDescriptor(), "bytes")
.put(Int64Type.getDescriptor(), "int64")
.put(Int32Type.getDescriptor(), "int32")
.put(DoubleType.getDescriptor(), "double")
Expand All @@ -59,9 +59,7 @@ public class JsonToProtoMessageTest {
.put(
BytesType.getDescriptor(),
new Message[] {
BytesType.newBuilder()
.setTestFieldType(ByteString.copyFrom("test".getBytes()))
.build()
BytesType.newBuilder().setTestFieldType(ByteString.copyFromUtf8("test")).build()
})
.put(
Int64Type.getDescriptor(),
Expand Down Expand Up @@ -101,15 +99,15 @@ public class JsonToProtoMessageTest {
})
.build();

private static ImmutableMap<Descriptor, String[]> AllRepeatedTypesToDebugMessageTest =
new ImmutableMap.Builder<Descriptor, String[]>()
.put(RepeatedBool.getDescriptor(), new String[] {"boolean"})
.put(RepeatedBytes.getDescriptor(), new String[] {"string", "bytes"})
.put(RepeatedInt64.getDescriptor(), new String[] {"int64"})
.put(RepeatedInt32.getDescriptor(), new String[] {"int32"})
.put(RepeatedDouble.getDescriptor(), new String[] {"double"})
.put(RepeatedString.getDescriptor(), new String[] {"string"})
.put(RepeatedObject.getDescriptor(), new String[] {"object"})
private static ImmutableMap<Descriptor, String> AllRepeatedTypesToDebugMessageTest =
new ImmutableMap.Builder<Descriptor, String>()
.put(RepeatedBool.getDescriptor(), "boolean")
.put(RepeatedBytes.getDescriptor(), "bytes")
.put(RepeatedInt64.getDescriptor(), "int64")
.put(RepeatedInt32.getDescriptor(), "int32")
.put(RepeatedDouble.getDescriptor(), "double")
.put(RepeatedString.getDescriptor(), "string")
.put(RepeatedObject.getDescriptor(), "object")
.build();

private static ImmutableMap<Descriptor, Message[]> AllRepeatedTypesToCorrectProto =
Expand All @@ -123,8 +121,8 @@ public class JsonToProtoMessageTest {
RepeatedBytes.getDescriptor(),
new Message[] {
RepeatedBytes.newBuilder()
.addTestRepeated(ByteString.copyFrom("hello".getBytes()))
.addTestRepeated(ByteString.copyFrom("test".getBytes()))
.addTestRepeated(ByteString.copyFrom(new byte[] {0}))
.addTestRepeated(ByteString.copyFrom(new byte[] {0, -116, -122, 71}))
.build(),
RepeatedBytes.newBuilder()
.addTestRepeated(
Expand Down Expand Up @@ -206,9 +204,10 @@ public class JsonToProtoMessageTest {
new JSONObject().put("test_field_type", Integer.MAX_VALUE),
new JSONObject().put("test_field_type", 1.23),
new JSONObject().put("test_field_type", true),
new JSONObject().put("test_field_type", "test"),
new JSONObject().put("test_field_type", ByteString.copyFromUtf8("test")),
new JSONObject().put("test_field_type", new JSONArray("[1, 2, 3]")),
new JSONObject().put("test_field_type", new JSONObject().put("test_int", 1))
new JSONObject().put("test_field_type", new JSONObject().put("test_int", 1)),
new JSONObject().put("test_field_type", "test")
};

private static JSONObject[] simpleJSONArrays = {
Expand Down Expand Up @@ -264,6 +263,7 @@ public class JsonToProtoMessageTest {
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.2"))
.toByteArray()
})),
new JSONObject().put("test_repeated", new JSONArray(new int[][] {{11111, 22222}})),
new JSONObject().put("test_repeated", new JSONArray(new char[][] {{'a', 'b'}, {'c'}})),
new JSONObject().put("test_repeated", new JSONArray(new String[][] {{"hello"}, {"test"}})),
new JSONObject()
Expand Down Expand Up @@ -350,8 +350,10 @@ public void testAllTypes() throws Exception {
int success = 0;
for (JSONObject json : simpleJSONObjects) {
try {
LOG.info("Testing " + json + " over " + entry.getKey().getFullName());
DynamicMessage protoMsg =
JsonToProtoMessage.convertJsonToProtoMessage(entry.getKey(), json);
LOG.info("Convert Success!");
assertEquals(protoMsg, AllTypesToCorrectProto.get(entry.getKey())[success]);
success += 1;
} catch (IllegalArgumentException e) {
Expand All @@ -361,40 +363,45 @@ public void testAllTypes() throws Exception {
}
}
if (entry.getKey() == Int64Type.getDescriptor()) {
assertEquals(2, success);
assertEquals(entry.getKey().getFullName(), 2, success);
} else {
assertEquals(1, success);
assertEquals(entry.getKey().getFullName(), 1, success);
}
}
}

@Test
public void testAllRepeatedTypesWithLimits() throws Exception {
for (Map.Entry<Descriptor, String[]> entry : AllRepeatedTypesToDebugMessageTest.entrySet()) {
for (Map.Entry<Descriptor, String> entry : AllRepeatedTypesToDebugMessageTest.entrySet()) {
int success = 0;
for (JSONObject json : simpleJSONArrays) {
try {
LOG.info("Testing " + json + " over " + entry.getKey().getFullName());
DynamicMessage protoMsg =
JsonToProtoMessage.convertJsonToProtoMessage(entry.getKey(), json);
assertEquals(protoMsg, AllRepeatedTypesToCorrectProto.get(entry.getKey())[success]);
LOG.info("Convert Success!");
assertEquals(
protoMsg.toString(),
protoMsg,
AllRepeatedTypesToCorrectProto.get(entry.getKey())[success]);
success += 1;
} catch (IllegalArgumentException e) {
LOG.info(e.getMessage());
assertTrue(
e.getMessage()
.equals(
"JSONObject does not have a "
+ entry.getValue()[0]
+ entry.getValue()
+ " field at root.test_repeated[0].")
|| e.getMessage()
.equals("Error: root.test_repeated[0] could not be converted to byte[]."));
}
}
if (entry.getKey() == RepeatedInt64.getDescriptor()
|| entry.getKey() == RepeatedDouble.getDescriptor()
|| entry.getKey() == RepeatedBytes.getDescriptor()) {
assertEquals(2, success);
|| entry.getKey() == RepeatedDouble.getDescriptor()) {
assertEquals(entry.getKey().getFullName(), 2, success);
} else {
assertEquals(1, success);
assertEquals(entry.getKey().getFullName(), 1, success);
}
}
}
Expand Down Expand Up @@ -501,7 +508,7 @@ public void testStructComplex() throws Exception {
JSONObject json = new JSONObject();
json.put("test_int", 1);
json.put("test_string", new JSONArray(new String[] {"a", "b", "c"}));
json.put("test_bytes", "hello");
json.put("test_bytes", ByteString.copyFromUtf8("hello"));
json.put("test_bool", true);
json.put("test_DOUBLe", new JSONArray(new Double[] {1.1, 2.2, 3.3, 4.4}));
json.put("test_date", 1);
Expand All @@ -525,7 +532,7 @@ public void testStructComplexFail() throws Exception {
JSONObject json = new JSONObject();
json.put("test_int", 1);
json.put("test_string", new JSONArray(new String[] {"a", "b", "c"}));
json.put("test_bytes", "hello");
json.put("test_bytes", ByteString.copyFromUtf8("hello"));
json.put("test_bool", true);
json.put("test_double", new JSONArray(new Double[] {1.1, 2.2, 3.3, 4.4}));
json.put("test_date", 1);
Expand Down
1 change: 1 addition & 0 deletions google-cloud-bigquerystorage/src/test/proto/jsonTest.proto
View file
Open in desktop
Expand Up @@ -15,6 +15,7 @@ message ComplexRoot {
optional string test_geo = 10;
optional int64 test_timestamp = 11;
optional int64 test_time = 12;
repeated bytes test_numeric_repeated = 13;
}

message CasingComplex {
Expand Down