diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/JsonToProtoMessage.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/JsonToProtoMessage.java index eb18f780af..46cdacad77 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/JsonToProtoMessage.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1alpha2/JsonToProtoMessage.java @@ -19,6 +19,7 @@ import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.DynamicMessage; import com.google.protobuf.Message; +import java.util.HashMap; import java.util.List; import org.json.JSONArray; import org.json.JSONException; @@ -77,15 +78,21 @@ private static DynamicMessage convertJsonToProtoMessageImpl( if (JSONObject.getNames(json).length > protoFields.size()) { if (!allowUnknownFields) { throw new IllegalArgumentException( - "JSONObject has unknown fields. Set allowUnknownFields to True to ignore unknown fields."); + "JSONObject has fields unknown to BigQuery: f1. Set allowUnknownFields to True to allow unknown fields."); } } + HashMap jsonLowercaseNameToActualName = new HashMap(); + String[] actualNames = JSONObject.getNames(json); + for (int i = 0; i < actualNames.length; i++) { + jsonLowercaseNameToActualName.put(actualNames[i].toLowerCase(), actualNames[i]); + } for (FieldDescriptor field : protoFields) { String fieldName = field.getName(); + String lowercaseFieldName = fieldName.toLowerCase(); String currentScope = jsonScope + "." + fieldName; - if (!json.has(fieldName)) { + if (!jsonLowercaseNameToActualName.containsKey(lowercaseFieldName)) { if (field.isRequired()) { throw new IllegalArgumentException( "JSONObject does not have the required field " + currentScope + "."); @@ -95,9 +102,21 @@ private static DynamicMessage convertJsonToProtoMessageImpl( } matchedFields++; if (!field.isRepeated()) { - fillField(protoMsg, field, json, currentScope, allowUnknownFields); + fillField( + protoMsg, + field, + json, + jsonLowercaseNameToActualName.get(lowercaseFieldName), + currentScope, + allowUnknownFields); } else { - fillRepeatedField(protoMsg, field, json, currentScope, allowUnknownFields); + fillRepeatedField( + protoMsg, + field, + json, + jsonLowercaseNameToActualName.get(lowercaseFieldName), + currentScope, + allowUnknownFields); } } if (matchedFields == 0 && topLevel) { @@ -110,26 +129,27 @@ private static DynamicMessage convertJsonToProtoMessageImpl( /** * Fills a non-repetaed protoField with the json data. * - * @param protoMsg - * @param field + * @param protoMsg The protocol buffer message being constructed + * @param fieldDescriptor * @param json + * @param actualJsonKeyName Actual key name in JSONObject instead of lowercased version * @param currentScope Debugging purposes * @param allowUnknownFields Ignores unknown JSON fields. * @throws IllegalArgumentException when JSON data is not compatible with proto descriptor. */ private static void fillField( DynamicMessage.Builder protoMsg, - FieldDescriptor field, + FieldDescriptor fieldDescriptor, JSONObject json, + String actualJsonKeyName, String currentScope, boolean allowUnknownFields) throws IllegalArgumentException { - String fieldName = field.getName(); - switch (field.getType()) { + switch (fieldDescriptor.getType()) { case BOOL: try { - protoMsg.setField(field, new Boolean(json.getBoolean(fieldName))); + protoMsg.setField(fieldDescriptor, new Boolean(json.getBoolean(actualJsonKeyName))); } catch (JSONException e) { throw new IllegalArgumentException( "JSONObject does not have a boolean field at " + currentScope + "."); @@ -137,7 +157,7 @@ private static void fillField( break; case BYTES: try { - protoMsg.setField(field, json.getString(fieldName).getBytes()); + protoMsg.setField(fieldDescriptor, json.getString(actualJsonKeyName).getBytes()); } catch (JSONException e) { throw new IllegalArgumentException( "JSONObject does not have a string field at " + currentScope + "."); @@ -145,15 +165,15 @@ private static void fillField( break; case INT64: try { - java.lang.Object val = json.get(fieldName); + java.lang.Object val = json.get(actualJsonKeyName); if (val instanceof Byte) { - protoMsg.setField(field, new Long((Byte) val)); + protoMsg.setField(fieldDescriptor, new Long((Byte) val)); } else if (val instanceof Short) { - protoMsg.setField(field, new Long((Short) val)); + protoMsg.setField(fieldDescriptor, new Long((Short) val)); } else if (val instanceof Integer) { - protoMsg.setField(field, new Long((Integer) val)); + protoMsg.setField(fieldDescriptor, new Long((Integer) val)); } else if (val instanceof Long) { - protoMsg.setField(field, new Long((Long) val)); + protoMsg.setField(fieldDescriptor, new Long((Long) val)); } else { throw new IllegalArgumentException( "JSONObject does not have a int64 field at " + currentScope + "."); @@ -163,9 +183,27 @@ private static void fillField( "JSONObject does not have a int64 field at " + currentScope + "."); } break; + case INT32: + try { + java.lang.Object val = json.get(actualJsonKeyName); + if (val instanceof Byte) { + protoMsg.setField(fieldDescriptor, new Integer((Byte) val)); + } else if (val instanceof Short) { + protoMsg.setField(fieldDescriptor, new Integer((Short) val)); + } else if (val instanceof Integer) { + protoMsg.setField(fieldDescriptor, new Integer((Integer) val)); + } else { + throw new IllegalArgumentException( + "JSONObject does not have a int32 field at " + currentScope + "."); + } + } catch (JSONException e) { + throw new IllegalArgumentException( + "JSONObject does not have a int32 field at " + currentScope + "."); + } + break; case STRING: try { - protoMsg.setField(field, json.getString(fieldName)); + protoMsg.setField(fieldDescriptor, json.getString(actualJsonKeyName)); } catch (JSONException e) { throw new IllegalArgumentException( "JSONObject does not have a string field at " + currentScope + "."); @@ -173,11 +211,11 @@ private static void fillField( break; case DOUBLE: try { - java.lang.Object val = json.get(fieldName); + java.lang.Object val = json.get(actualJsonKeyName); if (val instanceof Double) { - protoMsg.setField(field, new Double((double) val)); + protoMsg.setField(fieldDescriptor, new Double((double) val)); } else if (val instanceof Float) { - protoMsg.setField(field, new Double((float) val)); + protoMsg.setField(fieldDescriptor, new Double((float) val)); } else { throw new IllegalArgumentException( "JSONObject does not have a double field at " + currentScope + "."); @@ -188,13 +226,13 @@ private static void fillField( } break; case MESSAGE: - Message.Builder message = protoMsg.newBuilderForField(field); + Message.Builder message = protoMsg.newBuilderForField(fieldDescriptor); try { protoMsg.setField( - field, + fieldDescriptor, convertJsonToProtoMessageImpl( - field.getMessageType(), - json.getJSONObject(fieldName), + fieldDescriptor.getMessageType(), + json.getJSONObject(actualJsonKeyName), currentScope, false, allowUnknownFields)); @@ -209,34 +247,36 @@ private static void fillField( /** * Fills a repeated protoField with the json data. * - * @param protoMsg - * @param field + * @param protoMsg The protocol buffer message being constructed + * @param fieldDescriptor * @param json If root level has no matching fields, throws exception. + * @param actualJsonKeyName Actual key name in JSONObject instead of lowercased version * @param currentScope Debugging purposes * @param allowUnknownFields Ignores unknown JSON fields. * @throws IllegalArgumentException when JSON data is not compatible with proto descriptor. */ private static void fillRepeatedField( DynamicMessage.Builder protoMsg, - FieldDescriptor field, + FieldDescriptor fieldDescriptor, JSONObject json, + String actualJsonKeyName, String currentScope, boolean allowUnknownFields) throws IllegalArgumentException { - String fieldName = field.getName(); + JSONArray jsonArray; try { - jsonArray = json.getJSONArray(fieldName); + jsonArray = json.getJSONArray(actualJsonKeyName); } catch (JSONException e) { throw new IllegalArgumentException( "JSONObject does not have a array field at " + currentScope + "."); } - switch (field.getType()) { + switch (fieldDescriptor.getType()) { case BOOL: for (int i = 0; i < jsonArray.length(); i++) { try { - protoMsg.addRepeatedField(field, new Boolean(jsonArray.getBoolean(i))); + protoMsg.addRepeatedField(fieldDescriptor, new Boolean(jsonArray.getBoolean(i))); } catch (JSONException e) { throw new IllegalArgumentException( "JSONObject does not have a boolean field at " @@ -251,7 +291,7 @@ private static void fillRepeatedField( case BYTES: for (int i = 0; i < jsonArray.length(); i++) { try { - protoMsg.addRepeatedField(field, jsonArray.getString(i).getBytes()); + protoMsg.addRepeatedField(fieldDescriptor, jsonArray.getString(i).getBytes()); } catch (JSONException e) { throw new IllegalArgumentException( "JSONObject does not have a string field at " + currentScope + "[" + i + "]" + "."); @@ -263,13 +303,13 @@ private static void fillRepeatedField( try { java.lang.Object val = jsonArray.get(i); if (val instanceof Byte) { - protoMsg.addRepeatedField(field, new Long((Byte) val)); + protoMsg.addRepeatedField(fieldDescriptor, new Long((Byte) val)); } else if (val instanceof Short) { - protoMsg.addRepeatedField(field, new Long((Short) val)); + protoMsg.addRepeatedField(fieldDescriptor, new Long((Short) val)); } else if (val instanceof Integer) { - protoMsg.addRepeatedField(field, new Long((Integer) val)); + protoMsg.addRepeatedField(fieldDescriptor, new Long((Integer) val)); } else if (val instanceof Long) { - protoMsg.addRepeatedField(field, new Long((Long) val)); + protoMsg.addRepeatedField(fieldDescriptor, new Long((Long) val)); } else { throw new IllegalArgumentException( "JSONObject does not have a int64 field at " @@ -285,10 +325,35 @@ private static void fillRepeatedField( } } break; + case INT32: + for (int i = 0; i < jsonArray.length(); i++) { + try { + java.lang.Object val = jsonArray.get(i); + if (val instanceof Byte) { + protoMsg.addRepeatedField(fieldDescriptor, new Integer((Byte) val)); + } else if (val instanceof Short) { + protoMsg.addRepeatedField(fieldDescriptor, new Integer((Short) val)); + } else if (val instanceof Integer) { + protoMsg.addRepeatedField(fieldDescriptor, new Integer((Integer) val)); + } else { + throw new IllegalArgumentException( + "JSONObject does not have a int32 field at " + + currentScope + + "[" + + i + + "]" + + "."); + } + } catch (JSONException e) { + throw new IllegalArgumentException( + "JSONObject does not have a int32 field at " + currentScope + "[" + i + "]" + "."); + } + } + break; case STRING: for (int i = 0; i < jsonArray.length(); i++) { try { - protoMsg.addRepeatedField(field, jsonArray.getString(i)); + protoMsg.addRepeatedField(fieldDescriptor, jsonArray.getString(i)); } catch (JSONException e) { throw new IllegalArgumentException( "JSONObject does not have a string field at " + currentScope + "[" + i + "]" + "."); @@ -300,9 +365,9 @@ private static void fillRepeatedField( try { java.lang.Object val = jsonArray.get(i); if (val instanceof Double) { - protoMsg.addRepeatedField(field, new Double((double) val)); + protoMsg.addRepeatedField(fieldDescriptor, new Double((double) val)); } else if (val instanceof Float) { - protoMsg.addRepeatedField(field, new Double((float) val)); + protoMsg.addRepeatedField(fieldDescriptor, new Double((float) val)); } else { throw new IllegalArgumentException( "JSONObject does not have a double field at " @@ -321,11 +386,11 @@ private static void fillRepeatedField( case MESSAGE: for (int i = 0; i < jsonArray.length(); i++) { try { - Message.Builder message = protoMsg.newBuilderForField(field); + Message.Builder message = protoMsg.newBuilderForField(fieldDescriptor); protoMsg.addRepeatedField( - field, + fieldDescriptor, convertJsonToProtoMessageImpl( - field.getMessageType(), + fieldDescriptor.getMessageType(), jsonArray.getJSONObject(i), currentScope, false, diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/JsonToProtoMessageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/JsonToProtoMessageTest.java index 5468ba92eb..14a7c349b0 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/JsonToProtoMessageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1alpha2/JsonToProtoMessageTest.java @@ -25,6 +25,7 @@ import com.google.protobuf.Descriptors.FieldDescriptor; import com.google.protobuf.DynamicMessage; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import org.json.JSONArray; @@ -40,6 +41,7 @@ public class JsonToProtoMessageTest { .put(BoolType.getDescriptor(), "boolean") .put(BytesType.getDescriptor(), "string") .put(Int64Type.getDescriptor(), "int64") + .put(Int32Type.getDescriptor(), "int32") .put(DoubleType.getDescriptor(), "double") .put(StringType.getDescriptor(), "string") .put(RepeatedType.getDescriptor(), "array") @@ -51,12 +53,14 @@ public class JsonToProtoMessageTest { .put(RepeatedBool.getDescriptor(), "boolean") .put(RepeatedBytes.getDescriptor(), "string") .put(RepeatedInt64.getDescriptor(), "int64") + .put(RepeatedInt32.getDescriptor(), "int32") .put(RepeatedDouble.getDescriptor(), "double") .put(RepeatedObject.getDescriptor(), "object") .build(); private static JSONObject[] simpleJSONObjects = { - new JSONObject().put("test_field_type", 123), + new JSONObject().put("test_field_type", Long.MAX_VALUE), + new JSONObject().put("test_field_type", Integer.MAX_VALUE), new JSONObject().put("test_field_type", 1.23), new JSONObject().put("test_field_type", true), new JSONObject().put("test_field_type", "test"), @@ -80,6 +84,19 @@ public class JsonToProtoMessageTest { (long) Byte.MIN_VALUE, 0L })), + new JSONObject() + .put( + "test_repeated", + new JSONArray( + new Integer[] { + (int) Integer.MAX_VALUE, + (int) Integer.MIN_VALUE, + (int) Short.MAX_VALUE, + (int) Short.MIN_VALUE, + (int) Byte.MAX_VALUE, + (int) Byte.MIN_VALUE, + 0 + })), new JSONObject() .put( "test_repeated", @@ -104,20 +121,28 @@ public class JsonToProtoMessageTest { }; private void AreMatchingFieldsFilledIn(DynamicMessage proto, JSONObject json) { + HashMap jsonLowercaseNameToActualName = new HashMap(); + String[] actualNames = JSONObject.getNames(json); + for (int i = 0; i < actualNames.length; i++) { + jsonLowercaseNameToActualName.put(actualNames[i].toLowerCase(), actualNames[i]); + } for (Map.Entry entry : proto.getAllFields().entrySet()) { FieldDescriptor key = entry.getKey(); java.lang.Object value = entry.getValue(); if (key.isRepeated()) { - isProtoArrayJsonArrayEqual(key, value, json); + isProtoArrayJsonArrayEqual(key, value, json, jsonLowercaseNameToActualName); } else { - isProtoFieldJsonFieldEqual(key, value, json); + isProtoFieldJsonFieldEqual(key, value, json, jsonLowercaseNameToActualName); } } } private void isProtoFieldJsonFieldEqual( - FieldDescriptor key, java.lang.Object value, JSONObject json) { - String fieldName = key.getName(); + FieldDescriptor key, + java.lang.Object value, + JSONObject json, + HashMap jsonLowercaseNameToActualName) { + String fieldName = jsonLowercaseNameToActualName.get(key.getName().toLowerCase()); switch (key.getType()) { case BOOL: assertTrue((Boolean) value == json.getBoolean(fieldName)); @@ -128,6 +153,9 @@ private void isProtoFieldJsonFieldEqual( case INT64: assertTrue((long) value == json.getLong(fieldName)); break; + case INT32: + assertTrue((int) value == json.getInt(fieldName)); + break; case STRING: assertTrue(((String) value).equals(json.getString(fieldName))); break; @@ -141,8 +169,11 @@ private void isProtoFieldJsonFieldEqual( } private void isProtoArrayJsonArrayEqual( - FieldDescriptor key, java.lang.Object value, JSONObject json) { - String fieldName = key.getName(); + FieldDescriptor key, + java.lang.Object value, + JSONObject json, + HashMap jsonLowercaseNameToActualName) { + String fieldName = jsonLowercaseNameToActualName.get(key.getName().toLowerCase()); JSONArray jsonArray = json.getJSONArray(fieldName); switch (key.getType()) { case BOOL: @@ -163,6 +194,12 @@ private void isProtoArrayJsonArrayEqual( assertTrue((longArr.get(i) == jsonArray.getLong(i))); } break; + case INT32: + List intArr = (List) value; + for (int i = 0; i < jsonArray.length(); i++) { + assertTrue((intArr.get(i) == jsonArray.getInt(i))); + } + break; case STRING: List stringArr = (List) value; for (int i = 0; i < jsonArray.length(); i++) { @@ -184,6 +221,18 @@ private void isProtoArrayJsonArrayEqual( } } + @Test + public void testDifferentNameCasing() throws Exception { + JSONObject json = new JSONObject(); + json.put("bYtE", (byte) 1); + json.put("SHORT", (short) 1); + json.put("inT", 1); + json.put("lONg", 1L); + DynamicMessage protoMsg = + JsonToProtoMessage.convertJsonToProtoMessage(TestInt64.getDescriptor(), json); + AreMatchingFieldsFilledIn(protoMsg, json); + } + @Test public void testInt64() throws Exception { JSONObject json = new JSONObject(); @@ -196,6 +245,31 @@ public void testInt64() throws Exception { AreMatchingFieldsFilledIn(protoMsg, json); } + @Test + public void testInt32() throws Exception { + JSONObject json = new JSONObject(); + json.put("byte", (byte) 1); + json.put("short", (short) 1); + json.put("int", 1); + DynamicMessage protoMsg = + JsonToProtoMessage.convertJsonToProtoMessage(TestInt32.getDescriptor(), json); + AreMatchingFieldsFilledIn(protoMsg, json); + } + + @Test + public void testInt32NotMatchInt64() throws Exception { + JSONObject json = new JSONObject(); + json.put("byte", (byte) 1); + json.put("short", (short) 1); + json.put("int", 1L); + try { + DynamicMessage protoMsg = + JsonToProtoMessage.convertJsonToProtoMessage(TestInt32.getDescriptor(), json); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), "JSONObject does not have a int32 field at root.int."); + } + } + @Test public void testDouble() throws Exception { JSONObject json = new JSONObject(); @@ -222,7 +296,11 @@ public void testAllTypes() throws Exception { "JSONObject does not have a " + entry.getValue() + " field at root.test_field_type."); } } - assertEquals(1, success); + if (entry.getKey() == Int64Type.getDescriptor()) { + assertEquals(2, success); + } else { + assertEquals(1, success); + } } } @@ -243,7 +321,11 @@ public void testAllRepeatedTypesWithLimits() throws Exception { + " field at root.test_repeated[0]."); } } - assertEquals(1, success); + if (entry.getKey() == RepeatedInt64.getDescriptor()) { + assertEquals(2, success); + } else { + assertEquals(1, success); + } } } @@ -323,9 +405,9 @@ public void testStructComplex() throws Exception { json.put("test_string", new JSONArray(new String[] {"a", "b", "c"})); json.put("test_bytes", "hello"); json.put("test_bool", true); - json.put("test_double", new JSONArray(new Double[] {1.1, 2.2, 3.3, 4.4})); + json.put("test_DOUBLe", new JSONArray(new Double[] {1.1, 2.2, 3.3, 4.4})); json.put("complexLvl1", complexLvl1); - json.put("complexLvl2", complexLvl2); + json.put("complexLVL2", complexLvl2); DynamicMessage protoMsg = JsonToProtoMessage.convertJsonToProtoMessage(ComplexRoot.getDescriptor(), json); @@ -436,7 +518,7 @@ public void testAllowUnknownFieldsError() throws Exception { } catch (IllegalArgumentException e) { assertEquals( e.getMessage(), - "JSONObject has unknown fields. Set allowUnknownFields to True to ignore unknown fields."); + "JSONObject has fields unknown to BigQuery: f1. Set allowUnknownFields to True to allow unknown fields."); } } diff --git a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto index b7d3c2b152..141b5abe9a 100644 --- a/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto +++ b/google-cloud-bigquerystorage/src/test/proto/jsonTest.proto @@ -54,6 +54,10 @@ message RepeatedInt64 { repeated int64 test_repeated = 1; } +message RepeatedInt32 { + repeated int32 test_repeated = 1; +} + message RepeatedDouble { repeated double test_repeated = 1; } @@ -81,6 +85,12 @@ message TestInt64 { optional int64 long = 4; } +message TestInt32 { + optional int32 byte = 1; + optional int32 short = 2; + optional int32 int = 3; +} + message TestDouble { optional double double = 1; optional double float = 2;