From b1dd04d89df6339a9624378c31f9ab26a6114a54 Mon Sep 17 00:00:00 2001 From: Astha Mohta <35952883+asthamohta@users.noreply.github.com> Date: Thu, 26 Aug 2021 10:33:24 +0530 Subject: [PATCH] feat: add support for JSON type (#353) * add support for JSON- proto changes * adding json support-synth tool * deleting synth.metadata * Revert "add support for JSON- proto changes" This reverts commit a2f111c2ce6eef0e1a79a4c0c4c9852a07b86ae4. * json changes * json changes * json changes * sorting keys and adding separators * adding changes to system test case * removing extra spaces * lint changes * changes to test_session * changes for lint Co-authored-by: Zoe Co-authored-by: larkee <31196561+larkee@users.noreply.github.com> --- google/cloud/spanner_v1/_helpers.py | 2 ++ google/cloud/spanner_v1/param_types.py | 1 + google/cloud/spanner_v1/streamed.py | 1 + tests/_fixtures.py | 5 +++- tests/system/test_session_api.py | 33 ++++++++++++++++++++++++-- tests/unit/test__helpers.py | 25 +++++++++++++++++++ 6 files changed, 64 insertions(+), 3 deletions(-) diff --git a/google/cloud/spanner_v1/_helpers.py b/google/cloud/spanner_v1/_helpers.py index 2d1bf322bf..9f9233210d 100644 --- a/google/cloud/spanner_v1/_helpers.py +++ b/google/cloud/spanner_v1/_helpers.py @@ -244,6 +244,8 @@ def _parse_value_pb(value_pb, field_type): ] elif type_code == TypeCode.NUMERIC: return decimal.Decimal(value_pb.string_value) + elif type_code == TypeCode.JSON: + return value_pb.string_value else: raise ValueError("Unknown type: %s" % (field_type,)) diff --git a/google/cloud/spanner_v1/param_types.py b/google/cloud/spanner_v1/param_types.py index c5a106d0aa..4b72bb46e9 100644 --- a/google/cloud/spanner_v1/param_types.py +++ b/google/cloud/spanner_v1/param_types.py @@ -28,6 +28,7 @@ DATE = Type(code=TypeCode.DATE) TIMESTAMP = Type(code=TypeCode.TIMESTAMP) NUMERIC = Type(code=TypeCode.NUMERIC) +JSON = Type(code=TypeCode.JSON) def Array(element_type): # pylint: disable=invalid-name diff --git a/google/cloud/spanner_v1/streamed.py b/google/cloud/spanner_v1/streamed.py index 9ee04867b3..b502b19cea 100644 --- a/google/cloud/spanner_v1/streamed.py +++ b/google/cloud/spanner_v1/streamed.py @@ -316,6 +316,7 @@ def _merge_struct(lhs, rhs, type_): TypeCode.STRUCT: _merge_struct, TypeCode.TIMESTAMP: _merge_string, TypeCode.NUMERIC: _merge_string, + TypeCode.JSON: _merge_string, } diff --git a/tests/_fixtures.py b/tests/_fixtures.py index efca8a9042..e4cd929835 100644 --- a/tests/_fixtures.py +++ b/tests/_fixtures.py @@ -45,7 +45,10 @@ timestamp_value TIMESTAMP, timestamp_array ARRAY, numeric_value NUMERIC, - numeric_array ARRAY) + numeric_array ARRAY, + json_value JSON, + json_array ARRAY, + ) PRIMARY KEY (pkey); CREATE TABLE counters ( name STRING(1024), diff --git a/tests/system/test_session_api.py b/tests/system/test_session_api.py index 747c64a9c1..88a20a7a92 100644 --- a/tests/system/test_session_api.py +++ b/tests/system/test_session_api.py @@ -19,7 +19,7 @@ import struct import threading import time - +import json import pytest import grpc @@ -43,6 +43,24 @@ BYTES_2 = b"Ym9vdHM=" NUMERIC_1 = decimal.Decimal("0.123456789") NUMERIC_2 = decimal.Decimal("1234567890") +JSON_1 = json.dumps( + { + "sample_boolean": True, + "sample_int": 872163, + "sample float": 7871.298, + "sample_null": None, + "sample_string": "abcdef", + "sample_array": [23, 76, 19], + }, + sort_keys=True, + separators=(",", ":"), +) +JSON_2 = json.dumps( + {"sample_object": {"name": "Anamika", "id": 2635}}, + sort_keys=True, + separators=(",", ":"), +) + COUNTERS_TABLE = "counters" COUNTERS_COLUMNS = ("name", "value") ALL_TYPES_TABLE = "all_types" @@ -64,8 +82,10 @@ "timestamp_array", "numeric_value", "numeric_array", + "json_value", + "json_array", ) -EMULATOR_ALL_TYPES_COLUMNS = LIVE_ALL_TYPES_COLUMNS[:-2] +EMULATOR_ALL_TYPES_COLUMNS = LIVE_ALL_TYPES_COLUMNS[:-4] AllTypesRowData = collections.namedtuple("AllTypesRowData", LIVE_ALL_TYPES_COLUMNS) AllTypesRowData.__new__.__defaults__ = tuple([None for colum in LIVE_ALL_TYPES_COLUMNS]) EmulatorAllTypesRowData = collections.namedtuple( @@ -88,6 +108,7 @@ AllTypesRowData(pkey=107, timestamp_value=SOME_TIME), AllTypesRowData(pkey=108, timestamp_value=NANO_TIME), AllTypesRowData(pkey=109, numeric_value=NUMERIC_1), + AllTypesRowData(pkey=110, json_value=JSON_1), # empty array values AllTypesRowData(pkey=201, int_array=[]), AllTypesRowData(pkey=202, bool_array=[]), @@ -97,6 +118,7 @@ AllTypesRowData(pkey=206, string_array=[]), AllTypesRowData(pkey=207, timestamp_array=[]), AllTypesRowData(pkey=208, numeric_array=[]), + AllTypesRowData(pkey=209, json_array=[]), # non-empty array values, including nulls AllTypesRowData(pkey=301, int_array=[123, 456, None]), AllTypesRowData(pkey=302, bool_array=[True, False, None]), @@ -106,6 +128,7 @@ AllTypesRowData(pkey=306, string_array=["One", "Two", None]), AllTypesRowData(pkey=307, timestamp_array=[SOME_TIME, NANO_TIME, None]), AllTypesRowData(pkey=308, numeric_array=[NUMERIC_1, NUMERIC_2, None]), + AllTypesRowData(pkey=309, json_array=[JSON_1, JSON_2, None]), ) EMULATOR_ALL_TYPES_ROWDATA = ( # all nulls @@ -1867,6 +1890,12 @@ def test_execute_sql_w_numeric_bindings(not_emulator, sessions_database): ) +def test_execute_sql_w_json_bindings(not_emulator, sessions_database): + _bind_test_helper( + sessions_database, spanner_v1.TypeCode.JSON, JSON_1, [JSON_1, JSON_2], + ) + + def test_execute_sql_w_query_param_struct(sessions_database): name = "Phred" count = 123 diff --git a/tests/unit/test__helpers.py b/tests/unit/test__helpers.py index cfdcea1ea0..25556f36fb 100644 --- a/tests/unit/test__helpers.py +++ b/tests/unit/test__helpers.py @@ -295,6 +295,17 @@ def test_w_numeric_precision_and_scale_invalid(self): ValueError, err_msg, lambda: self._callFUT(value), ) + def test_w_json(self): + import json + from google.protobuf.struct_pb2 import Value + + value = json.dumps( + {"id": 27863, "Name": "Anamika"}, sort_keys=True, separators=(",", ":") + ) + value_pb = self._callFUT(value) + self.assertIsInstance(value_pb, Value) + self.assertEqual(value_pb.string_value, value) + class Test_make_list_value_pb(unittest.TestCase): def _callFUT(self, *args, **kw): @@ -552,6 +563,20 @@ def test_w_numeric(self): self.assertEqual(self._callFUT(value_pb, field_type), VALUE) + def test_w_json(self): + import json + from google.protobuf.struct_pb2 import Value + from google.cloud.spanner_v1 import Type + from google.cloud.spanner_v1 import TypeCode + + VALUE = json.dumps( + {"id": 27863, "Name": "Anamika"}, sort_keys=True, separators=(",", ":") + ) + field_type = Type(code=TypeCode.JSON) + value_pb = Value(string_value=VALUE) + + self.assertEqual(self._callFUT(value_pb, field_type), VALUE) + def test_w_unknown_type(self): from google.protobuf.struct_pb2 import Value from google.cloud.spanner_v1 import Type