From 68bd2ba9e695cc3b5b8a16c66a2da6a4dfff91f6 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Fri, 16 Apr 2021 16:15:07 +0200 Subject: [PATCH 1/3] fix: consistent percents handling in DB API query Percents in the query string are now always de-escaped, regardless of whether any query parameters are passed or not. In addition, misformatting placeholders that don't match parameter values now consistently raise ProgrammingError. --- google/cloud/bigquery/dbapi/cursor.py | 6 +-- tests/unit/test_dbapi_cursor.py | 53 +++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/google/cloud/bigquery/dbapi/cursor.py b/google/cloud/bigquery/dbapi/cursor.py index ee09158d8..5b35cb70d 100644 --- a/google/cloud/bigquery/dbapi/cursor.py +++ b/google/cloud/bigquery/dbapi/cursor.py @@ -389,7 +389,7 @@ def _format_operation_list(operation, parameters): try: return operation % tuple(formatted_params) - except TypeError as exc: + except (TypeError, ValueError) as exc: raise exceptions.ProgrammingError(exc) @@ -419,7 +419,7 @@ def _format_operation_dict(operation, parameters): try: return operation % formatted_params - except KeyError as exc: + except (KeyError, ValueError) as exc: raise exceptions.ProgrammingError(exc) @@ -441,7 +441,7 @@ def _format_operation(operation, parameters=None): ``parameters`` argument. """ if parameters is None or len(parameters) == 0: - return operation + return operation.replace("%%", "%") # Still do precents de-escaping. if isinstance(parameters, collections_abc.Mapping): return _format_operation_dict(operation, parameters) diff --git a/tests/unit/test_dbapi_cursor.py b/tests/unit/test_dbapi_cursor.py index 0f44e3895..7eab28d79 100644 --- a/tests/unit/test_dbapi_cursor.py +++ b/tests/unit/test_dbapi_cursor.py @@ -633,6 +633,14 @@ def test__format_operation_w_wrong_dict(self): {"somevalue-not-here": "hi", "othervalue": "world"}, ) + def test__format_operation_w_redundant_dict_key(self): + from google.cloud.bigquery.dbapi import cursor + + formatted_operation = cursor._format_operation( + "SELECT %(somevalue)s;", {"somevalue": "foo", "value-not-used": "bar"} + ) + self.assertEqual(formatted_operation, "SELECT @`somevalue`;") + def test__format_operation_w_sequence(self): from google.cloud.bigquery.dbapi import cursor @@ -652,8 +660,53 @@ def test__format_operation_w_too_short_sequence(self): ("hello",), ) + def test__format_operation_w_too_long_sequence(self): + from google.cloud.bigquery import dbapi + from google.cloud.bigquery.dbapi import cursor + + self.assertRaises( + dbapi.ProgrammingError, + cursor._format_operation, + "SELECT %s, %s;", + ("hello", "world", "everyone"), + ) + def test__format_operation_w_empty_dict(self): from google.cloud.bigquery.dbapi import cursor formatted_operation = cursor._format_operation("SELECT '%f'", {}) self.assertEqual(formatted_operation, "SELECT '%f'") + + def test__format_operation_wo_params_single_percent(self): + from google.cloud.bigquery.dbapi import cursor + + formatted_operation = cursor._format_operation("SELECT '%'", {}) + self.assertEqual(formatted_operation, "SELECT '%'") + + def test__format_operation_wo_params_double_percents(self): + from google.cloud.bigquery.dbapi import cursor + + formatted_operation = cursor._format_operation("SELECT '%%'", {}) + self.assertEqual(formatted_operation, "SELECT '%'") + + def test__format_operation_unescaped_percent_w_dict_param(self): + from google.cloud.bigquery import dbapi + from google.cloud.bigquery.dbapi import cursor + + self.assertRaises( + dbapi.ProgrammingError, + cursor._format_operation, + "SELECT '100 %';", + {"foo": "bar"}, + ) + + def test__format_operation_unescaped_percent_w_list_param(self): + from google.cloud.bigquery import dbapi + from google.cloud.bigquery.dbapi import cursor + + self.assertRaises( + dbapi.ProgrammingError, + cursor._format_operation, + "SELECT '100 %';", + ["foo", "bar"], + ) From 42f14e4225e0bd7b6f69fb4d18f6355fe6a043a2 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Fri, 16 Apr 2021 17:21:48 +0200 Subject: [PATCH 2/3] Fix typo in comment --- google/cloud/bigquery/dbapi/cursor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/bigquery/dbapi/cursor.py b/google/cloud/bigquery/dbapi/cursor.py index 5b35cb70d..7d9b53eea 100644 --- a/google/cloud/bigquery/dbapi/cursor.py +++ b/google/cloud/bigquery/dbapi/cursor.py @@ -441,7 +441,7 @@ def _format_operation(operation, parameters=None): ``parameters`` argument. """ if parameters is None or len(parameters) == 0: - return operation.replace("%%", "%") # Still do precents de-escaping. + return operation.replace("%%", "%") # Still do percent de-escaping. if isinstance(parameters, collections_abc.Mapping): return _format_operation_dict(operation, parameters) From e50cba2fc34f46b015c8b955f918b8a2b54954ab Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Fri, 16 Apr 2021 17:37:19 +0200 Subject: [PATCH 3/3] Handle type errors with dict params formatting --- google/cloud/bigquery/dbapi/cursor.py | 2 +- tests/unit/test_dbapi_cursor.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/google/cloud/bigquery/dbapi/cursor.py b/google/cloud/bigquery/dbapi/cursor.py index 7d9b53eea..039adbfcd 100644 --- a/google/cloud/bigquery/dbapi/cursor.py +++ b/google/cloud/bigquery/dbapi/cursor.py @@ -419,7 +419,7 @@ def _format_operation_dict(operation, parameters): try: return operation % formatted_params - except (KeyError, ValueError) as exc: + except (KeyError, ValueError, TypeError) as exc: raise exceptions.ProgrammingError(exc) diff --git a/tests/unit/test_dbapi_cursor.py b/tests/unit/test_dbapi_cursor.py index 7eab28d79..56e49d51b 100644 --- a/tests/unit/test_dbapi_cursor.py +++ b/tests/unit/test_dbapi_cursor.py @@ -696,7 +696,7 @@ def test__format_operation_unescaped_percent_w_dict_param(self): self.assertRaises( dbapi.ProgrammingError, cursor._format_operation, - "SELECT '100 %';", + "SELECT %(foo)s, '100 %';", {"foo": "bar"}, ) @@ -707,6 +707,6 @@ def test__format_operation_unescaped_percent_w_list_param(self): self.assertRaises( dbapi.ProgrammingError, cursor._format_operation, - "SELECT '100 %';", + "SELECT %s, %s, '100 %';", ["foo", "bar"], )