Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: add BIGNUMERIC support #527

Merged
merged 9 commits into from Feb 23, 2021
Merged

feat: add BIGNUMERIC support #527

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
8f0b338
feat: add support of BIGNUMERIC
HemangChothani Nov 19, 2020
c3cb1d0
feat: resolve conflict
HemangChothani Dec 22, 2020
7be79b2
feat: add BIGNUMERIC support
HemangChothani Dec 22, 2020
c2b3594
Merge branch 'master' into iss-367
plamut Feb 17, 2021
fcfcd04
Add bignumeric_type extra
plamut Feb 17, 2021
21ae123
Add additional BIGNUMERIC tests
plamut Feb 18, 2021
5a2fc40
Prevent import time error if no BIGNUMERIC support
plamut Feb 18, 2021
88a5303
Add/improve a few comments
plamut Feb 19, 2021
9af4327
Add feature flag for BIGNUMERIC suppport
plamut Feb 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 9 additions & 1 deletion google/cloud/bigquery/_pandas_helpers.py
View file
Open in desktop
Expand Up @@ -80,6 +80,10 @@ def pyarrow_numeric():
return pyarrow.decimal128(38, 9)


def pyarrow_bignumeric():
return pyarrow.decimal256(76, 38)


def pyarrow_time():
return pyarrow.time64("us")

Expand Down Expand Up @@ -129,9 +133,13 @@ def pyarrow_timestamp():
pyarrow.binary().id: "BYTES",
pyarrow.string().id: "STRING", # also alias for pyarrow.utf8()
pyarrow.decimal128(38, scale=9).id: "NUMERIC",
}

if int(pyarrow.__version__.split(".")[0]) >= 3:
plamut marked this conversation as resolved.
Show resolved Hide resolved
BQ_TO_ARROW_SCALARS["BIGNUMERIC"] = pyarrow_bignumeric
# The exact decimal's scale and precision are not important, as only
# the type ID matters, and it's the same for all decimal128 instances.
plamut marked this conversation as resolved.
Show resolved Hide resolved
}
ARROW_SCALAR_IDS_TO_BQ[pyarrow.decimal256(76, scale=38).id] = "BIGNUMERIC"

else: # pragma: NO COVER
BQ_TO_ARROW_SCALARS = {} # pragma: NO COVER
Expand Down
11 changes: 10 additions & 1 deletion google/cloud/bigquery/dbapi/_helpers.py
View file
Open in desktop
Expand Up @@ -19,6 +19,11 @@
import functools
import numbers

try:
import pyarrow
except ImportError: # pragma: NO COVER
pyarrow = None

from google.cloud import bigquery
from google.cloud.bigquery import table
from google.cloud.bigquery.dbapi import exceptions
Expand Down Expand Up @@ -184,7 +189,11 @@ def bigquery_scalar_type(value):
elif isinstance(value, numbers.Real):
return "FLOAT64"
elif isinstance(value, decimal.Decimal):
return "NUMERIC"
scalar_object = pyarrow.scalar(value)
if isinstance(scalar_object, pyarrow.Decimal128Scalar):
plamut marked this conversation as resolved.
Show resolved Hide resolved
return "NUMERIC"
else:
return "BIGNUMERIC"
elif isinstance(value, str):
return "STRING"
elif isinstance(value, bytes):
Expand Down
2 changes: 1 addition & 1 deletion google/cloud/bigquery/dbapi/types.py
View file
Open in desktop
Expand Up @@ -78,7 +78,7 @@ def __eq__(self, other):
STRING = "STRING"
BINARY = _DBAPITypeObject("BYTES", "RECORD", "STRUCT")
NUMBER = _DBAPITypeObject(
"INTEGER", "INT64", "FLOAT", "FLOAT64", "NUMERIC", "BOOLEAN", "BOOL"
"INTEGER", "INT64", "FLOAT", "FLOAT64", "NUMERIC", "BIGNUMERIC", "BOOLEAN", "BOOL"
)
DATETIME = _DBAPITypeObject("TIMESTAMP", "DATE", "TIME", "DATETIME")
ROWID = "ROWID"
8 changes: 4 additions & 4 deletions google/cloud/bigquery/query.py
View file
Open in desktop
Expand Up @@ -83,7 +83,7 @@ class ScalarQueryParameter(_AbstractQueryParameter):

type_ (str):
Name of parameter type. One of 'STRING', 'INT64',
'FLOAT64', 'NUMERIC', 'BOOL', 'TIMESTAMP', 'DATETIME', or
'FLOAT64', 'NUMERIC', 'BIGNUMERIC', 'BOOL', 'TIMESTAMP', 'DATETIME', or
'DATE'.

value (Union[str, int, float, decimal.Decimal, bool, datetime.datetime, datetime.date]):
Expand All @@ -102,7 +102,7 @@ def positional(cls, type_, value):
Args:
type_ (str):
Name of parameter type. One of 'STRING', 'INT64',
'FLOAT64', 'NUMERIC', 'BOOL', 'TIMESTAMP', 'DATETIME', or
'FLOAT64', 'NUMERIC', 'BIGNUMERIC', 'BOOL', 'TIMESTAMP', 'DATETIME', or
'DATE'.

value (Union[str, int, float, decimal.Decimal, bool, datetime.datetime, datetime.date]):
Expand Down Expand Up @@ -186,7 +186,7 @@ class ArrayQueryParameter(_AbstractQueryParameter):

array_type (str):
Name of type of array elements. One of `'STRING'`, `'INT64'`,
`'FLOAT64'`, `'NUMERIC'`, `'BOOL'`, `'TIMESTAMP'`, or `'DATE'`.
`'FLOAT64'`, `'NUMERIC'`, `'BIGNUMERIC'`, `'BOOL'`, `'TIMESTAMP'`, or `'DATE'`.

values (List[appropriate scalar type]): The parameter array values.
"""
Expand All @@ -203,7 +203,7 @@ def positional(cls, array_type, values):
Args:
array_type (str):
Name of type of array elements. One of `'STRING'`, `'INT64'`,
`'FLOAT64'`, `'NUMERIC'`, `'BOOL'`, `'TIMESTAMP'`, or `'DATE'`.
`'FLOAT64'`, `'NUMERIC'`, `'BIGNUMERIC'`, `'BOOL'`, `'TIMESTAMP'`, or `'DATE'`.

values (List[appropriate scalar type]): The parameter array values.

Expand Down
1 change: 1 addition & 0 deletions google/cloud/bigquery/schema.py
View file
Open in desktop
Expand Up @@ -32,6 +32,7 @@
"FLOAT": types.StandardSqlDataType.TypeKind.FLOAT64,
"FLOAT64": types.StandardSqlDataType.TypeKind.FLOAT64,
"NUMERIC": types.StandardSqlDataType.TypeKind.NUMERIC,
"BIGNUMERIC": types.StandardSqlDataType.TypeKind.BIGNUMERIC,
"BOOLEAN": types.StandardSqlDataType.TypeKind.BOOL,
"BOOL": types.StandardSqlDataType.TypeKind.BOOL,
"GEOGRAPHY": types.StandardSqlDataType.TypeKind.GEOGRAPHY,
Expand Down
1 change: 1 addition & 0 deletions setup.py
View file
Open in desktop
Expand Up @@ -48,6 +48,7 @@
"pyarrow >= 1.0.0, < 4.0dev",
],
"pandas": ["pandas>=0.23.0", "pyarrow >= 1.0.0, < 4.0dev",],
"bignumeric_type": ["pyarrow >= 3.0.0, < 4.0dev"],
plamut marked this conversation as resolved.
Show resolved Hide resolved
"tqdm": ["tqdm >= 4.7.4, <5.0.0dev"],
"opentelemetry": [
"opentelemetry-api==0.11b0",
Expand Down
29 changes: 29 additions & 0 deletions tests/system/test_client.py
View file
Open in desktop
Expand Up @@ -887,6 +887,7 @@ def test_load_table_from_dataframe_w_nulls(self):
bigquery.SchemaField("geo_col", "GEOGRAPHY"),
bigquery.SchemaField("int_col", "INTEGER"),
bigquery.SchemaField("num_col", "NUMERIC"),
bigquery.SchemaField("bignum_col", "BIGNUMERIC"),
bigquery.SchemaField("str_col", "STRING"),
bigquery.SchemaField("time_col", "TIME"),
bigquery.SchemaField("ts_col", "TIMESTAMP"),
Expand All @@ -912,6 +913,7 @@ def test_load_table_from_dataframe_w_nulls(self):
("geo_col", nulls),
("int_col", nulls),
("num_col", nulls),
("bignum_col", nulls),
("str_col", nulls),
("time_col", nulls),
("ts_col", nulls),
Expand Down Expand Up @@ -999,6 +1001,7 @@ def test_load_table_from_dataframe_w_explicit_schema(self):
bigquery.SchemaField("geo_col", "GEOGRAPHY"),
bigquery.SchemaField("int_col", "INTEGER"),
bigquery.SchemaField("num_col", "NUMERIC"),
bigquery.SchemaField("bignum_col", "BIGNUMERIC"),
bigquery.SchemaField("str_col", "STRING"),
bigquery.SchemaField("time_col", "TIME"),
bigquery.SchemaField("ts_col", "TIMESTAMP"),
Expand Down Expand Up @@ -1046,6 +1049,14 @@ def test_load_table_from_dataframe_w_explicit_schema(self):
decimal.Decimal("99999999999999999999999999999.999999999"),
],
),
(
"bignum_col",
[
decimal.Decimal("-{d38}.{d38}".format(d38="9" * 38)),
None,
decimal.Decimal("{d38}.{d38}".format(d38="9" * 38)),
],
),
("str_col", [u"abc", None, u"def"]),
(
"time_col",
Expand Down Expand Up @@ -1172,6 +1183,7 @@ def test_load_table_from_dataframe_w_explicit_schema_source_format_csv(self):
bigquery.SchemaField("geo_col", "GEOGRAPHY"),
bigquery.SchemaField("int_col", "INTEGER"),
bigquery.SchemaField("num_col", "NUMERIC"),
bigquery.SchemaField("bignum_col", "BIGNUMERIC"),
bigquery.SchemaField("str_col", "STRING"),
bigquery.SchemaField("time_col", "TIME"),
bigquery.SchemaField("ts_col", "TIMESTAMP"),
Expand Down Expand Up @@ -1210,6 +1222,14 @@ def test_load_table_from_dataframe_w_explicit_schema_source_format_csv(self):
decimal.Decimal("99999999999999999999999999999.999999999"),
],
),
(
"bignum_col",
[
decimal.Decimal("-{d38}.{d38}".format(d38="9" * 38)),
None,
decimal.Decimal("{d38}.{d38}".format(d38="9" * 38)),
],
),
("str_col", [u"abc", None, u"def"]),
(
"time_col",
Expand Down Expand Up @@ -2157,6 +2177,10 @@ def test_query_w_query_params(self):
pi_numeric_param = ScalarQueryParameter(
name="pi_numeric_param", type_="NUMERIC", value=pi_numeric
)
bignum = decimal.Decimal("-{d38}.{d38}".format(d38="9" * 38))
bignum_param = ScalarQueryParameter(
name="bignum_param", type_="BIGNUMERIC", value=bignum
)
truthy = True
truthy_param = ScalarQueryParameter(name="truthy", type_="BOOL", value=truthy)
beef = b"DEADBEEF"
Expand Down Expand Up @@ -2237,6 +2261,11 @@ def test_query_w_query_params(self):
"expected": pi_numeric,
"query_parameters": [pi_numeric_param],
},
{
"sql": "SELECT @bignum_param",
"expected": bignum,
"query_parameters": [bignum_param],
},
{
"sql": "SELECT @truthy",
"expected": truthy,
Expand Down