Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: error using empty array of structs parameter #474

Merged
merged 15 commits into from Feb 24, 2021
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
6 changes: 6 additions & 0 deletions google/cloud/bigquery/__init__.py
Expand Up @@ -66,8 +66,11 @@
from google.cloud.bigquery.model import Model
from google.cloud.bigquery.model import ModelReference
from google.cloud.bigquery.query import ArrayQueryParameter
from google.cloud.bigquery.query import ArrayQueryParameterType
from google.cloud.bigquery.query import ScalarQueryParameter
from google.cloud.bigquery.query import ScalarQueryParameterType
from google.cloud.bigquery.query import StructQueryParameter
from google.cloud.bigquery.query import StructQueryParameterType
from google.cloud.bigquery.query import UDFResource
from google.cloud.bigquery.retry import DEFAULT_RETRY
from google.cloud.bigquery.routine import Routine
Expand All @@ -92,6 +95,9 @@
"ArrayQueryParameter",
"ScalarQueryParameter",
"StructQueryParameter",
"ArrayQueryParameterType",
"ScalarQueryParameterType",
"StructQueryParameterType",
# Datasets
"Dataset",
"DatasetReference",
Expand Down
286 changes: 272 additions & 14 deletions google/cloud/bigquery/query.py
Expand Up @@ -48,6 +48,226 @@ def __ne__(self, other):
return not self == other


class _AbstractQueryParameterType:
"""Base class for representing query parameter types.

https://cloud.google.com/bigquery/docs/reference/rest/v2/QueryParameter#queryparametertype
"""

@classmethod
def from_api_repr(cls, resource):
"""Factory: construct parameter type from JSON resource.

Args:
resource (Dict): JSON mapping of parameter

Returns:
google.cloud.bigquery.query.QueryParameterType: Instance
"""
raise NotImplementedError

def to_api_repr(self):
"""Construct JSON API representation for the parameter type.

Returns:
Dict: JSON mapping
"""
raise NotImplementedError


class ScalarQueryParameterType(_AbstractQueryParameterType):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we should do this in this PR, but I'm tempted to ask for an additional set of constants in google.cloud.bigquery.enums that has all the scalar types defined as objects (without name & description). I imagine that will be useful when we eventually create an "array of structs query parameter" code sample.

Copy link
Contributor Author

@plamut plamut Feb 20, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds reasonable, let's do it in a follow-up PR. 👍

Edit: It's actually very straightforward (if I understood it correctly, that is), thus added that in an extra commit.

"""Type representation for scalar query parameters.

Args:
type_ (str):
One of 'STRING', 'INT64', 'FLOAT64', 'NUMERIC', 'BOOL', 'TIMESTAMP',
'DATETIME', or 'DATE'.
name (Optional[str]):
The name of the query parameter. Primarily used if the type is
one of the subfields in ``StructQueryParameterType`` instance.
description (Optional[str]):
The query parameter description. Primarily used if the type is
one of the subfields in ``StructQueryParameterType`` instance.
"""

def __init__(self, type_, *, name=None, description=None):
self._type = type_
self.name = name
self.description = description

@classmethod
def from_api_repr(cls, resource):
"""Factory: construct parameter type from JSON resource.

Args:
resource (Dict): JSON mapping of parameter

Returns:
google.cloud.bigquery.query.ScalarQueryParameterType: Instance
"""
type_ = resource["type"]
return cls(type_)

def to_api_repr(self):
"""Construct JSON API representation for the parameter type.

Returns:
Dict: JSON mapping
"""
return {"type": self._type}
plamut marked this conversation as resolved.
Show resolved Hide resolved

def __repr__(self):
name = f", name={self.name!r}" if self.name is not None else ""
description = (
f", description={self.description!r}"
if self.description is not None
else ""
)
return f"{self.__class__.__name__}({self._type!r}{name}{description})"


class ArrayQueryParameterType(_AbstractQueryParameterType):
"""Type representation for array query parameters.

Args:
array_type (Union[ScalarQueryParameterType, StructQueryParameterType]):
The type of array elements.
name (Optional[str]):
The name of the query parameter. Primarily used if the type is
one of the subfields in ``StructQueryParameterType`` instance.
description (Optional[str]):
The query parameter description. Primarily used if the type is
one of the subfields in ``StructQueryParameterType`` instance.
"""

def __init__(self, array_type, *, name=None, description=None):
self._array_type = array_type
self.name = name
self.description = description

@classmethod
def from_api_repr(cls, resource):
"""Factory: construct parameter type from JSON resource.

Args:
resource (Dict): JSON mapping of parameter

Returns:
google.cloud.bigquery.query.ArrayQueryParameterType: Instance
"""
array_item_type = resource["arrayType"]["type"]

if array_item_type in {"STRUCT", "RECORD"}:
klass = StructQueryParameterType
else:
klass = ScalarQueryParameterType

item_type_instance = klass.from_api_repr(resource["arrayType"])
return cls(item_type_instance)

def to_api_repr(self):
"""Construct JSON API representation for the parameter type.

Returns:
Dict: JSON mapping
"""
return {
"type": "ARRAY",
"arrayType": self._array_type.to_api_repr(),
}

def __repr__(self):
name = f", name={self.name!r}" if self.name is not None else ""
description = (
f", description={self.description!r}"
if self.description is not None
else ""
)
return f"{self.__class__.__name__}({self._array_type!r}{name}{description})"


class StructQueryParameterType(_AbstractQueryParameterType):
"""Type representation for struct query parameters.

Args:
subtypes (Iterable[Union[ \
plamut marked this conversation as resolved.
Show resolved Hide resolved
ArrayQueryParameterType, ScalarQueryParameterType, StructQueryParameterType \
]]):
An non-empty iterable describing the struct's field types.
name (Optional[str]):
The name of the query parameter. Primarily used if the type is
one of the subfields in ``StructQueryParameterType`` instance.
description (Optional[str]):
The query parameter description. Primarily used if the type is
one of the subfields in ``StructQueryParameterType`` instance.
"""

def __init__(self, *subtypes, name=None, description=None):
self._subtypes = [type_ for type_ in subtypes] # make a shallow copy
plamut marked this conversation as resolved.
Show resolved Hide resolved
self.name = name
self.description = description

@classmethod
def from_api_repr(cls, resource):
"""Factory: construct parameter type from JSON resource.

Args:
resource (Dict): JSON mapping of parameter

Returns:
google.cloud.bigquery.query.StructQueryParameterType: Instance
"""
subtypes = []

for struct_subtype in resource["structTypes"]:
type_repr = struct_subtype["type"]
if type_repr["type"] in {"STRUCT", "RECORD"}:
klass = StructQueryParameterType
elif type_repr["type"] == "ARRAY":
klass = ArrayQueryParameterType
else:
klass = ScalarQueryParameterType

type_instance = klass.from_api_repr(type_repr)
type_instance.name = struct_subtype.get("name")
type_instance.description = struct_subtype.get("description")
subtypes.append(type_instance)

return cls(*subtypes)

def to_api_repr(self):
"""Construct JSON API representation for the parameter type.

Returns:
Dict: JSON mapping
"""
struct_types = []

for subtype in self._subtypes:
item = {"type": subtype.to_api_repr()}
if subtype.name is not None:
item["name"] = subtype.name
if subtype.description is not None:
item["description"] = subtype.description

struct_types.append(item)

return {
"type": "STRUCT",
"structTypes": struct_types,
}

def __repr__(self):
name = f", name={self.name!r}" if self.name is not None else ""
description = (
f", description={self.description!r}"
if self.description is not None
else ""
)
items = ", ".join(repr(subtype) for subtype in self._subtypes)
return f"{self.__class__.__name__}({items}{name}{description})"


class _AbstractQueryParameter(object):
"""Base class for named / positional query parameters.
"""
Expand Down Expand Up @@ -184,28 +404,43 @@ class ArrayQueryParameter(_AbstractQueryParameter):
Parameter name, used via ``@foo`` syntax. If None, the
parameter can only be addressed via position (``?``).

array_type (str):
Name of type of array elements. One of `'STRING'`, `'INT64'`,
`'FLOAT64'`, `'NUMERIC'`, `'BOOL'`, `'TIMESTAMP'`, or `'DATE'`.
array_type (Union[str, ScalarQueryParameterType, StructQueryParameterType]):
The type of array elements. If given as a string, it must be one of
`'STRING'`, `'INT64'`, `'FLOAT64'`, `'NUMERIC'`, `'BOOL'`, `'TIMESTAMP'`,
`'DATE'`, or `'STRUCT'`/`'RECORD'`.
If the type is ``'STRUCT'``/``'RECORD'`` and ``values`` is empty,
the exact item type cannot be deduced, thus a ``StructQueryParameterType``
instance needs to be passed in.

values (List[appropriate scalar type]): The parameter array values.
values (List[appropriate type]): The parameter array values.
"""

def __init__(self, name, array_type, values):
self.name = name
self.array_type = array_type
self.values = values

if isinstance(array_type, str):
if not values and array_type in {"RECORD", "STRUCT"}:
raise ValueError(
"Missing detailed struct item type info for an empty array, "
"please provide a StructQueryParameterType instance."
)
self.array_type = array_type

@classmethod
def positional(cls, array_type, values):
"""Factory for positional parameters.

Args:
array_type (str):
Name of type of array elements. One of `'STRING'`, `'INT64'`,
`'FLOAT64'`, `'NUMERIC'`, `'BOOL'`, `'TIMESTAMP'`, or `'DATE'`.
array_type (Union[str, ScalarQueryParameterType, StructQueryParameterType]):
The type of array elements. If given as a string, it must be one of
`'STRING'`, `'INT64'`, `'FLOAT64'`, `'NUMERIC'`, `'BOOL'`, `'TIMESTAMP'`,
`'DATE'`, or `'STRUCT'`/`'RECORD'`.
If the type is ``'STRUCT'``/``'RECORD'`` and ``values`` is empty,
the exact item type cannot be deduced, thus a ``StructQueryParameterType``
instance needs to be passed in.

values (List[appropriate scalar type]): The parameter array values.
values (List[appropriate type]): The parameter array values.

Returns:
google.cloud.bigquery.query.ArrayQueryParameter: Instance without name
Expand Down Expand Up @@ -263,22 +498,38 @@ def to_api_repr(self):
Dict: JSON mapping
"""
values = self.values
if self.array_type == "RECORD" or self.array_type == "STRUCT":

if self.array_type in {"RECORD", "STRUCT"} or isinstance(
self.array_type, StructQueryParameterType
):
reprs = [value.to_api_repr() for value in values]
a_type = reprs[0]["parameterType"]
a_values = [repr_["parameterValue"] for repr_ in reprs]

if reprs:
a_type = reprs[0]["parameterType"]
else:
# The constructor disallows STRUCT/RECORD type when empty values.
plamut marked this conversation as resolved.
Show resolved Hide resolved
assert isinstance(self.array_type, StructQueryParameterType)
a_type = self.array_type.to_api_repr()
else:
a_type = {"type": self.array_type}
converter = _SCALAR_VALUE_TO_JSON_PARAM.get(self.array_type)
# Scalar array item type.
if isinstance(self.array_type, str):
a_type = {"type": self.array_type}
else:
a_type = self.array_type.to_api_repr()

converter = _SCALAR_VALUE_TO_JSON_PARAM.get(a_type["type"])
if converter is not None:
values = [converter(value) for value in values]
a_values = [{"value": value} for value in values]

resource = {
"parameterType": {"type": "ARRAY", "arrayType": a_type},
"parameterValue": {"arrayValues": a_values},
}
if self.name is not None:
resource["name"] = self.name

return resource

def _key(self):
Expand All @@ -289,7 +540,14 @@ def _key(self):
Returns:
Tuple: The contents of this :class:`~google.cloud.bigquery.query.ArrayQueryParameter`.
"""
return (self.name, self.array_type.upper(), self.values)
if isinstance(self.array_type, str):
item_type = self.array_type
elif isinstance(self.array_type, ScalarQueryParameterType):
item_type = self.array_type._type
else:
item_type = "STRUCT"

return (self.name, item_type.upper(), self.values)

def __eq__(self, other):
if not isinstance(other, ArrayQueryParameter):
Expand Down
15 changes: 15 additions & 0 deletions tests/system/test_client.py
Expand Up @@ -2140,7 +2140,9 @@ def test_query_w_query_params(self):
from google.cloud.bigquery.job import QueryJobConfig
from google.cloud.bigquery.query import ArrayQueryParameter
from google.cloud.bigquery.query import ScalarQueryParameter
from google.cloud.bigquery.query import ScalarQueryParameterType
from google.cloud.bigquery.query import StructQueryParameter
from google.cloud.bigquery.query import StructQueryParameterType

question = "What is the answer to life, the universe, and everything?"
question_param = ScalarQueryParameter(
Expand Down Expand Up @@ -2195,6 +2197,14 @@ def test_query_w_query_params(self):
characters_param = ArrayQueryParameter(
name=None, array_type="RECORD", values=[phred_param, bharney_param]
)
empty_struct_array_param = ArrayQueryParameter(
name="empty_array_param",
values=[],
array_type=StructQueryParameterType(
ScalarQueryParameterType(name="foo", type_="INT64"),
ScalarQueryParameterType(name="bar", type_="STRING"),
),
)
hero_param = StructQueryParameter("hero", phred_name_param, phred_age_param)
sidekick_param = StructQueryParameter(
"sidekick", bharney_name_param, bharney_age_param
Expand Down Expand Up @@ -2285,6 +2295,11 @@ def test_query_w_query_params(self):
],
"query_parameters": [characters_param],
},
{
"sql": "SELECT @empty_array_param",
"expected": [],
"query_parameters": [empty_struct_array_param],
},
{
"sql": "SELECT @roles",
"expected": {
Expand Down