Skip to content

Commit

Permalink
Parquet options should be reflected in options
Browse files Browse the repository at this point in the history
  • Loading branch information
plamut committed May 26, 2021
1 parent 64b3166 commit cd1c77b
Show file tree
Hide file tree
Showing 2 changed files with 58 additions and 6 deletions.
13 changes: 7 additions & 6 deletions google/cloud/bigquery/external_config.py
Expand Up @@ -799,15 +799,16 @@ def parquet_options(self):
See:
https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#ExternalDataConfiguration.FIELDS.parquet_options
"""
prop = self._properties.get("parquetOptions")
if prop is not None:
prop = ParquetOptions.from_api_repr(prop)
return prop
if self.source_format != ExternalSourceFormat.PARQUET:
return None
return self._options

@parquet_options.setter
def parquet_options(self, value):
prop = value.to_api_repr() if value is not None else None
self._properties["parquetOptions"] = prop
if self.source_format != ExternalSourceFormat.PARQUET:
msg = f"Cannot set Parquet options, source format is {self.source_format}"
raise TypeError(msg)
self._options = value

def to_api_repr(self) -> dict:
"""Build an API representation of this object.
Expand Down
51 changes: 51 additions & 0 deletions tests/unit/test_external_config.py
Expand Up @@ -425,6 +425,57 @@ def test_to_api_repr_bigtable(self):

self.assertEqual(got_resource, exp_resource)

def test_parquet_options_getter(self):
from google.cloud.bigquery.format_options import ParquetOptions

parquet_options = ParquetOptions.from_api_repr(
{"enumAsString": True, "enableListInference": False}
)
ec = external_config.ExternalConfig(
external_config.ExternalSourceFormat.PARQUET
)

self.assertIsNone(ec.parquet_options.enum_as_string)
self.assertIsNone(ec.parquet_options.enable_list_inference)

ec._options = parquet_options

self.assertTrue(ec.parquet_options.enum_as_string)
self.assertFalse(ec.parquet_options.enable_list_inference)

self.assertIs(ec.parquet_options, ec.options)

def test_parquet_options_getter_non_parquet_format(self):
ec = external_config.ExternalConfig(external_config.ExternalSourceFormat.CSV)
self.assertIsNone(ec.parquet_options)

def test_parquet_options_setter(self):
from google.cloud.bigquery.format_options import ParquetOptions

parquet_options = ParquetOptions.from_api_repr(
{"enumAsString": False, "enableListInference": True}
)
ec = external_config.ExternalConfig(
external_config.ExternalSourceFormat.PARQUET
)

ec.parquet_options = parquet_options

# Setting Parquet options should be reflected in the generic options attribute.
self.assertFalse(ec.options.enum_as_string)
self.assertTrue(ec.options.enable_list_inference)

def test_parquet_options_setter_non_parquet_format(self):
from google.cloud.bigquery.format_options import ParquetOptions

parquet_options = ParquetOptions.from_api_repr(
{"enumAsString": False, "enableListInference": True}
)
ec = external_config.ExternalConfig(external_config.ExternalSourceFormat.CSV)

with self.assertRaisesRegex(TypeError, "Cannot set.*source format is CSV"):
ec.parquet_options = parquet_options

def test_from_api_repr_parquet(self):
from google.cloud.bigquery.format_options import ParquetOptions

Expand Down

0 comments on commit cd1c77b

Please sign in to comment.