From 5f807f04c78df377cc83513a6ac9386fe985069b Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Tue, 22 Dec 2020 13:30:07 +0530 Subject: [PATCH 01/22] feat: add type hint for public methods --- .gitignore | 1 + google/cloud/bigquery/_helpers.py | 12 +- google/cloud/bigquery/_http.py | 3 +- google/cloud/bigquery/_pandas_helpers.py | 10 +- google/cloud/bigquery/_tqdm_helpers.py | 2 +- google/cloud/bigquery/client.py | 462 ++++++++++++------ google/cloud/bigquery/dataset.py | 12 +- google/cloud/bigquery/dbapi/cursor.py | 4 +- google/cloud/bigquery/job/base.py | 32 +- google/cloud/bigquery/job/query.py | 35 +- google/cloud/bigquery/model.py | 4 +- .../cloud/bigquery/opentelemetry_tracing.py | 8 +- google/cloud/bigquery/retry.py | 4 +- google/cloud/bigquery/routine/routine.py | 10 +- google/cloud/bigquery/table.py | 48 +- noxfile.py | 9 + setup.cfg | 10 + 17 files changed, 429 insertions(+), 237 deletions(-) diff --git a/.gitignore b/.gitignore index b4243ced7..99c3a1444 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ pip-log.txt .nox .cache .pytest_cache +.pytype # Mac diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py index daa14b92a..0afe29a83 100644 --- a/google/cloud/bigquery/_helpers.py +++ b/google/cloud/bigquery/_helpers.py @@ -19,12 +19,12 @@ import decimal import re -from google.cloud._helpers import UTC -from google.cloud._helpers import _date_from_iso8601_date -from google.cloud._helpers import _datetime_from_microseconds -from google.cloud._helpers import _RFC3339_MICROS -from google.cloud._helpers import _RFC3339_NO_FRACTION -from google.cloud._helpers import _to_bytes +from google.cloud._helpers import UTC # type: ignore +from google.cloud._helpers import _date_from_iso8601_date # type: ignore +from google.cloud._helpers import _datetime_from_microseconds # type: ignore +from google.cloud._helpers import _RFC3339_MICROS # type: ignore +from google.cloud._helpers import _RFC3339_NO_FRACTION # type: ignore +from google.cloud._helpers import _to_bytes # type: ignore _RFC3339_MICROS_NO_ZULU = "%Y-%m-%dT%H:%M:%S.%f" _TIMEONLY_WO_MICROS = "%H:%M:%S" diff --git a/google/cloud/bigquery/_http.py b/google/cloud/bigquery/_http.py index ede26cc70..2726fb8c9 100644 --- a/google/cloud/bigquery/_http.py +++ b/google/cloud/bigquery/_http.py @@ -17,8 +17,7 @@ import os import pkg_resources -from google.cloud import _http - +from google.cloud import _http # type: ignore from google.cloud.bigquery import __version__ diff --git a/google/cloud/bigquery/_pandas_helpers.py b/google/cloud/bigquery/_pandas_helpers.py index 7553726fa..c92839d4e 100644 --- a/google/cloud/bigquery/_pandas_helpers.py +++ b/google/cloud/bigquery/_pandas_helpers.py @@ -23,13 +23,13 @@ from packaging import version try: - import pandas + import pandas # type: ignore except ImportError: # pragma: NO COVER pandas = None try: - import pyarrow - import pyarrow.parquet + import pyarrow # type: ignore + import pyarrow.parquet # type: ignore except ImportError: # pragma: NO COVER pyarrow = None @@ -373,6 +373,7 @@ def augment_schema(dataframe, current_bq_schema): Returns: Optional[Sequence[google.cloud.bigquery.schema.SchemaField]] """ + # pytype: disable=attribute-error augmented_schema = [] unknown_type_fields = [] @@ -406,6 +407,7 @@ def augment_schema(dataframe, current_bq_schema): return None return augmented_schema + # pytype: enable=attribute-error def dataframe_to_arrow(dataframe, bq_schema): @@ -624,7 +626,7 @@ def _download_table_bqstorage( # Passing a BQ Storage client in implies that the BigQuery Storage library # is available and can be imported. - from google.cloud import bigquery_storage + from google.cloud import bigquery_storage # type: ignore if "$" in table.table_id: raise ValueError( diff --git a/google/cloud/bigquery/_tqdm_helpers.py b/google/cloud/bigquery/_tqdm_helpers.py index 2fcf2a981..8717b5be4 100644 --- a/google/cloud/bigquery/_tqdm_helpers.py +++ b/google/cloud/bigquery/_tqdm_helpers.py @@ -19,7 +19,7 @@ import warnings try: - import tqdm + import tqdm # type: ignore except ImportError: # pragma: NO COVER tqdm = None diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 10127e10d..8d57cbf09 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -74,6 +74,19 @@ from google.cloud.bigquery.table import TableReference from google.cloud.bigquery.table import RowIterator +# Types needed only for Type Hints +import datetime +import pandas +from google.cloud.bigquery.job import ( + _AsyncJob, + LoadJobConfig, + QueryJobConfig, + CopyJobConfig, + ExtractJobConfig, +) +from google.api_core import retry as retries # type: ignore +from typing import Any, BinaryIO, Dict, Iterable, Sequence, Tuple, Union + _DEFAULT_CHUNKSIZE = 1048576 # 1024 * 1024 B = 1 MB _MAX_MULTIPART_SIZE = 5 * 1024 * 1024 @@ -216,7 +229,10 @@ def close(self): self._http.close() def get_service_account_email( - self, project=None, retry=DEFAULT_RETRY, timeout=None + self, + project: str = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): """Get the email address of the project's BigQuery service account @@ -259,7 +275,11 @@ def get_service_account_email( return api_response["email"] def list_projects( - self, max_results=None, page_token=None, retry=DEFAULT_RETRY, timeout=None + self, + max_results: int = None, + page_token: str = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): """List projects for the project associated with this client. @@ -313,13 +333,13 @@ def api_request(*args, **kwargs): def list_datasets( self, - project=None, - include_all=False, - filter=None, - max_results=None, - page_token=None, - retry=DEFAULT_RETRY, - timeout=None, + project: str = None, + include_all: bool = False, + filter: str = None, + max_results: int = None, + page_token: str = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): """List datasets for the project associated with this client. @@ -390,7 +410,7 @@ def api_request(*args, **kwargs): extra_params=extra_params, ) - def dataset(self, dataset_id, project=None): + def dataset(self, dataset_id: str, project: str = None): """Deprecated: Construct a reference to a dataset. .. deprecated:: 1.24.0 @@ -466,7 +486,11 @@ def _dataset_from_arg(self, dataset): return dataset def create_dataset( - self, dataset, exists_ok=False, retry=DEFAULT_RETRY, timeout=None + self, + dataset: Union[str, Dataset, DatasetReference], + exists_ok: bool = False, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): """API call: create the dataset via a POST request. @@ -537,7 +561,11 @@ def create_dataset( return self.get_dataset(dataset.reference, retry=retry) def create_routine( - self, routine, exists_ok=False, retry=DEFAULT_RETRY, timeout=None + self, + routine: Routine, + exists_ok: bool = False, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): """[Beta] Create a routine via a POST request. @@ -587,7 +615,13 @@ def create_routine( raise return self.get_routine(routine.reference, retry=retry) - def create_table(self, table, exists_ok=False, retry=DEFAULT_RETRY, timeout=None): + def create_table( + self, + table: Union[str, Table, TableReference], + exists_ok: bool = False, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + ): """API call: create a table via a PUT request See @@ -654,7 +688,12 @@ def _call_api( return call() return call() - def get_dataset(self, dataset_ref, retry=DEFAULT_RETRY, timeout=None): + def get_dataset( + self, + dataset_ref: Union[DatasetReference, str], + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + ): """Fetch the dataset referenced by ``dataset_ref`` Args: @@ -693,7 +732,11 @@ def get_dataset(self, dataset_ref, retry=DEFAULT_RETRY, timeout=None): return Dataset.from_api_repr(api_response) def get_iam_policy( - self, table, requested_policy_version=1, retry=DEFAULT_RETRY, timeout=None, + self, + table: Union[Table, TableReference], + requested_policy_version: int = 1, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): if not isinstance(table, (Table, TableReference)): raise TypeError("table must be a Table or TableReference") @@ -718,7 +761,12 @@ def get_iam_policy( return Policy.from_api_repr(response) def set_iam_policy( - self, table, policy, updateMask=None, retry=DEFAULT_RETRY, timeout=None, + self, + table: Union[Table, TableReference], + policy: Policy, + updateMask: str = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): if not isinstance(table, (Table, TableReference)): raise TypeError("table must be a Table or TableReference") @@ -747,7 +795,11 @@ def set_iam_policy( return Policy.from_api_repr(response) def test_iam_permissions( - self, table, permissions, retry=DEFAULT_RETRY, timeout=None, + self, + table: Union[Table, TableReference], + permissions: Sequence[str], + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): if not isinstance(table, (Table, TableReference)): raise TypeError("table must be a Table or TableReference") @@ -768,7 +820,12 @@ def test_iam_permissions( return response - def get_model(self, model_ref, retry=DEFAULT_RETRY, timeout=None): + def get_model( + self, + model_ref: Union[ModelReference, str], + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + ): """[Beta] Fetch the model referenced by ``model_ref``. Args: @@ -806,7 +863,12 @@ def get_model(self, model_ref, retry=DEFAULT_RETRY, timeout=None): ) return Model.from_api_repr(api_response) - def get_routine(self, routine_ref, retry=DEFAULT_RETRY, timeout=None): + def get_routine( + self, + routine_ref: Union[Routine, RoutineReference, str], + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + ): """[Beta] Get the routine referenced by ``routine_ref``. Args: @@ -845,7 +907,12 @@ def get_routine(self, routine_ref, retry=DEFAULT_RETRY, timeout=None): ) return Routine.from_api_repr(api_response) - def get_table(self, table, retry=DEFAULT_RETRY, timeout=None): + def get_table( + self, + table: Union[Table, TableReference, str], + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + ): """Fetch the table referenced by ``table``. Args: @@ -881,7 +948,13 @@ def get_table(self, table, retry=DEFAULT_RETRY, timeout=None): ) return Table.from_api_repr(api_response) - def update_dataset(self, dataset, fields, retry=DEFAULT_RETRY, timeout=None): + def update_dataset( + self, + dataset: Dataset, + fields: Sequence[str], + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + ): """Change some fields of a dataset. Use ``fields`` to specify which fields to update. At least one field @@ -945,7 +1018,13 @@ def update_dataset(self, dataset, fields, retry=DEFAULT_RETRY, timeout=None): ) return Dataset.from_api_repr(api_response) - def update_model(self, model, fields, retry=DEFAULT_RETRY, timeout=None): + def update_model( + self, + model: Model, + fields: Sequence[str], + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + ): """[Beta] Change some fields of a model. Use ``fields`` to specify which fields to update. At least one field @@ -1003,7 +1082,13 @@ def update_model(self, model, fields, retry=DEFAULT_RETRY, timeout=None): ) return Model.from_api_repr(api_response) - def update_routine(self, routine, fields, retry=DEFAULT_RETRY, timeout=None): + def update_routine( + self, + routine: Routine, + fields: Sequence[str], + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + ): """[Beta] Change some fields of a routine. Use ``fields`` to specify which fields to update. At least one field @@ -1071,7 +1156,13 @@ def update_routine(self, routine, fields, retry=DEFAULT_RETRY, timeout=None): ) return Routine.from_api_repr(api_response) - def update_table(self, table, fields, retry=DEFAULT_RETRY, timeout=None): + def update_table( + self, + table: Table, + fields: Sequence[str], + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + ): """Change some fields of a table. Use ``fields`` to specify which fields to update. At least one field @@ -1132,11 +1223,11 @@ def update_table(self, table, fields, retry=DEFAULT_RETRY, timeout=None): def list_models( self, - dataset, - max_results=None, - page_token=None, - retry=DEFAULT_RETRY, - timeout=None, + dataset: Union[Dataset, DatasetReference, str], + max_results: int = None, + page_token: str = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): """[Beta] List models in the dataset. @@ -1204,11 +1295,11 @@ def api_request(*args, **kwargs): def list_routines( self, - dataset, - max_results=None, - page_token=None, - retry=DEFAULT_RETRY, - timeout=None, + dataset: Union[Dataset, DatasetReference, str], + max_results: int = None, + page_token: str = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): """[Beta] List routines in the dataset. @@ -1276,11 +1367,11 @@ def api_request(*args, **kwargs): def list_tables( self, - dataset, - max_results=None, - page_token=None, - retry=DEFAULT_RETRY, - timeout=None, + dataset: Union[Dataset, DatasetReference, str], + max_results: int = None, + page_token: str = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): """List tables in the dataset. @@ -1347,11 +1438,11 @@ def api_request(*args, **kwargs): def delete_dataset( self, - dataset, - delete_contents=False, - retry=DEFAULT_RETRY, - timeout=None, - not_found_ok=False, + dataset: Union[Dataset, DatasetReference, str], + delete_contents: bool = False, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + not_found_ok: bool = False, ): """Delete a dataset. @@ -1406,7 +1497,11 @@ def delete_dataset( raise def delete_model( - self, model, retry=DEFAULT_RETRY, timeout=None, not_found_ok=False + self, + model: Union[Model, ModelReference, str], + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + not_found_ok: bool = False, ): """[Beta] Delete a model @@ -1454,7 +1549,11 @@ def delete_model( raise def delete_routine( - self, routine, retry=DEFAULT_RETRY, timeout=None, not_found_ok=False + self, + routine: Union[Routine, RoutineReference, str], + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + not_found_ok: bool = False, ): """[Beta] Delete a routine. @@ -1504,7 +1603,11 @@ def delete_routine( raise def delete_table( - self, table, retry=DEFAULT_RETRY, timeout=None, not_found_ok=False + self, + table: Union[Table, TableReference, str], + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + not_found_ok: bool = False, ): """Delete a table @@ -1550,7 +1653,13 @@ def delete_table( raise def _get_query_results( - self, job_id, retry, project=None, timeout_ms=None, location=None, timeout=None, + self, + job_id: str, + retry: retries.Retry, + project: str = None, + timeout_ms: int = None, + location: str = None, + timeout: float = None, ): """Get the query results object for a query job. @@ -1609,7 +1718,7 @@ def _get_query_results( ) return _QueryResults.from_api_repr(resource) - def job_from_resource(self, resource): + def job_from_resource(self, resource: dict): """Detect correct job type from resource and instantiate. Args: @@ -1635,7 +1744,12 @@ def job_from_resource(self, resource): return job.QueryJob.from_api_repr(resource, self) return job.UnknownJob.from_api_repr(resource, self) - def create_job(self, job_config, retry=DEFAULT_RETRY, timeout=None): + def create_job( + self, + job_config: dict, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + ): """Create a new job. Args: job_config (dict): configuration job representation returned from the API. @@ -1726,7 +1840,12 @@ def create_job(self, job_config, retry=DEFAULT_RETRY, timeout=None): raise TypeError("Invalid job configuration received.") def get_job( - self, job_id, project=None, location=None, retry=DEFAULT_RETRY, timeout=None + self, + job_id: str, + project: str = None, + location: str = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): """Fetch a job for the project associated with this client. @@ -1783,7 +1902,12 @@ def get_job( return self.job_from_resource(resource) def cancel_job( - self, job_id, project=None, location=None, retry=DEFAULT_RETRY, timeout=None + self, + job_id: str, + project: str = None, + location: str = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): """Attempt to cancel a job from a job ID. @@ -1841,16 +1965,16 @@ def cancel_job( def list_jobs( self, - project=None, - parent_job=None, - max_results=None, - page_token=None, - all_users=None, - state_filter=None, - retry=DEFAULT_RETRY, - timeout=None, - min_creation_time=None, - max_creation_time=None, + project: str = None, + parent_job: Union[_AsyncJob, str] = None, + max_results: int = None, + page_token: str = None, + all_users: bool = None, + state_filter: str = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + min_creation_time: datetime.datetime = None, + max_creation_time: datetime.datetime = None, ): """List jobs for the project associated with this client. @@ -1951,15 +2075,15 @@ def api_request(*args, **kwargs): def load_table_from_uri( self, - source_uris, - destination, - job_id=None, - job_id_prefix=None, - location=None, - project=None, - job_config=None, - retry=DEFAULT_RETRY, - timeout=None, + source_uris: Union[str, Sequence[str]], + destination: Union[Table, TableReference, str], + job_id: str = None, + job_id_prefix: str = None, + location: str = None, + project: str = None, + job_config: LoadJobConfig = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): """Starts a job for loading data into a table from CloudStorage. @@ -2033,17 +2157,17 @@ def load_table_from_uri( def load_table_from_file( self, - file_obj, - destination, - rewind=False, - size=None, - num_retries=_DEFAULT_NUM_RETRIES, - job_id=None, - job_id_prefix=None, - location=None, - project=None, - job_config=None, - timeout=None, + file_obj: BinaryIO, + destination: Union[Table, TableReference, str], + rewind: bool = False, + size: int = None, + num_retries: int = _DEFAULT_NUM_RETRIES, + job_id: str = None, + job_id_prefix: str = None, + location: str = None, + project: str = None, + job_config: LoadJobConfig = None, + timeout: float = None, ): """Upload the contents of this table from a file-like object. @@ -2137,16 +2261,16 @@ def load_table_from_file( def load_table_from_dataframe( self, - dataframe, - destination, - num_retries=_DEFAULT_NUM_RETRIES, - job_id=None, - job_id_prefix=None, - location=None, - project=None, - job_config=None, - parquet_compression="snappy", - timeout=None, + dataframe: pandas.DataFrame, + destination: Union[Table, TableReference, str], + num_retries: int = _DEFAULT_NUM_RETRIES, + job_id: str = None, + job_id_prefix: str = None, + location: str = None, + project: str = None, + job_config: LoadJobConfig = None, + parquet_compression: str = "snappy", + timeout: float = None, ): """Upload the contents of a table from a pandas DataFrame. @@ -2364,15 +2488,15 @@ def load_table_from_dataframe( def load_table_from_json( self, - json_rows, - destination, - num_retries=_DEFAULT_NUM_RETRIES, - job_id=None, - job_id_prefix=None, - location=None, - project=None, - job_config=None, - timeout=None, + json_rows: Iterable[Dict[str, Any]], + destination: Union[Table, TableReference, str], + num_retries: int = _DEFAULT_NUM_RETRIES, + job_id: str = None, + job_id_prefix: str = None, + location: str = None, + project: str = None, + job_config: LoadJobConfig = None, + timeout: float = None, ): """Upload the contents of a table from a JSON string or dict. @@ -2645,15 +2769,15 @@ def _do_multipart_upload( def copy_table( self, - sources, - destination, - job_id=None, - job_id_prefix=None, - location=None, - project=None, - job_config=None, - retry=DEFAULT_RETRY, - timeout=None, + sources: Union[Table, TableReference, str], + destination: Union[Table, TableReference, str], + job_id: str = None, + job_id_prefix: str = None, + location: str = None, + project: str = None, + job_config: CopyJobConfig = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): """Copy one or more tables to another table. @@ -2748,16 +2872,16 @@ def copy_table( def extract_table( self, - source, - destination_uris, - job_id=None, - job_id_prefix=None, - location=None, - project=None, - job_config=None, - retry=DEFAULT_RETRY, - timeout=None, - source_type="Table", + source: Union[Table, TableReference, Model, ModelReference, str], + destination_uris: Union[str, Sequence[str]], + job_id: str = None, + job_id_prefix: str = None, + location: str = None, + project: str = None, + job_config: ExtractJobConfig = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + source_type: str = "Table", ): """Start a job to extract a table into Cloud Storage files. @@ -2847,14 +2971,14 @@ def extract_table( def query( self, - query, - job_config=None, - job_id=None, - job_id_prefix=None, - location=None, - project=None, - retry=DEFAULT_RETRY, - timeout=None, + query: str, + job_config: QueryJobConfig = None, + job_id: str = None, + job_id_prefix: str = None, + location: str = None, + project: str = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): """Run a SQL query. @@ -2932,7 +3056,13 @@ def query( return query_job - def insert_rows(self, table, rows, selected_fields=None, **kwargs): + def insert_rows( + self, + table: Union[Table, TableReference, str], + rows: Union[Sequence[Tuple], Sequence[Dict]], + selected_fields: Sequence[SchemaField] = None, + **kwargs: dict, + ): """Insert rows into a table via the streaming API. See @@ -2995,7 +3125,12 @@ def insert_rows(self, table, rows, selected_fields=None, **kwargs): return self.insert_rows_json(table, json_rows, **kwargs) def insert_rows_from_dataframe( - self, table, dataframe, selected_fields=None, chunk_size=500, **kwargs + self, + table: Union[Table, TableReference, str], + dataframe: pandas.DataFrame, + selected_fields: Sequence[SchemaField] = None, + chunk_size: int = 500, + **kwargs: Dict, ): """Insert rows into a table from a dataframe via the streaming API. @@ -3044,14 +3179,14 @@ def insert_rows_from_dataframe( def insert_rows_json( self, - table, - json_rows, - row_ids=None, - skip_invalid_rows=None, - ignore_unknown_values=None, - template_suffix=None, - retry=DEFAULT_RETRY, - timeout=None, + table: Union[Table, TableReference, str], + json_rows: Sequence[Dict], + row_ids: Sequence[str] = None, + skip_invalid_rows: bool = None, + ignore_unknown_values: bool = None, + template_suffix: str = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): """Insert rows into a table without applying local type conversions. @@ -3148,7 +3283,12 @@ def insert_rows_json( return errors - def list_partitions(self, table, retry=DEFAULT_RETRY, timeout=None): + def list_partitions( + self, + table: Union[Table, TableReference, str], + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + ): """List the partitions in a table. Args: @@ -3190,14 +3330,14 @@ def list_partitions(self, table, retry=DEFAULT_RETRY, timeout=None): def list_rows( self, - table, - selected_fields=None, - max_results=None, - page_token=None, - start_index=None, - page_size=None, - retry=DEFAULT_RETRY, - timeout=None, + table: Union[Table, TableListItem, TableReference, str], + selected_fields: Sequence[SchemaField] = None, + max_results: int = None, + page_token: str = None, + start_index: int = None, + page_size: int = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): """List the rows of the table. @@ -3299,17 +3439,17 @@ def list_rows( def _list_rows_from_query_results( self, - job_id, - location, - project, - schema, - total_rows=None, - destination=None, - max_results=None, - start_index=None, - page_size=None, - retry=DEFAULT_RETRY, - timeout=None, + job_id: str, + location: str, + project: str, + schema: SchemaField, + total_rows: int = None, + destination: Union[Table, TableReference, TableListItem, str] = None, + max_results: int = None, + start_index: int = None, + page_size: int = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, ): """List the rows of a completed query. See @@ -3395,7 +3535,7 @@ def _schema_to_json_file_object(self, schema_list, file_obj): """ json.dump(schema_list, file_obj, indent=2, sort_keys=True) - def schema_from_json(self, file_or_path): + def schema_from_json(self, file_or_path: Union[str, BinaryIO]): """Takes a file object or file path that contains json that describes a table schema. @@ -3408,7 +3548,9 @@ def schema_from_json(self, file_or_path): with open(file_or_path) as file_obj: return self._schema_from_json_file_object(file_obj) - def schema_to_json(self, schema_list, destination): + def schema_to_json( + self, schema_list: Sequence[SchemaField], destination: Union[str, BinaryIO] + ): """Takes a list of schema field objects. Serializes the list of schema field objects as json to a file. diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 2d3a4755f..2943dc8b5 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -18,7 +18,7 @@ import copy -import google.cloud._helpers +import google.cloud._helpers # type: ignore from google.cloud.bigquery import _helpers from google.cloud.bigquery.model import ModelReference @@ -220,7 +220,7 @@ def to_api_repr(self): return resource @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict): """Factory: construct an access entry given its API representation Args: @@ -288,7 +288,7 @@ def path(self): routine = _get_routine_reference @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict): """Factory: construct a dataset reference given its API representation Args: @@ -304,7 +304,7 @@ def from_api_repr(cls, resource): return cls(project, dataset_id) @classmethod - def from_string(cls, dataset_id, default_project=None): + def from_string(cls, dataset_id: str, default_project: str = None): """Construct a dataset reference from dataset ID string. Args: @@ -640,7 +640,7 @@ def default_encryption_configuration(self, value): self._properties["defaultEncryptionConfiguration"] = api_repr @classmethod - def from_string(cls, full_dataset_id): + def from_string(cls, full_dataset_id: str): """Construct a dataset from fully-qualified dataset ID. Args: @@ -664,7 +664,7 @@ def from_string(cls, full_dataset_id): return cls(DatasetReference.from_string(full_dataset_id)) @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict): """Factory: construct a dataset given its API representation Args: diff --git a/google/cloud/bigquery/dbapi/cursor.py b/google/cloud/bigquery/dbapi/cursor.py index ee09158d8..2a83fe830 100644 --- a/google/cloud/bigquery/dbapi/cursor.py +++ b/google/cloud/bigquery/dbapi/cursor.py @@ -30,7 +30,7 @@ from google.cloud.bigquery import job from google.cloud.bigquery.dbapi import _helpers from google.cloud.bigquery.dbapi import exceptions -import google.cloud.exceptions +import google.cloud.exceptions # type: ignore _LOGGER = logging.getLogger(__name__) @@ -255,7 +255,7 @@ def _bqstorage_fetch(self, bqstorage_client): """ # Hitting this code path with a BQ Storage client instance implies that # bigquery_storage can indeed be imported here without errors. - from google.cloud import bigquery_storage + from google.cloud import bigquery_storage # type: ignore table_reference = self._query_job.destination diff --git a/google/cloud/bigquery/job/base.py b/google/cloud/bigquery/job/base.py index f24e972c8..eaea49d84 100644 --- a/google/cloud/bigquery/job/base.py +++ b/google/cloud/bigquery/job/base.py @@ -19,12 +19,15 @@ import http import threading -from google.api_core import exceptions -import google.api_core.future.polling +from google.api_core import exceptions # type: ignore +import google.api_core.future.polling # type: ignore from google.cloud.bigquery import _helpers from google.cloud.bigquery.retry import DEFAULT_RETRY +# Types needed only for Type Hints +from google.api_core import retry as retries # type: ignore + _DONE_STATE = "DONE" _STOPPED_REASON = "stopped" @@ -466,7 +469,9 @@ def _begin(self, client=None, retry=DEFAULT_RETRY, timeout=None): ) self._set_properties(api_response) - def exists(self, client=None, retry=DEFAULT_RETRY, timeout=None): + def exists( + self, client=None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None + ): """API call: test for the existence of the job via a GET request See @@ -509,7 +514,9 @@ def exists(self, client=None, retry=DEFAULT_RETRY, timeout=None): else: return True - def reload(self, client=None, retry=DEFAULT_RETRY, timeout=None): + def reload( + self, client=None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None + ): """API call: refresh job properties via a GET request. See @@ -544,7 +551,9 @@ def reload(self, client=None, retry=DEFAULT_RETRY, timeout=None): ) self._set_properties(api_response) - def cancel(self, client=None, retry=DEFAULT_RETRY, timeout=None): + def cancel( + self, client=None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None + ): """API call: cancel job via a POST request See @@ -610,7 +619,12 @@ def _set_future_result(self): else: self.set_result(self) - def done(self, retry=DEFAULT_RETRY, timeout=None, reload=True): + def done( + self, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + reload: bool = True, + ): """Checks if the job is complete. Args: @@ -633,7 +647,7 @@ def done(self, retry=DEFAULT_RETRY, timeout=None, reload=True): self.reload(retry=retry, timeout=timeout) return self.state == _DONE_STATE - def result(self, retry=DEFAULT_RETRY, timeout=None): + def result(self, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None): """Start the job and wait for it to complete and get the result. Args: @@ -818,7 +832,7 @@ def _fill_from_default(self, default_job_config): + repr(default_job_config._job_type) ) - new_job_config = self.__class__() + new_job_config = self.__class__() # type: ignore default_job_properties = copy.deepcopy(default_job_config._properties) for key in self._properties: @@ -842,7 +856,7 @@ def from_api_repr(cls, resource): Returns: google.cloud.bigquery.job._JobConfig: Configuration parsed from ``resource``. """ - job_config = cls() + job_config = cls() # type: ignore job_config._properties = resource return job_config diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index 491983f8e..b0db28851 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -46,6 +46,11 @@ from google.cloud.bigquery.job.base import _JobConfig from google.cloud.bigquery.job.base import _JobReference +# Types needed only for Type Hints +from google.api_core import retry as retries # type: ignore +from google.cloud import bigquery_storage # type: ignore +from typing import Any, Dict + _CONTAINS_ORDER_BY = re.compile(r"ORDER\s+BY", re.IGNORECASE) _TIMEOUT_BUFFER_SECS = 0.1 @@ -1036,7 +1041,9 @@ def _begin(self, client=None, retry=DEFAULT_RETRY, timeout=None): exc.query_job = self raise - def _reload_query_results(self, retry=DEFAULT_RETRY, timeout=None): + def _reload_query_results( + self, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None + ): """Refresh the cached query results. Args: @@ -1111,11 +1118,11 @@ def _done_or_raise(self, retry=DEFAULT_RETRY, timeout=None): def result( self, - page_size=None, - max_results=None, - retry=DEFAULT_RETRY, - timeout=None, - start_index=None, + page_size: int = None, + max_results: int = None, + retry: retries.Retry = DEFAULT_RETRY, + timeout: float = None, + start_index: int = None, ): """Start the job and wait for it to complete and get the result. @@ -1196,9 +1203,9 @@ def result( # changes to table.RowIterator.to_arrow() def to_arrow( self, - progress_bar_type=None, - bqstorage_client=None, - create_bqstorage_client=True, + progress_bar_type: str = None, + bqstorage_client: bigquery_storage.BigQueryReadClient = None, + create_bqstorage_client: bool = True, ): """[Beta] Create a class:`pyarrow.Table` by loading all pages of a table or query. @@ -1265,11 +1272,11 @@ def to_arrow( # changes to table.RowIterator.to_dataframe() def to_dataframe( self, - bqstorage_client=None, - dtypes=None, - progress_bar_type=None, - create_bqstorage_client=True, - date_as_object=True, + bqstorage_client: bigquery_storage.BigQueryReadClient = None, + dtypes: Dict[str, Any] = None, + progress_bar_type: str = None, + create_bqstorage_client: bool = True, + date_as_object: bool = True, ): """Return a pandas DataFrame from a QueryJob diff --git a/google/cloud/bigquery/model.py b/google/cloud/bigquery/model.py index 55846bd1a..860c3a907 100644 --- a/google/cloud/bigquery/model.py +++ b/google/cloud/bigquery/model.py @@ -20,8 +20,8 @@ from google.protobuf import json_format -import google.cloud._helpers -from google.api_core import datetime_helpers +import google.cloud._helpers # type: ignore +from google.api_core import datetime_helpers # type: ignore from google.cloud.bigquery import _helpers from google.cloud.bigquery_v2 import types from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration diff --git a/google/cloud/bigquery/opentelemetry_tracing.py b/google/cloud/bigquery/opentelemetry_tracing.py index 57f258ac4..e1e1d2157 100644 --- a/google/cloud/bigquery/opentelemetry_tracing.py +++ b/google/cloud/bigquery/opentelemetry_tracing.py @@ -14,13 +14,13 @@ import logging from contextlib import contextmanager -from google.api_core.exceptions import GoogleAPICallError +from google.api_core.exceptions import GoogleAPICallError # type: ignore logger = logging.getLogger(__name__) try: - from opentelemetry import trace - from opentelemetry.instrumentation.utils import http_status_to_canonical_code - from opentelemetry.trace.status import Status + from opentelemetry import trace # type: ignore + from opentelemetry.instrumentation.utils import http_status_to_canonical_code # type: ignore + from opentelemetry.trace.status import Status # type: ignore HAS_OPENTELEMETRY = True _warned_telemetry = True diff --git a/google/cloud/bigquery/retry.py b/google/cloud/bigquery/retry.py index 20a8e7b13..de7310db9 100644 --- a/google/cloud/bigquery/retry.py +++ b/google/cloud/bigquery/retry.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.api_core import exceptions -from google.api_core import retry +from google.api_core import exceptions # type: ignore +from google.api_core import retry # type: ignore import requests.exceptions diff --git a/google/cloud/bigquery/routine/routine.py b/google/cloud/bigquery/routine/routine.py index 103799e8f..766eb1dc5 100644 --- a/google/cloud/bigquery/routine/routine.py +++ b/google/cloud/bigquery/routine/routine.py @@ -119,12 +119,14 @@ def created(self): Read-only. """ + # pytype: disable=module-attr value = self._properties.get(self._PROPERTY_TO_API_FIELD["created"]) if value is not None and value != 0: # value will be in milliseconds. return google.cloud._helpers._datetime_from_microseconds( 1000.0 * float(value) ) + # pytype: enable=module-attr @property def modified(self): @@ -133,12 +135,14 @@ def modified(self): Read-only. """ + # pytype: disable=module-attr value = self._properties.get(self._PROPERTY_TO_API_FIELD["modified"]) if value is not None and value != 0: # value will be in milliseconds. return google.cloud._helpers._datetime_from_microseconds( 1000.0 * float(value) ) + # pytype: enable=module-attr @property def language(self): @@ -438,17 +442,17 @@ def __init__(self): @property def project(self): """str: ID of the project containing the routine.""" - return self._properties["projectId"] + return self._properties["projectId"] # pytype: disable=key-error @property def dataset_id(self): """str: ID of dataset containing the routine.""" - return self._properties["datasetId"] + return self._properties["datasetId"] # pytype: disable=key-error @property def routine_id(self): """str: The routine ID.""" - return self._properties["routineId"] + return self._properties["routineId"] # pytype: disable=key-error @property def path(self): diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py index bd5bca30f..b73b42ac4 100644 --- a/google/cloud/bigquery/table.py +++ b/google/cloud/bigquery/table.py @@ -25,19 +25,19 @@ import warnings try: - import pandas + import pandas # type: ignore except ImportError: # pragma: NO COVER pandas = None try: - import pyarrow + import pyarrow # type: ignore except ImportError: # pragma: NO COVER pyarrow = None import google.api_core.exceptions -from google.api_core.page_iterator import HTTPIterator +from google.api_core.page_iterator import HTTPIterator # type: ignore -import google.cloud._helpers +import google.cloud._helpers # type: ignore from google.cloud.bigquery import _helpers from google.cloud.bigquery import _pandas_helpers from google.cloud.bigquery.schema import _build_schema_resource @@ -47,6 +47,10 @@ from google.cloud.bigquery.external_config import ExternalConfig from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration +# Types needed only for Type Hints +from google.cloud import bigquery_storage # type: ignore +from typing import Any, Dict + _LOGGER = logging.getLogger(__name__) @@ -143,7 +147,7 @@ def path(self): ) @classmethod - def from_string(cls, table_id, default_project=None): + def from_string(cls, table_id: str, default_project: str = None): """Construct a table reference from table ID string. Args: @@ -182,7 +186,7 @@ def from_string(cls, table_id, default_project=None): ) @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict): """Factory: construct a table reference given its API representation Args: @@ -847,7 +851,7 @@ def external_data_configuration(self, value): self._properties["externalDataConfiguration"] = api_repr @classmethod - def from_string(cls, full_table_id): + def from_string(cls, full_table_id: str): """Construct a table from fully-qualified table ID. Args: @@ -871,7 +875,7 @@ def from_string(cls, full_table_id): return cls(TableReference.from_string(full_table_id)) @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict): """Factory: construct a table given its API representation Args: @@ -1104,7 +1108,7 @@ def clustering_fields(self): return list(prop.get("fields", ())) @classmethod - def from_string(cls, full_table_id): + def from_string(cls, full_table_id: str): """Construct a table from fully-qualified table ID. Args: @@ -1259,7 +1263,7 @@ def items(self): for key, index in self._xxx_field_to_index.items(): yield (key, copy.deepcopy(self._xxx_values[index])) - def get(self, key, default=None): + def get(self, key: str, default: Any = None): """Return a value for key, with a default value if it does not exist. Args: @@ -1520,9 +1524,9 @@ def _to_arrow_iterable(self, bqstorage_client=None): # changes to job.QueryJob.to_arrow() def to_arrow( self, - progress_bar_type=None, - bqstorage_client=None, - create_bqstorage_client=True, + progress_bar_type: str = None, + bqstorage_client: bigquery_storage.BigQueryReadClient = None, + create_bqstorage_client: bool = True, ): """[Beta] Create a class:`pyarrow.Table` by loading all pages of a table or query. @@ -1623,9 +1627,9 @@ def to_arrow( def to_dataframe_iterable( self, - bqstorage_client=None, - dtypes=None, - max_queue_size=_pandas_helpers._MAX_QUEUE_SIZE_DEFAULT, + bqstorage_client: bigquery_storage.BigQueryReadClient = None, + dtypes: Dict[str, Any] = None, + max_queue_size: int = _pandas_helpers._MAX_QUEUE_SIZE_DEFAULT, ): """Create an iterable of pandas DataFrames, to process the table as a stream. @@ -1698,11 +1702,11 @@ def to_dataframe_iterable( # changes to job.QueryJob.to_dataframe() def to_dataframe( self, - bqstorage_client=None, - dtypes=None, - progress_bar_type=None, - create_bqstorage_client=True, - date_as_object=True, + bqstorage_client: bigquery_storage.BigQueryReadClient = None, + dtypes: Dict[str, Any] = None, + progress_bar_type: str = None, + create_bqstorage_client: bool = True, + date_as_object: bool = True, ): """Create a pandas DataFrame by loading all pages of a query. @@ -2164,7 +2168,7 @@ def require_partition_filter(self, value): self._properties["requirePartitionFilter"] = value @classmethod - def from_api_repr(cls, api_repr): + def from_api_repr(cls, api_repr: dict): """Return a :class:`TimePartitioning` object deserialized from a dict. This method creates a new ``TimePartitioning`` instance that points to diff --git a/noxfile.py b/noxfile.py index a738d8c00..5561833f8 100644 --- a/noxfile.py +++ b/noxfile.py @@ -21,6 +21,7 @@ import nox +PYTYPE_VERSION = "pytype==2020.7.24" BLACK_VERSION = "black==19.10b0" BLACK_PATHS = ("docs", "google", "samples", "tests", "noxfile.py", "setup.py") @@ -90,6 +91,14 @@ def unit(session): default(session) +@nox.session(python="3.8") +def pytype(session): + """Run pytype + """ + session.install(PYTYPE_VERSION) + session.run("pytype") + + @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) def system(session): """Run the system test suite.""" diff --git a/setup.cfg b/setup.cfg index c3a2b39f6..0e2e0437b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,3 +17,13 @@ # Generated by synthtool. DO NOT EDIT! [bdist_wheel] universal = 1 +[pytype] +python_version = 3.8 +inputs = + google/cloud/ +exclude = + tests/ + google/cloud/bigquery_v2/ +output = .pytype/ +# Workaround for https://github.com/google/pytype/issues/150 +disable = pyi-error \ No newline at end of file From 6a84a87f848ae1e32880f97db8da879d5407f114 Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Tue, 22 Dec 2020 15:51:55 +0530 Subject: [PATCH 02/22] feat: add bigquery-storage in requirement file --- samples/geography/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 6939c07e0..96819343c 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -1,3 +1,4 @@ geojson==2.5.0 google-cloud-bigquery==2.13.1 +google-cloud-bigquery-storage==2.1.0 Shapely==1.7.1 From 887ec2716818a1dc7ad134ea6de323ff19f03e38 Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Tue, 22 Dec 2020 17:02:29 +0530 Subject: [PATCH 03/22] feat: add pandas in requirement file --- samples/geography/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 96819343c..42e047890 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -1,4 +1,5 @@ geojson==2.5.0 google-cloud-bigquery==2.13.1 google-cloud-bigquery-storage==2.1.0 +pandas==1.1.5 Shapely==1.7.1 From 984d8a0d5640cb6ca3628e2cbd880334eb70a37d Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Thu, 24 Dec 2020 21:28:39 +0530 Subject: [PATCH 04/22] feat: add return type hint --- google/cloud/bigquery/client.py | 89 ++++++++++++------------ google/cloud/bigquery/dataset.py | 16 +++-- google/cloud/bigquery/external_config.py | 28 ++++---- google/cloud/bigquery/job/base.py | 16 +++-- google/cloud/bigquery/job/extract.py | 2 +- google/cloud/bigquery/job/load.py | 2 +- google/cloud/bigquery/job/query.py | 17 ++--- google/cloud/bigquery/model.py | 10 +-- google/cloud/bigquery/query.py | 20 +++--- google/cloud/bigquery/routine/routine.py | 16 +++-- google/cloud/bigquery/schema.py | 10 +-- google/cloud/bigquery/table.py | 46 ++++++------ samples/geography/requirements.txt | 1 - 13 files changed, 141 insertions(+), 132 deletions(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 8d57cbf09..c14f6db9a 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -76,7 +76,6 @@ # Types needed only for Type Hints import datetime -import pandas from google.cloud.bigquery.job import ( _AsyncJob, LoadJobConfig, @@ -233,7 +232,7 @@ def get_service_account_email( project: str = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> str: """Get the email address of the project's BigQuery service account Note: @@ -280,7 +279,7 @@ def list_projects( page_token: str = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> page_iterator.Iterator: """List projects for the project associated with this client. See @@ -340,7 +339,7 @@ def list_datasets( page_token: str = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> page_iterator.Iterator: """List datasets for the project associated with this client. See @@ -410,7 +409,7 @@ def api_request(*args, **kwargs): extra_params=extra_params, ) - def dataset(self, dataset_id: str, project: str = None): + def dataset(self, dataset_id: str, project: str = None) -> DatasetReference: """Deprecated: Construct a reference to a dataset. .. deprecated:: 1.24.0 @@ -491,7 +490,7 @@ def create_dataset( exists_ok: bool = False, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Dataset: """API call: create the dataset via a POST request. See @@ -566,7 +565,7 @@ def create_routine( exists_ok: bool = False, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Routine: """[Beta] Create a routine via a POST request. See @@ -621,7 +620,7 @@ def create_table( exists_ok: bool = False, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Table: """API call: create a table via a PUT request See @@ -693,7 +692,7 @@ def get_dataset( dataset_ref: Union[DatasetReference, str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Dataset: """Fetch the dataset referenced by ``dataset_ref`` Args: @@ -737,7 +736,7 @@ def get_iam_policy( requested_policy_version: int = 1, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Policy: if not isinstance(table, (Table, TableReference)): raise TypeError("table must be a Table or TableReference") @@ -767,7 +766,7 @@ def set_iam_policy( updateMask: str = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Policy: if not isinstance(table, (Table, TableReference)): raise TypeError("table must be a Table or TableReference") @@ -825,7 +824,7 @@ def get_model( model_ref: Union[ModelReference, str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Model: """[Beta] Fetch the model referenced by ``model_ref``. Args: @@ -868,7 +867,7 @@ def get_routine( routine_ref: Union[Routine, RoutineReference, str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Routine: """[Beta] Get the routine referenced by ``routine_ref``. Args: @@ -912,7 +911,7 @@ def get_table( table: Union[Table, TableReference, str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Table: """Fetch the table referenced by ``table``. Args: @@ -954,7 +953,7 @@ def update_dataset( fields: Sequence[str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Dataset: """Change some fields of a dataset. Use ``fields`` to specify which fields to update. At least one field @@ -1024,7 +1023,7 @@ def update_model( fields: Sequence[str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Model: """[Beta] Change some fields of a model. Use ``fields`` to specify which fields to update. At least one field @@ -1088,7 +1087,7 @@ def update_routine( fields: Sequence[str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Routine: """[Beta] Change some fields of a routine. Use ``fields`` to specify which fields to update. At least one field @@ -1162,7 +1161,7 @@ def update_table( fields: Sequence[str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Table: """Change some fields of a table. Use ``fields`` to specify which fields to update. At least one field @@ -1228,7 +1227,7 @@ def list_models( page_token: str = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> page_iterator.Iterator: """[Beta] List models in the dataset. See @@ -1300,7 +1299,7 @@ def list_routines( page_token: str = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> page_iterator.Iterator: """[Beta] List routines in the dataset. See @@ -1372,7 +1371,7 @@ def list_tables( page_token: str = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> page_iterator.Iterator: """List tables in the dataset. See @@ -1443,7 +1442,7 @@ def delete_dataset( retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, not_found_ok: bool = False, - ): + ) -> None: """Delete a dataset. See @@ -1502,7 +1501,7 @@ def delete_model( retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, not_found_ok: bool = False, - ): + ) -> None: """[Beta] Delete a model See @@ -1554,7 +1553,7 @@ def delete_routine( retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, not_found_ok: bool = False, - ): + ) -> None: """[Beta] Delete a routine. See @@ -1608,7 +1607,7 @@ def delete_table( retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, not_found_ok: bool = False, - ): + ) -> None: """Delete a table See @@ -1718,7 +1717,7 @@ def _get_query_results( ) return _QueryResults.from_api_repr(resource) - def job_from_resource(self, resource: dict): + def job_from_resource(self, resource: dict) -> job.UnknownJob: """Detect correct job type from resource and instantiate. Args: @@ -1749,7 +1748,7 @@ def create_job( job_config: dict, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Union[job.LoadJob, job.CopyJob, job.ExtractJob, job.QueryJob]: """Create a new job. Args: job_config (dict): configuration job representation returned from the API. @@ -1846,7 +1845,7 @@ def get_job( location: str = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Union[job.LoadJob, job.CopyJob, job.ExtractJob, job.QueryJob]: """Fetch a job for the project associated with this client. See @@ -1908,7 +1907,7 @@ def cancel_job( location: str = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Union[job.LoadJob, job.CopyJob, job.ExtractJob, job.QueryJob]: """Attempt to cancel a job from a job ID. See @@ -1975,7 +1974,7 @@ def list_jobs( timeout: float = None, min_creation_time: datetime.datetime = None, max_creation_time: datetime.datetime = None, - ): + ) -> page_iterator.Iterator: """List jobs for the project associated with this client. See @@ -2084,7 +2083,7 @@ def load_table_from_uri( job_config: LoadJobConfig = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> job.LoadJob: """Starts a job for loading data into a table from CloudStorage. See @@ -2168,7 +2167,7 @@ def load_table_from_file( project: str = None, job_config: LoadJobConfig = None, timeout: float = None, - ): + ) -> job.LoadJob: """Upload the contents of this table from a file-like object. Similar to :meth:`load_table_from_uri`, this method creates, starts and @@ -2261,7 +2260,7 @@ def load_table_from_file( def load_table_from_dataframe( self, - dataframe: pandas.DataFrame, + dataframe: "pandas.DataFrame", destination: Union[Table, TableReference, str], num_retries: int = _DEFAULT_NUM_RETRIES, job_id: str = None, @@ -2271,7 +2270,7 @@ def load_table_from_dataframe( job_config: LoadJobConfig = None, parquet_compression: str = "snappy", timeout: float = None, - ): + ) -> job.LoadJob: """Upload the contents of a table from a pandas DataFrame. Similar to :meth:`load_table_from_uri`, this method creates, starts and @@ -2497,7 +2496,7 @@ def load_table_from_json( project: str = None, job_config: LoadJobConfig = None, timeout: float = None, - ): + ) -> job.LoadJob: """Upload the contents of a table from a JSON string or dict. Args: @@ -2778,7 +2777,7 @@ def copy_table( job_config: CopyJobConfig = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> job.CopyJob: """Copy one or more tables to another table. See @@ -2882,7 +2881,7 @@ def extract_table( retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, source_type: str = "Table", - ): + ) -> job.ExtractJob: """Start a job to extract a table into Cloud Storage files. See @@ -2979,7 +2978,7 @@ def query( project: str = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> job.QueryJob: """Run a SQL query. See @@ -3062,7 +3061,7 @@ def insert_rows( rows: Union[Sequence[Tuple], Sequence[Dict]], selected_fields: Sequence[SchemaField] = None, **kwargs: dict, - ): + ) -> Sequence[dict]: """Insert rows into a table via the streaming API. See @@ -3127,11 +3126,11 @@ def insert_rows( def insert_rows_from_dataframe( self, table: Union[Table, TableReference, str], - dataframe: pandas.DataFrame, + dataframe: "pandas.DataFrame", selected_fields: Sequence[SchemaField] = None, chunk_size: int = 500, **kwargs: Dict, - ): + ) -> Sequence[Sequence[dict]]: """Insert rows into a table from a dataframe via the streaming API. Args: @@ -3187,7 +3186,7 @@ def insert_rows_json( template_suffix: str = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Sequence[dict]: """Insert rows into a table without applying local type conversions. See @@ -3288,7 +3287,7 @@ def list_partitions( table: Union[Table, TableReference, str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Sequence[str]: """List the partitions in a table. Args: @@ -3338,7 +3337,7 @@ def list_rows( page_size: int = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> RowIterator: """List the rows of the table. See @@ -3450,7 +3449,7 @@ def _list_rows_from_query_results( page_size: int = None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> RowIterator: """List the rows of a completed query. See https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/getQueryResults diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index 2943dc8b5..ff015d605 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -220,7 +220,7 @@ def to_api_repr(self): return resource @classmethod - def from_api_repr(cls, resource: dict): + def from_api_repr(cls, resource: dict) -> "AccessEntry": """Factory: construct an access entry given its API representation Args: @@ -288,7 +288,7 @@ def path(self): routine = _get_routine_reference @classmethod - def from_api_repr(cls, resource: dict): + def from_api_repr(cls, resource: dict) -> "DatasetReference": """Factory: construct a dataset reference given its API representation Args: @@ -304,7 +304,9 @@ def from_api_repr(cls, resource: dict): return cls(project, dataset_id) @classmethod - def from_string(cls, dataset_id: str, default_project: str = None): + def from_string( + cls, dataset_id: str, default_project: str = None + ) -> "DatasetReference": """Construct a dataset reference from dataset ID string. Args: @@ -350,7 +352,7 @@ def from_string(cls, dataset_id: str, default_project: str = None): return cls(output_project_id, output_dataset_id) - def to_api_repr(self): + def to_api_repr(self) -> dict: """Construct the API resource representation of this dataset reference Returns: @@ -640,7 +642,7 @@ def default_encryption_configuration(self, value): self._properties["defaultEncryptionConfiguration"] = api_repr @classmethod - def from_string(cls, full_dataset_id: str): + def from_string(cls, full_dataset_id: str) -> "Dataset": """Construct a dataset from fully-qualified dataset ID. Args: @@ -664,7 +666,7 @@ def from_string(cls, full_dataset_id: str): return cls(DatasetReference.from_string(full_dataset_id)) @classmethod - def from_api_repr(cls, resource: dict): + def from_api_repr(cls, resource: dict) -> "Dataset": """Factory: construct a dataset given its API representation Args: @@ -689,7 +691,7 @@ def from_api_repr(cls, resource: dict): dataset._properties = copy.deepcopy(resource) return dataset - def to_api_repr(self): + def to_api_repr(self) -> dict: """Construct the API resource representation of this dataset Returns: diff --git a/google/cloud/bigquery/external_config.py b/google/cloud/bigquery/external_config.py index 59e4960f9..ef4d569fa 100644 --- a/google/cloud/bigquery/external_config.py +++ b/google/cloud/bigquery/external_config.py @@ -149,7 +149,7 @@ def type_(self): def type_(self, value): self._properties["type"] = value - def to_api_repr(self): + def to_api_repr(self) -> dict: """Build an API representation of this object. Returns: @@ -159,7 +159,7 @@ def to_api_repr(self): return copy.deepcopy(self._properties) @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "BigtableColumn": """Factory: construct a :class:`~.external_config.BigtableColumn` instance given its API representation. @@ -251,7 +251,7 @@ def columns(self): def columns(self, value): self._properties["columns"] = [col.to_api_repr() for col in value] - def to_api_repr(self): + def to_api_repr(self) -> dict: """Build an API representation of this object. Returns: @@ -261,7 +261,7 @@ def to_api_repr(self): return copy.deepcopy(self._properties) @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "BigtableColumnFamily": """Factory: construct a :class:`~.external_config.BigtableColumnFamily` instance given its API representation. @@ -333,7 +333,7 @@ def column_families(self): def column_families(self, value): self._properties["columnFamilies"] = [cf.to_api_repr() for cf in value] - def to_api_repr(self): + def to_api_repr(self) -> dict: """Build an API representation of this object. Returns: @@ -343,7 +343,7 @@ def to_api_repr(self): return copy.deepcopy(self._properties) @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "BigtableOptions": """Factory: construct a :class:`~.external_config.BigtableOptions` instance given its API representation. @@ -450,7 +450,7 @@ def skip_leading_rows(self): def skip_leading_rows(self, value): self._properties["skipLeadingRows"] = str(value) - def to_api_repr(self): + def to_api_repr(self) -> dict: """Build an API representation of this object. Returns: @@ -459,7 +459,7 @@ def to_api_repr(self): return copy.deepcopy(self._properties) @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "CSVOptions": """Factory: construct a :class:`~.external_config.CSVOptions` instance given its API representation. @@ -513,7 +513,7 @@ def range(self): def range(self, value): self._properties["range"] = value - def to_api_repr(self): + def to_api_repr(self) -> dict: """Build an API representation of this object. Returns: @@ -522,7 +522,7 @@ def to_api_repr(self): return copy.deepcopy(self._properties) @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "GoogleSheetsOptions": """Factory: construct a :class:`~.external_config.GoogleSheetsOptions` instance given its API representation. @@ -601,7 +601,7 @@ def require_partition_filter(self): def require_partition_filter(self, value): self._properties["requirePartitionFilter"] = value - def to_api_repr(self): + def to_api_repr(self) -> dict: """Build an API representation of this object. Returns: @@ -610,7 +610,7 @@ def to_api_repr(self): return copy.deepcopy(self._properties) @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "HivePartitioningOptions": """Factory: construct a :class:`~.external_config.HivePartitioningOptions` instance given its API representation. @@ -784,7 +784,7 @@ def schema(self, value): prop = {"fields": [field.to_api_repr() for field in value]} self._properties["schema"] = prop - def to_api_repr(self): + def to_api_repr(self) -> dict: """Build an API representation of this object. Returns: @@ -799,7 +799,7 @@ def to_api_repr(self): return config @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "ExternalConfig": """Factory: construct an :class:`~.external_config.ExternalConfig` instance given its API representation. diff --git a/google/cloud/bigquery/job/base.py b/google/cloud/bigquery/job/base.py index eaea49d84..90f8a1278 100644 --- a/google/cloud/bigquery/job/base.py +++ b/google/cloud/bigquery/job/base.py @@ -471,7 +471,7 @@ def _begin(self, client=None, retry=DEFAULT_RETRY, timeout=None): def exists( self, client=None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None - ): + ) -> bool: """API call: test for the existence of the job via a GET request See @@ -553,7 +553,7 @@ def reload( def cancel( self, client=None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None - ): + ) -> bool: """API call: cancel job via a POST request See @@ -624,7 +624,7 @@ def done( retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, reload: bool = True, - ): + ) -> bool: """Checks if the job is complete. Args: @@ -647,7 +647,9 @@ def done( self.reload(retry=retry, timeout=timeout) return self.state == _DONE_STATE - def result(self, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None): + def result( + self, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None + ) -> "_AsyncJob": """Start the job and wait for it to complete and get the result. Args: @@ -802,7 +804,7 @@ def _del_sub_prop(self, key): """ _helpers._del_sub_prop(self._properties, [self._job_type, key]) - def to_api_repr(self): + def to_api_repr(self) -> dict: """Build an API representation of the job config. Returns: @@ -845,7 +847,7 @@ def _fill_from_default(self, default_job_config): return new_job_config @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "_JobConfig": """Factory: construct a job configuration given its API representation Args: @@ -943,7 +945,7 @@ class UnknownJob(_AsyncJob): """A job whose type cannot be determined.""" @classmethod - def from_api_repr(cls, resource, client): + def from_api_repr(cls, resource: dict, client) -> "UnknownJob": """Construct an UnknownJob from the JSON representation. Args: diff --git a/google/cloud/bigquery/job/extract.py b/google/cloud/bigquery/job/extract.py index a6e262a32..3373bcdef 100644 --- a/google/cloud/bigquery/job/extract.py +++ b/google/cloud/bigquery/job/extract.py @@ -241,7 +241,7 @@ def to_api_repr(self): } @classmethod - def from_api_repr(cls, resource, client): + def from_api_repr(cls, resource: dict, client) -> "ExtractJob": """Factory: construct a job given its API representation .. note: diff --git a/google/cloud/bigquery/job/load.py b/google/cloud/bigquery/job/load.py index e784af0a6..b8174af3e 100644 --- a/google/cloud/bigquery/job/load.py +++ b/google/cloud/bigquery/job/load.py @@ -733,7 +733,7 @@ def to_api_repr(self): } @classmethod - def from_api_repr(cls, resource, client): + def from_api_repr(cls, resource: dict, client) -> "LoadJob": """Factory: construct a job given its API representation .. note: diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index b0db28851..6de8a7a3a 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -49,7 +49,8 @@ # Types needed only for Type Hints from google.api_core import retry as retries # type: ignore from google.cloud import bigquery_storage # type: ignore -from typing import Any, Dict +from google.cloud.bigquery.table import RowIterator +from typing import Any, Dict, Union _CONTAINS_ORDER_BY = re.compile(r"ORDER\s+BY", re.IGNORECASE) @@ -496,7 +497,7 @@ def schema_update_options(self): def schema_update_options(self, values): self._set_sub_prop("schemaUpdateOptions", values) - def to_api_repr(self): + def to_api_repr(self) -> dict: """Build an API representation of the query job config. Returns: @@ -723,7 +724,7 @@ def to_api_repr(self): } @classmethod - def from_api_repr(cls, resource, client): + def from_api_repr(cls, resource: dict, client) -> "QueryJob": """Factory: construct a job given its API representation Args: @@ -1123,7 +1124,7 @@ def result( retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, start_index: int = None, - ): + ) -> Union[RowIterator, _EmptyRowIterator]: """Start the job and wait for it to complete and get the result. Args: @@ -1206,7 +1207,7 @@ def to_arrow( progress_bar_type: str = None, bqstorage_client: bigquery_storage.BigQueryReadClient = None, create_bqstorage_client: bool = True, - ): + ) -> Any: """[Beta] Create a class:`pyarrow.Table` by loading all pages of a table or query. @@ -1277,7 +1278,7 @@ def to_dataframe( progress_bar_type: str = None, create_bqstorage_client: bool = True, date_as_object: bool = True, - ): + ) -> Any: """Return a pandas DataFrame from a QueryJob Args: @@ -1357,7 +1358,7 @@ def __init__(self, kind, substeps): self.substeps = list(substeps) @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "QueryPlanEntryStep": """Factory: construct instance from the JSON repr. Args: @@ -1387,7 +1388,7 @@ def __init__(self): self._properties = {} @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "QueryPlanEntry": """Factory: construct instance from the JSON repr. Args: diff --git a/google/cloud/bigquery/model.py b/google/cloud/bigquery/model.py index 860c3a907..cdb411e08 100644 --- a/google/cloud/bigquery/model.py +++ b/google/cloud/bigquery/model.py @@ -279,7 +279,7 @@ def encryption_configuration(self, value): self._properties["encryptionConfiguration"] = api_repr @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "Model": """Factory: construct a model resource given its API representation Args: @@ -322,7 +322,7 @@ def _build_resource(self, filter_fields): def __repr__(self): return "Model(reference={})".format(repr(self.reference)) - def to_api_repr(self): + def to_api_repr(self) -> dict: """Construct the API resource representation of this model. Returns: @@ -389,7 +389,9 @@ def from_api_repr(cls, resource): return ref @classmethod - def from_string(cls, model_id, default_project=None): + def from_string( + cls, model_id: str, default_project: str = None + ) -> "ModelReference": """Construct a model reference from model ID string. Args: @@ -417,7 +419,7 @@ def from_string(cls, model_id, default_project=None): {"projectId": proj, "datasetId": dset, "modelId": model} ) - def to_api_repr(self): + def to_api_repr(self) -> dict: """Construct the API resource representation of this model reference. Returns: diff --git a/google/cloud/bigquery/query.py b/google/cloud/bigquery/query.py index 495c4effb..3751eb124 100644 --- a/google/cloud/bigquery/query.py +++ b/google/cloud/bigquery/query.py @@ -286,7 +286,7 @@ class _AbstractQueryParameter(object): """ @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "ScalarQueryParameter": """Factory: construct parameter from JSON resource. Args: @@ -297,7 +297,7 @@ def from_api_repr(cls, resource): """ raise NotImplementedError - def to_api_repr(self): + def to_api_repr(self) -> dict: """Construct JSON API representation for the parameter. Returns: @@ -329,7 +329,7 @@ def __init__(self, name, type_, value): self.value = value @classmethod - def positional(cls, type_, value): + def positional(cls, type_: str, value) -> "ScalarQueryParameter": """Factory for positional paramater. Args: @@ -347,7 +347,7 @@ def positional(cls, type_, value): return cls(None, type_, value) @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "ScalarQueryParameter": """Factory: construct parameter from JSON resource. Args: @@ -369,7 +369,7 @@ def from_api_repr(cls, resource): return cls(name, type_, converted) - def to_api_repr(self): + def to_api_repr(self) -> dict: """Construct JSON API representation for the parameter. Returns: @@ -441,7 +441,7 @@ def __init__(self, name, array_type, values): self.array_type = array_type @classmethod - def positional(cls, array_type, values): + def positional(cls, array_type: str, values: list) -> "ArrayQueryParameter": """Factory for positional parameters. Args: @@ -490,7 +490,7 @@ def _from_api_repr_scalar(cls, resource): return cls(name, array_type, converted) @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "ArrayQueryParameter": """Factory: construct parameter from JSON resource. Args: @@ -504,7 +504,7 @@ def from_api_repr(cls, resource): return cls._from_api_repr_struct(resource) return cls._from_api_repr_scalar(resource) - def to_api_repr(self): + def to_api_repr(self) -> dict: """Construct JSON API representation for the parameter. Returns: @@ -623,7 +623,7 @@ def positional(cls, *sub_params): return cls(None, *sub_params) @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "StructQueryParameter": """Factory: construct parameter from JSON resource. Args: @@ -663,7 +663,7 @@ def from_api_repr(cls, resource): instance.struct_values[key] = converted return instance - def to_api_repr(self): + def to_api_repr(self) -> dict: """Construct JSON API representation for the parameter. Returns: diff --git a/google/cloud/bigquery/routine/routine.py b/google/cloud/bigquery/routine/routine.py index 766eb1dc5..670fdf580 100644 --- a/google/cloud/bigquery/routine/routine.py +++ b/google/cloud/bigquery/routine/routine.py @@ -270,7 +270,7 @@ def determinism_level(self, value): self._properties[self._PROPERTY_TO_API_FIELD["determinism_level"]] = value @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "Routine": """Factory: construct a routine given its API representation. Args: @@ -285,7 +285,7 @@ def from_api_repr(cls, resource): ref._properties = resource return ref - def to_api_repr(self): + def to_api_repr(self) -> dict: """Construct the API resource representation of this routine. Returns: @@ -391,7 +391,7 @@ def data_type(self, value): self._properties[self._PROPERTY_TO_API_FIELD["data_type"]] = resource @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "RoutineArgument": """Factory: construct a routine argument given its API representation. Args: @@ -405,7 +405,7 @@ def from_api_repr(cls, resource): ref._properties = resource return ref - def to_api_repr(self): + def to_api_repr(self) -> dict: """Construct the API resource representation of this routine argument. Returns: @@ -464,7 +464,7 @@ def path(self): ) @classmethod - def from_api_repr(cls, resource): + def from_api_repr(cls, resource: dict) -> "RoutineReference": """Factory: construct a routine reference given its API representation. Args: @@ -480,7 +480,9 @@ def from_api_repr(cls, resource): return ref @classmethod - def from_string(cls, routine_id, default_project=None): + def from_string( + cls, routine_id: str, default_project: str = None + ) -> "RoutineReference": """Factory: construct a routine reference from routine ID string. Args: @@ -508,7 +510,7 @@ def from_string(cls, routine_id, default_project=None): {"projectId": proj, "datasetId": dset, "routineId": routine} ) - def to_api_repr(self): + def to_api_repr(self) -> dict: """Construct the API resource representation of this routine reference. Returns: diff --git a/google/cloud/bigquery/schema.py b/google/cloud/bigquery/schema.py index 680dcc138..9635e830d 100644 --- a/google/cloud/bigquery/schema.py +++ b/google/cloud/bigquery/schema.py @@ -90,7 +90,7 @@ def __init__( self._policy_tags = policy_tags @classmethod - def from_api_repr(cls, api_repr): + def from_api_repr(cls, api_repr: dict) -> "SchemaField": """Return a ``SchemaField`` object deserialized from a dictionary. Args: @@ -163,7 +163,7 @@ def policy_tags(self): """ return self._policy_tags - def to_api_repr(self): + def to_api_repr(self) -> dict: """Return a dictionary representing this schema field. Returns: @@ -200,7 +200,7 @@ def _key(self): self._policy_tags, ) - def to_standard_sql(self): + def to_standard_sql(self) -> types.StandardSqlField: """Return the field as the standard SQL field representation object. Returns: @@ -375,7 +375,7 @@ def __repr__(self): return "PolicyTagList{}".format(self._key()) @classmethod - def from_api_repr(cls, api_repr): + def from_api_repr(cls, api_repr: dict) -> "PolicyTagList": """Return a :class:`PolicyTagList` object deserialized from a dict. This method creates a new ``PolicyTagList`` instance that points to @@ -398,7 +398,7 @@ def from_api_repr(cls, api_repr): names = api_repr.get("names", ()) return cls(names=names) - def to_api_repr(self): + def to_api_repr(self) -> dict: """Return a dictionary representing this object. This method returns the properties dict of the ``PolicyTagList`` diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py index b73b42ac4..0230a941c 100644 --- a/google/cloud/bigquery/table.py +++ b/google/cloud/bigquery/table.py @@ -49,7 +49,7 @@ # Types needed only for Type Hints from google.cloud import bigquery_storage # type: ignore -from typing import Any, Dict +from typing import Any, Dict, Iterable, Tuple _LOGGER = logging.getLogger(__name__) @@ -147,7 +147,9 @@ def path(self): ) @classmethod - def from_string(cls, table_id: str, default_project: str = None): + def from_string( + cls, table_id: str, default_project: str = None + ) -> "TableReference": """Construct a table reference from table ID string. Args: @@ -186,7 +188,7 @@ def from_string(cls, table_id: str, default_project: str = None): ) @classmethod - def from_api_repr(cls, resource: dict): + def from_api_repr(cls, resource: dict) -> "TableReference": """Factory: construct a table reference given its API representation Args: @@ -204,7 +206,7 @@ def from_api_repr(cls, resource: dict): table_id = resource["tableId"] return cls(DatasetReference(project, dataset_id), table_id) - def to_api_repr(self): + def to_api_repr(self) -> dict: """Construct the API resource representation of this table reference. Returns: @@ -216,7 +218,7 @@ def to_api_repr(self): "tableId": self._table_id, } - def to_bqstorage(self): + def to_bqstorage(self) -> str: """Construct a BigQuery Storage API representation of this table. Install the ``google-cloud-bigquery-storage`` package to use this @@ -851,7 +853,7 @@ def external_data_configuration(self, value): self._properties["externalDataConfiguration"] = api_repr @classmethod - def from_string(cls, full_table_id: str): + def from_string(cls, full_table_id: str) -> "Table": """Construct a table from fully-qualified table ID. Args: @@ -875,7 +877,7 @@ def from_string(cls, full_table_id: str): return cls(TableReference.from_string(full_table_id)) @classmethod - def from_api_repr(cls, resource: dict): + def from_api_repr(cls, resource: dict) -> "Table": """Factory: construct a table given its API representation Args: @@ -911,7 +913,7 @@ def from_api_repr(cls, resource: dict): return table - def to_api_repr(self): + def to_api_repr(self) -> dict: """Constructs the API resource of this table Returns: @@ -919,7 +921,7 @@ def to_api_repr(self): """ return copy.deepcopy(self._properties) - def to_bqstorage(self): + def to_bqstorage(self) -> str: """Construct a BigQuery Storage API representation of this table. Returns: @@ -1108,7 +1110,7 @@ def clustering_fields(self): return list(prop.get("fields", ())) @classmethod - def from_string(cls, full_table_id: str): + def from_string(cls, full_table_id: str) -> "TableListItem": """Construct a table from fully-qualified table ID. Args: @@ -1133,7 +1135,7 @@ def from_string(cls, full_table_id: str): {"tableReference": TableReference.from_string(full_table_id).to_api_repr()} ) - def to_bqstorage(self): + def to_bqstorage(self) -> str: """Construct a BigQuery Storage API representation of this table. Returns: @@ -1141,7 +1143,7 @@ def to_bqstorage(self): """ return self.reference.to_bqstorage() - def to_api_repr(self): + def to_api_repr(self) -> dict: """Constructs the API resource of this table Returns: @@ -1235,7 +1237,7 @@ def values(self): """ return copy.deepcopy(self._xxx_values) - def keys(self): + def keys(self) -> Iterable[str]: """Return the keys for using a row as a dict. Returns: @@ -1248,7 +1250,7 @@ def keys(self): """ return self._xxx_field_to_index.keys() - def items(self): + def items(self) -> Iterable[Tuple[str, Any]]: """Return items as ``(key, value)`` pairs. Returns: @@ -1263,7 +1265,7 @@ def items(self): for key, index in self._xxx_field_to_index.items(): yield (key, copy.deepcopy(self._xxx_values[index])) - def get(self, key: str, default: Any = None): + def get(self, key: str, default: Any = None) -> Any: """Return a value for key, with a default value if it does not exist. Args: @@ -1527,7 +1529,7 @@ def to_arrow( progress_bar_type: str = None, bqstorage_client: bigquery_storage.BigQueryReadClient = None, create_bqstorage_client: bool = True, - ): + ) -> Any: """[Beta] Create a class:`pyarrow.Table` by loading all pages of a table or query. @@ -1630,7 +1632,7 @@ def to_dataframe_iterable( bqstorage_client: bigquery_storage.BigQueryReadClient = None, dtypes: Dict[str, Any] = None, max_queue_size: int = _pandas_helpers._MAX_QUEUE_SIZE_DEFAULT, - ): + ) -> Any: """Create an iterable of pandas DataFrames, to process the table as a stream. Args: @@ -1707,7 +1709,7 @@ def to_dataframe( progress_bar_type: str = None, create_bqstorage_client: bool = True, date_as_object: bool = True, - ): + ) -> Any: """Create a pandas DataFrame by loading all pages of a query. Args: @@ -1835,7 +1837,7 @@ def to_arrow( progress_bar_type=None, bqstorage_client=None, create_bqstorage_client=True, - ): + ) -> Any: """[Beta] Create an empty class:`pyarrow.Table`. Args: @@ -1857,7 +1859,7 @@ def to_dataframe( progress_bar_type=None, create_bqstorage_client=True, date_as_object=True, - ): + ) -> Any: """Create an empty dataframe. Args: @@ -2168,7 +2170,7 @@ def require_partition_filter(self, value): self._properties["requirePartitionFilter"] = value @classmethod - def from_api_repr(cls, api_repr: dict): + def from_api_repr(cls, api_repr: dict) -> "TimePartitioning": """Return a :class:`TimePartitioning` object deserialized from a dict. This method creates a new ``TimePartitioning`` instance that points to @@ -2196,7 +2198,7 @@ def from_api_repr(cls, api_repr: dict): instance._properties = api_repr return instance - def to_api_repr(self): + def to_api_repr(self) -> dict: """Return a dictionary representing this object. This method returns the properties dict of the ``TimePartitioning`` diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 42e047890..96819343c 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -1,5 +1,4 @@ geojson==2.5.0 google-cloud-bigquery==2.13.1 google-cloud-bigquery-storage==2.1.0 -pandas==1.1.5 Shapely==1.7.1 From 63a5729c12dffebf682ccd6d3733253e840ff2b2 Mon Sep 17 00:00:00 2001 From: HemangChothani Date: Mon, 28 Dec 2020 10:33:32 +0530 Subject: [PATCH 05/22] feat: remove pandas import as a string --- google/cloud/bigquery/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index c14f6db9a..c6b6b919c 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -2260,7 +2260,7 @@ def load_table_from_file( def load_table_from_dataframe( self, - dataframe: "pandas.DataFrame", + dataframe, destination: Union[Table, TableReference, str], num_retries: int = _DEFAULT_NUM_RETRIES, job_id: str = None, @@ -3126,7 +3126,7 @@ def insert_rows( def insert_rows_from_dataframe( self, table: Union[Table, TableReference, str], - dataframe: "pandas.DataFrame", + dataframe, selected_fields: Sequence[SchemaField] = None, chunk_size: int = 500, **kwargs: Dict, From c9fd3b798763ca4ee4791439ecb9e52c86d9a6d7 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 15 Apr 2021 10:39:00 +0200 Subject: [PATCH 06/22] Use the latest pytype version (2021.4.9) --- noxfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 5561833f8..916f73549 100644 --- a/noxfile.py +++ b/noxfile.py @@ -21,7 +21,7 @@ import nox -PYTYPE_VERSION = "pytype==2020.7.24" +PYTYPE_VERSION = "pytype==2021.4.9" BLACK_VERSION = "black==19.10b0" BLACK_PATHS = ("docs", "google", "samples", "tests", "noxfile.py", "setup.py") From 61cb93ca68c43127cb8fedb0a660fbc595bc1645 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 15 Apr 2021 11:30:44 +0200 Subject: [PATCH 07/22] Silence false import and module attribute errors --- google/cloud/bigquery/_pandas_helpers.py | 6 ++- google/cloud/bigquery/client.py | 52 ++++++++++++++---------- google/cloud/bigquery/dbapi/cursor.py | 4 +- google/cloud/bigquery/job/query.py | 6 ++- google/cloud/bigquery/magics/magics.py | 22 +++++----- 5 files changed, 53 insertions(+), 37 deletions(-) diff --git a/google/cloud/bigquery/_pandas_helpers.py b/google/cloud/bigquery/_pandas_helpers.py index c92839d4e..d2fccbb00 100644 --- a/google/cloud/bigquery/_pandas_helpers.py +++ b/google/cloud/bigquery/_pandas_helpers.py @@ -20,7 +20,7 @@ import queue import warnings -from packaging import version +from packaging import version # pytype: disable=import-error try: import pandas # type: ignore @@ -34,7 +34,9 @@ pyarrow = None try: - from google.cloud.bigquery_storage import ArrowSerializationOptions + from google.cloud.bigquery_storage import ( # pytype: disable=import-error + ArrowSerializationOptions, + ) except ImportError: _ARROW_COMPRESSION_SUPPORT = False else: diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index c6b6b919c..33fcedaf6 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -31,21 +31,25 @@ import warnings try: - import pyarrow + import pyarrow # pytype: disable=import-error except ImportError: # pragma: NO COVER pyarrow = None -from google import resumable_media -from google.resumable_media.requests import MultipartUpload -from google.resumable_media.requests import ResumableUpload +from google import resumable_media # pytype: disable=import-error +from google.resumable_media.requests import ( # pytype: disable=import-error + MultipartUpload, +) +from google.resumable_media.requests import ( # pytype: disable=import-error + ResumableUpload, +) import google.api_core.client_options -import google.api_core.exceptions -from google.api_core.iam import Policy -from google.api_core import page_iterator +import google.api_core.exceptions as core_exceptions # pytype: disable=import-error +from google.api_core.iam import Policy # pytype: disable=import-error +from google.api_core import page_iterator # pytype: disable=import-error import google.cloud._helpers -from google.cloud import exceptions -from google.cloud.client import ClientWithProject +from google.cloud import exceptions # pytype: disable=import-error +from google.cloud.client import ClientWithProject # pytype: disable=import-error from google.cloud.bigquery._helpers import _del_sub_prop from google.cloud.bigquery._helpers import _get_sub_prop @@ -199,7 +203,7 @@ def __init__( kw_args = {"client_info": client_info} if client_options: if type(client_options) == dict: - client_options = google.api_core.client_options.from_dict( + client_options = google.api_core.client_options.from_dict( # pytype: disable=module-attr client_options ) if client_options.api_endpoint: @@ -458,7 +462,7 @@ def _create_bqstorage_client(self): A BigQuery Storage API client. """ try: - from google.cloud import bigquery_storage + from google.cloud import bigquery_storage # pytype: disable=import-error except ImportError: warnings.warn( "Cannot create BigQuery Storage client, the dependency " @@ -554,7 +558,7 @@ def create_dataset( timeout=timeout, ) return Dataset.from_api_repr(api_response) - except google.api_core.exceptions.Conflict: + except core_exceptions.Conflict: if not exists_ok: raise return self.get_dataset(dataset.reference, retry=retry) @@ -609,7 +613,7 @@ def create_routine( timeout=timeout, ) return Routine.from_api_repr(api_response) - except google.api_core.exceptions.Conflict: + except core_exceptions.Conflict: # pytype: disable=module-attr if not exists_ok: raise return self.get_routine(routine.reference, retry=retry) @@ -669,7 +673,7 @@ def create_table( timeout=timeout, ) return Table.from_api_repr(api_response) - except google.api_core.exceptions.Conflict: + except core_exceptions.Conflict: if not exists_ok: raise return self.get_table(table.reference, retry=retry) @@ -1491,7 +1495,7 @@ def delete_dataset( query_params=params, timeout=timeout, ) - except google.api_core.exceptions.NotFound: + except core_exceptions.NotFound: if not not_found_ok: raise @@ -1543,7 +1547,7 @@ def delete_model( path=path, timeout=timeout, ) - except google.api_core.exceptions.NotFound: + except core_exceptions.NotFound: if not not_found_ok: raise @@ -1597,7 +1601,7 @@ def delete_routine( path=path, timeout=timeout, ) - except google.api_core.exceptions.NotFound: + except core_exceptions.NotFound: if not not_found_ok: raise @@ -1647,7 +1651,7 @@ def delete_table( path=path, timeout=timeout, ) - except google.api_core.exceptions.NotFound: + except core_exceptions.NotFound: if not not_found_ok: raise @@ -2031,10 +2035,14 @@ def list_jobs( "allUsers": all_users, "stateFilter": state_filter, "minCreationTime": _str_or_none( - google.cloud._helpers._millis_from_datetime(min_creation_time) + google.cloud._helpers._millis_from_datetime( # pytype: disable=module-attr + min_creation_time + ) ), "maxCreationTime": _str_or_none( - google.cloud._helpers._millis_from_datetime(max_creation_time) + google.cloud._helpers._millis_from_datetime( # pytype: disable=module-attr + max_creation_time + ) ), "projection": "full", "parentJobId": parent_job, @@ -2398,7 +2406,7 @@ def load_table_from_dataframe( ): try: table = self.get_table(destination) - except google.api_core.exceptions.NotFound: + except core_exceptions.NotFound: table = None else: columns_and_indexes = frozenset( @@ -3692,7 +3700,7 @@ def _check_mode(stream): mode = getattr(stream, "mode", None) if isinstance(stream, gzip.GzipFile): - if mode != gzip.READ: + if mode != gzip.READ: # pytype: disable=module-attr raise ValueError( "Cannot upload gzip files opened in write mode: use " "gzip.GzipFile(filename, mode='rb')" diff --git a/google/cloud/bigquery/dbapi/cursor.py b/google/cloud/bigquery/dbapi/cursor.py index 2a83fe830..ab20a3413 100644 --- a/google/cloud/bigquery/dbapi/cursor.py +++ b/google/cloud/bigquery/dbapi/cursor.py @@ -20,7 +20,9 @@ import logging try: - from google.cloud.bigquery_storage import ArrowSerializationOptions + from google.cloud.bigquery_storage import ( # pytype: disable=import-error + ArrowSerializationOptions, + ) except ImportError: _ARROW_COMPRESSION_SUPPORT = False else: diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index 6de8a7a3a..7d2d1be7d 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -18,8 +18,10 @@ import copy import re -from google.api_core import exceptions -from google.api_core.future import polling as polling_future +from google.api_core import exceptions # pytype: disable=import-error +from google.api_core.future import ( # pytype: disable=import-error + polling as polling_future, +) import requests from google.cloud.bigquery.dataset import Dataset diff --git a/google/cloud/bigquery/magics/magics.py b/google/cloud/bigquery/magics/magics.py index 6ae7cae12..70e4a25c5 100644 --- a/google/cloud/bigquery/magics/magics.py +++ b/google/cloud/bigquery/magics/magics.py @@ -156,15 +156,15 @@ from concurrent import futures try: - import IPython - from IPython import display - from IPython.core import magic_arguments + import IPython # pytype: disable=import-error + from IPython import display # pytype: disable=import-error + from IPython.core import magic_arguments # pytype: disable=import-error except ImportError: # pragma: NO COVER raise ImportError("This module can only be loaded in IPython.") -from google.api_core import client_info -from google.api_core import client_options -from google.api_core.exceptions import NotFound +from google.api_core import client_info # pytype: disable=import-error +from google.api_core import client_options # pytype: disable=import-error +from google.api_core.exceptions import NotFound # pytype: disable=import-error import google.auth from google.cloud import bigquery import google.cloud.bigquery.dataset @@ -219,7 +219,7 @@ def credentials(self): /en/latest/user-guide.html#obtaining-credentials """ if self._credentials is None: - self._credentials, _ = google.auth.default() + self._credentials, _ = google.auth.default() # pytype: disable=module-attr return self._credentials @credentials.setter @@ -244,7 +244,7 @@ def project(self): >>> magics.context.project = 'my-project' """ if self._project is None: - _, self._project = google.auth.default() + _, self._project = google.auth.default() # pytype: disable=module-attr return self._project @project.setter @@ -767,7 +767,7 @@ def _make_bqstorage_client(use_bqstorage_api, credentials, client_options): return None try: - from google.cloud import bigquery_storage + from google.cloud import bigquery_storage # pytype: disable=import-error except ImportError as err: customized_error = ImportError( "The default BigQuery Storage API client cannot be used, install " @@ -778,7 +778,9 @@ def _make_bqstorage_client(use_bqstorage_api, credentials, client_options): raise customized_error from err try: - from google.api_core.gapic_v1 import client_info as gapic_client_info + from google.api_core.gapic_v1 import ( # pytype: disable=import-error + client_info as gapic_client_info, + ) except ImportError as err: customized_error = ImportError( "Install the grpcio package to use the BigQuery Storage API." From c17993e178da98b5bf2e64d103a0991932f569c7 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 15 Apr 2021 11:30:59 +0200 Subject: [PATCH 08/22] Fix misc. pytype warnings and false postiives --- google/cloud/bigquery/client.py | 12 +++++++----- .../cloud/bigquery/magics/line_arg_parser/lexer.py | 2 +- google/cloud/bigquery/schema.py | 3 ++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 33fcedaf6..fd4630164 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -88,7 +88,7 @@ ExtractJobConfig, ) from google.api_core import retry as retries # type: ignore -from typing import Any, BinaryIO, Dict, Iterable, Sequence, Tuple, Union +from typing import Any, BinaryIO, Dict, Iterable, Sequence, Tuple, Union, Optional _DEFAULT_CHUNKSIZE = 1048576 # 1024 * 1024 B = 1 MB @@ -1969,7 +1969,7 @@ def cancel_job( def list_jobs( self, project: str = None, - parent_job: Union[_AsyncJob, str] = None, + parent_job: Optional[Union[_AsyncJob, str]] = None, max_results: int = None, page_token: str = None, all_users: bool = None, @@ -2029,7 +2029,7 @@ def list_jobs( Iterable of job instances. """ if isinstance(parent_job, job._AsyncJob): - parent_job = parent_job.job_id + parent_job = parent_job.job_id # pytype: disable=attribute-error extra_params = { "allUsers": all_users, @@ -2776,7 +2776,9 @@ def _do_multipart_upload( def copy_table( self, - sources: Union[Table, TableReference, str], + sources: Union[ + Table, TableReference, str, Sequence[Union[Table, TableReference, str]] + ], destination: Union[Table, TableReference, str], job_id: str = None, job_id_prefix: str = None, @@ -3066,7 +3068,7 @@ def query( def insert_rows( self, table: Union[Table, TableReference, str], - rows: Union[Sequence[Tuple], Sequence[Dict]], + rows: Union[Iterable[Tuple], Iterable[Dict]], selected_fields: Sequence[SchemaField] = None, **kwargs: dict, ) -> Sequence[dict]: diff --git a/google/cloud/bigquery/magics/line_arg_parser/lexer.py b/google/cloud/bigquery/magics/line_arg_parser/lexer.py index 5a6ee1a83..cd809c389 100644 --- a/google/cloud/bigquery/magics/line_arg_parser/lexer.py +++ b/google/cloud/bigquery/magics/line_arg_parser/lexer.py @@ -98,7 +98,7 @@ def _generate_next_value_(name, start, count, last_values): return name -TokenType = AutoStrEnum( +TokenType = AutoStrEnum( # pytype: disable=wrong-arg-types "TokenType", [ (name, enum.auto()) diff --git a/google/cloud/bigquery/schema.py b/google/cloud/bigquery/schema.py index 9635e830d..cb221d6de 100644 --- a/google/cloud/bigquery/schema.py +++ b/google/cloud/bigquery/schema.py @@ -194,7 +194,8 @@ def _key(self): return ( self.name, self.field_type.upper(), - self.mode.upper(), + # Mode is always str, if not given it defaults to a str value + self.mode.upper(), # pytype: disable=attribute-error self.description, self._fields, self._policy_tags, From b76029cd1091b78d42329d37142d143ca0a9b83c Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 15 Apr 2021 14:30:02 +0200 Subject: [PATCH 09/22] Make changes to generated files persistent --- setup.cfg | 8 ++++++-- synth.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 0e2e0437b..8eefc4435 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,6 +17,7 @@ # Generated by synthtool. DO NOT EDIT! [bdist_wheel] universal = 1 + [pytype] python_version = 3.8 inputs = @@ -25,5 +26,8 @@ exclude = tests/ google/cloud/bigquery_v2/ output = .pytype/ -# Workaround for https://github.com/google/pytype/issues/150 -disable = pyi-error \ No newline at end of file +disable = + # There's some issue with finding some pyi files, thus disabling. + # The issue https://github.com/google/pytype/issues/150 is closed, but the + # error still occurs for some reason. + pyi-error diff --git a/synth.py b/synth.py index 3c6440600..d99f368cc 100644 --- a/synth.py +++ b/synth.py @@ -13,6 +13,7 @@ # limitations under the License. """This script is used to synthesize generated parts of this library.""" +import textwrap import synthtool as s from synthtool import gcp @@ -120,4 +121,32 @@ '\g<0>\n "bigquery_v2/services.rst", # generated by the code generator', ) +# ---------------------------------------------------------------------------- +# pytype-related changes +# ---------------------------------------------------------------------------- + +# Add .pytype to .gitignore +s.replace(".gitignore", r"\.pytest_cache", "\g<0>\n.pytype") + +# Add pytype config to setup.cfg +s.replace( + "setup.cfg", + r"universal = 1", + textwrap.dedent(""" \g<0> + + [pytype] + python_version = 3.8 + inputs = + google/cloud/ + exclude = + tests/ + google/cloud/bigquery_v2/ + output = .pytype/ + disable = + # There's some issue with finding some pyi files, thus disabling. + # The issue https://github.com/google/pytype/issues/150 is closed, but the + # error still occurs for some reason. + pyi-error""") +) + s.shell.run(["nox", "-s", "blacken"], hide_output=False) From 558163d1607b9450490012a6277dbeea0f3056ef Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 15 Apr 2021 15:00:24 +0200 Subject: [PATCH 10/22] Make final cleanup of client.py --- google/cloud/bigquery/client.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index fd4630164..371399844 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -19,6 +19,7 @@ from collections import abc as collections_abc import copy +import datetime import functools import gzip import io @@ -27,6 +28,7 @@ import math import os import tempfile +from typing import Any, BinaryIO, Dict, Iterable, Optional, Sequence, Tuple, Union import uuid import warnings @@ -47,6 +49,7 @@ import google.api_core.exceptions as core_exceptions # pytype: disable=import-error from google.api_core.iam import Policy # pytype: disable=import-error from google.api_core import page_iterator # pytype: disable=import-error +from google.api_core import retry as retries # type: ignore import google.cloud._helpers from google.cloud import exceptions # pytype: disable=import-error from google.cloud.client import ClientWithProject # pytype: disable=import-error @@ -63,6 +66,13 @@ from google.cloud.bigquery.dataset import DatasetReference from google.cloud.bigquery.opentelemetry_tracing import create_span from google.cloud.bigquery import job +from google.cloud.bigquery.job import ( + LoadJobConfig, + QueryJob, + QueryJobConfig, + CopyJobConfig, + ExtractJobConfig, +) from google.cloud.bigquery.model import Model from google.cloud.bigquery.model import ModelReference from google.cloud.bigquery.model import _model_arg_to_model_ref @@ -78,18 +88,6 @@ from google.cloud.bigquery.table import TableReference from google.cloud.bigquery.table import RowIterator -# Types needed only for Type Hints -import datetime -from google.cloud.bigquery.job import ( - _AsyncJob, - LoadJobConfig, - QueryJobConfig, - CopyJobConfig, - ExtractJobConfig, -) -from google.api_core import retry as retries # type: ignore -from typing import Any, BinaryIO, Dict, Iterable, Sequence, Tuple, Union, Optional - _DEFAULT_CHUNKSIZE = 1048576 # 1024 * 1024 B = 1 MB _MAX_MULTIPART_SIZE = 5 * 1024 * 1024 @@ -803,7 +801,7 @@ def test_iam_permissions( permissions: Sequence[str], retry: retries.Retry = DEFAULT_RETRY, timeout: float = None, - ): + ) -> Dict[str, Any]: if not isinstance(table, (Table, TableReference)): raise TypeError("table must be a Table or TableReference") @@ -1969,7 +1967,7 @@ def cancel_job( def list_jobs( self, project: str = None, - parent_job: Optional[Union[_AsyncJob, str]] = None, + parent_job: Optional[Union[QueryJob, str]] = None, max_results: int = None, page_token: str = None, all_users: bool = None, @@ -3094,7 +3092,7 @@ def insert_rows( selected_fields (Sequence[google.cloud.bigquery.schema.SchemaField]): The fields to return. Required if ``table`` is a :class:`~google.cloud.bigquery.table.TableReference`. - kwargs (Dict): + kwargs (dict): Keyword arguments to :meth:`~google.cloud.bigquery.client.Client.insert_rows_json`. From b3a9054dd5d05ccfadcb07826c310b9af973b367 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 15 Apr 2021 15:21:03 +0200 Subject: [PATCH 11/22] Change import ignores to more specific errors --- google/cloud/bigquery/_helpers.py | 16 ++++++++++------ google/cloud/bigquery/_http.py | 2 +- google/cloud/bigquery/_pandas_helpers.py | 6 +++--- google/cloud/bigquery/_tqdm_helpers.py | 2 +- google/cloud/bigquery/client.py | 2 +- google/cloud/bigquery/dbapi/cursor.py | 6 +++--- google/cloud/bigquery/job/base.py | 14 ++++++++------ google/cloud/bigquery/job/query.py | 4 ++-- google/cloud/bigquery/model.py | 4 ++-- google/cloud/bigquery/opentelemetry_tracing.py | 12 ++++++++---- google/cloud/bigquery/retry.py | 4 ++-- google/cloud/bigquery/table.py | 10 +++++----- 12 files changed, 46 insertions(+), 36 deletions(-) diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py index 0afe29a83..f39078e84 100644 --- a/google/cloud/bigquery/_helpers.py +++ b/google/cloud/bigquery/_helpers.py @@ -19,12 +19,16 @@ import decimal import re -from google.cloud._helpers import UTC # type: ignore -from google.cloud._helpers import _date_from_iso8601_date # type: ignore -from google.cloud._helpers import _datetime_from_microseconds # type: ignore -from google.cloud._helpers import _RFC3339_MICROS # type: ignore -from google.cloud._helpers import _RFC3339_NO_FRACTION # type: ignore -from google.cloud._helpers import _to_bytes # type: ignore +from google.cloud._helpers import UTC # pytype: disable=import-error +from google.cloud._helpers import ( # pytype: disable=import-error + _date_from_iso8601_date, +) +from google.cloud._helpers import ( # pytype: disable=import-error + _datetime_from_microseconds, +) +from google.cloud._helpers import _RFC3339_MICROS # pytype: disable=import-error +from google.cloud._helpers import _RFC3339_NO_FRACTION # pytype: disable=import-error +from google.cloud._helpers import _to_bytes # pytype: disable=import-error _RFC3339_MICROS_NO_ZULU = "%Y-%m-%dT%H:%M:%S.%f" _TIMEONLY_WO_MICROS = "%H:%M:%S" diff --git a/google/cloud/bigquery/_http.py b/google/cloud/bigquery/_http.py index 2726fb8c9..81e7922e6 100644 --- a/google/cloud/bigquery/_http.py +++ b/google/cloud/bigquery/_http.py @@ -17,7 +17,7 @@ import os import pkg_resources -from google.cloud import _http # type: ignore +from google.cloud import _http # pytype: disable=import-error from google.cloud.bigquery import __version__ diff --git a/google/cloud/bigquery/_pandas_helpers.py b/google/cloud/bigquery/_pandas_helpers.py index d2fccbb00..81882e1f6 100644 --- a/google/cloud/bigquery/_pandas_helpers.py +++ b/google/cloud/bigquery/_pandas_helpers.py @@ -23,13 +23,13 @@ from packaging import version # pytype: disable=import-error try: - import pandas # type: ignore + import pandas # pytype: disable=import-error except ImportError: # pragma: NO COVER pandas = None try: - import pyarrow # type: ignore - import pyarrow.parquet # type: ignore + import pyarrow # pytype: disable=import-error + import pyarrow.parquet # pytype: disable=import-error except ImportError: # pragma: NO COVER pyarrow = None diff --git a/google/cloud/bigquery/_tqdm_helpers.py b/google/cloud/bigquery/_tqdm_helpers.py index 8717b5be4..b21ad9ed7 100644 --- a/google/cloud/bigquery/_tqdm_helpers.py +++ b/google/cloud/bigquery/_tqdm_helpers.py @@ -19,7 +19,7 @@ import warnings try: - import tqdm # type: ignore + import tqdm # pytype: disable=import-error except ImportError: # pragma: NO COVER tqdm = None diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 371399844..b3cee4e50 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -49,7 +49,7 @@ import google.api_core.exceptions as core_exceptions # pytype: disable=import-error from google.api_core.iam import Policy # pytype: disable=import-error from google.api_core import page_iterator # pytype: disable=import-error -from google.api_core import retry as retries # type: ignore +from google.api_core import retry as retries # pytype: disable=import-error import google.cloud._helpers from google.cloud import exceptions # pytype: disable=import-error from google.cloud.client import ClientWithProject # pytype: disable=import-error diff --git a/google/cloud/bigquery/dbapi/cursor.py b/google/cloud/bigquery/dbapi/cursor.py index ab20a3413..bcd408765 100644 --- a/google/cloud/bigquery/dbapi/cursor.py +++ b/google/cloud/bigquery/dbapi/cursor.py @@ -32,7 +32,7 @@ from google.cloud.bigquery import job from google.cloud.bigquery.dbapi import _helpers from google.cloud.bigquery.dbapi import exceptions -import google.cloud.exceptions # type: ignore +import google.cloud.exceptions # pytype: disable=import-error _LOGGER = logging.getLogger(__name__) @@ -195,7 +195,7 @@ def execute(self, operation, parameters=None, job_id=None, job_config=None): # Wait for the query to finish. try: self._query_job.result() - except google.cloud.exceptions.GoogleCloudError as exc: + except google.cloud.exceptions.GoogleCloudError as exc: # pytype: disable=module-attr raise exceptions.DatabaseError(exc) query_results = self._query_job._query_results @@ -257,7 +257,7 @@ def _bqstorage_fetch(self, bqstorage_client): """ # Hitting this code path with a BQ Storage client instance implies that # bigquery_storage can indeed be imported here without errors. - from google.cloud import bigquery_storage # type: ignore + from google.cloud import bigquery_storage # pytype: disable=import-error table_reference = self._query_job.destination diff --git a/google/cloud/bigquery/job/base.py b/google/cloud/bigquery/job/base.py index 90f8a1278..475a12853 100644 --- a/google/cloud/bigquery/job/base.py +++ b/google/cloud/bigquery/job/base.py @@ -19,14 +19,14 @@ import http import threading -from google.api_core import exceptions # type: ignore -import google.api_core.future.polling # type: ignore +from google.api_core import exceptions # pytype: disable=import-error +import google.api_core.future.polling # pytype: disable=import-error from google.cloud.bigquery import _helpers from google.cloud.bigquery.retry import DEFAULT_RETRY # Types needed only for Type Hints -from google.api_core import retry as retries # type: ignore +from google.api_core import retry as retries # pytype: disable=import-error _DONE_STATE = "DONE" @@ -131,7 +131,9 @@ def _from_api_repr(cls, resource): return job_ref -class _AsyncJob(google.api_core.future.polling.PollingFuture): +class _AsyncJob( + google.api_core.future.polling.PollingFuture # pytype: disable=module-attr +): """Base class for asynchronous jobs. Args: @@ -834,7 +836,7 @@ def _fill_from_default(self, default_job_config): + repr(default_job_config._job_type) ) - new_job_config = self.__class__() # type: ignore + new_job_config = self.__class__() # pytype: disable=import-error default_job_properties = copy.deepcopy(default_job_config._properties) for key in self._properties: @@ -858,7 +860,7 @@ def from_api_repr(cls, resource: dict) -> "_JobConfig": Returns: google.cloud.bigquery.job._JobConfig: Configuration parsed from ``resource``. """ - job_config = cls() # type: ignore + job_config = cls() # pytype: disable=import-error job_config._properties = resource return job_config diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index 7d2d1be7d..cac574972 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -49,8 +49,8 @@ from google.cloud.bigquery.job.base import _JobReference # Types needed only for Type Hints -from google.api_core import retry as retries # type: ignore -from google.cloud import bigquery_storage # type: ignore +from google.api_core import retry as retries # pytype: disable=import-error +from google.cloud import bigquery_storage # pytype: disable=import-error from google.cloud.bigquery.table import RowIterator from typing import Any, Dict, Union diff --git a/google/cloud/bigquery/model.py b/google/cloud/bigquery/model.py index cdb411e08..cb19bb5a2 100644 --- a/google/cloud/bigquery/model.py +++ b/google/cloud/bigquery/model.py @@ -20,8 +20,8 @@ from google.protobuf import json_format -import google.cloud._helpers # type: ignore -from google.api_core import datetime_helpers # type: ignore +import google.cloud._helpers # pytype: disable=import-error +from google.api_core import datetime_helpers # pytype: disable=import-error from google.cloud.bigquery import _helpers from google.cloud.bigquery_v2 import types from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration diff --git a/google/cloud/bigquery/opentelemetry_tracing.py b/google/cloud/bigquery/opentelemetry_tracing.py index e1e1d2157..d4c914b5c 100644 --- a/google/cloud/bigquery/opentelemetry_tracing.py +++ b/google/cloud/bigquery/opentelemetry_tracing.py @@ -14,13 +14,17 @@ import logging from contextlib import contextmanager -from google.api_core.exceptions import GoogleAPICallError # type: ignore +from google.api_core.exceptions import ( # pytype: disable=import-error + GoogleAPICallError, +) logger = logging.getLogger(__name__) try: - from opentelemetry import trace # type: ignore - from opentelemetry.instrumentation.utils import http_status_to_canonical_code # type: ignore - from opentelemetry.trace.status import Status # type: ignore + from opentelemetry import trace # pytype: disable=import-error + from opentelemetry.instrumentation.utils import ( # pytype: disable=import-error + http_status_to_canonical_code, + ) + from opentelemetry.trace.status import Status # pytype: disable=import-error HAS_OPENTELEMETRY = True _warned_telemetry = True diff --git a/google/cloud/bigquery/retry.py b/google/cloud/bigquery/retry.py index de7310db9..878f5492b 100644 --- a/google/cloud/bigquery/retry.py +++ b/google/cloud/bigquery/retry.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.api_core import exceptions # type: ignore -from google.api_core import retry # type: ignore +from google.api_core import exceptions # pytype: disable=import-error +from google.api_core import retry # pytype: disable=import-error import requests.exceptions diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py index 0230a941c..54ae1a78b 100644 --- a/google/cloud/bigquery/table.py +++ b/google/cloud/bigquery/table.py @@ -25,19 +25,19 @@ import warnings try: - import pandas # type: ignore + import pandas # pytype: disable=import-error except ImportError: # pragma: NO COVER pandas = None try: - import pyarrow # type: ignore + import pyarrow # pytype: disable=import-error except ImportError: # pragma: NO COVER pyarrow = None import google.api_core.exceptions -from google.api_core.page_iterator import HTTPIterator # type: ignore +from google.api_core.page_iterator import HTTPIterator # pytype: disable=import-error -import google.cloud._helpers # type: ignore +import google.cloud._helpers # pytype: disable=import-error from google.cloud.bigquery import _helpers from google.cloud.bigquery import _pandas_helpers from google.cloud.bigquery.schema import _build_schema_resource @@ -48,7 +48,7 @@ from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration # Types needed only for Type Hints -from google.cloud import bigquery_storage # type: ignore +from google.cloud import bigquery_storage # pytype: disable=import-error from typing import Any, Dict, Iterable, Tuple From 5529b34581057464c77c0a804e444a3902c5097f Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 15 Apr 2021 16:05:28 +0200 Subject: [PATCH 12/22] Silence false positive type warning in job config --- google/cloud/bigquery/job/base.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/google/cloud/bigquery/job/base.py b/google/cloud/bigquery/job/base.py index 475a12853..c3669bfa2 100644 --- a/google/cloud/bigquery/job/base.py +++ b/google/cloud/bigquery/job/base.py @@ -836,7 +836,10 @@ def _fill_from_default(self, default_job_config): + repr(default_job_config._job_type) ) - new_job_config = self.__class__() # pytype: disable=import-error + # cls is one of the job config subclasses that provides the job_type argument to + # this base class on instantiation, thus missing-parameter warning is a false + # positive here. + new_job_config = self.__class__() # pytype: disable=missing-parameter default_job_properties = copy.deepcopy(default_job_config._properties) for key in self._properties: @@ -860,7 +863,10 @@ def from_api_repr(cls, resource: dict) -> "_JobConfig": Returns: google.cloud.bigquery.job._JobConfig: Configuration parsed from ``resource``. """ - job_config = cls() # pytype: disable=import-error + # cls is one of the job config subclasses that provides the job_type argument to + # this base class on instantiation, thus missing-parameter warning is a false + # positive here. + job_config = cls() # pytype: disable=missing-parameter job_config._properties = resource return job_config From 77623663736bbb6f37c68e9c620956d43587931a Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Thu, 15 Apr 2021 16:32:31 +0200 Subject: [PATCH 13/22] Silence noisy _helper type warnings --- google/cloud/bigquery/model.py | 2 +- google/cloud/bigquery/table.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/google/cloud/bigquery/model.py b/google/cloud/bigquery/model.py index cb19bb5a2..5ec19ed99 100644 --- a/google/cloud/bigquery/model.py +++ b/google/cloud/bigquery/model.py @@ -20,7 +20,7 @@ from google.protobuf import json_format -import google.cloud._helpers # pytype: disable=import-error +import google.cloud._helpers # type: ignore from google.api_core import datetime_helpers # pytype: disable=import-error from google.cloud.bigquery import _helpers from google.cloud.bigquery_v2 import types diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py index 54ae1a78b..44b2016b6 100644 --- a/google/cloud/bigquery/table.py +++ b/google/cloud/bigquery/table.py @@ -37,7 +37,7 @@ import google.api_core.exceptions from google.api_core.page_iterator import HTTPIterator # pytype: disable=import-error -import google.cloud._helpers # pytype: disable=import-error +import google.cloud._helpers # type: ignore from google.cloud.bigquery import _helpers from google.cloud.bigquery import _pandas_helpers from google.cloud.bigquery.schema import _build_schema_resource From bf32b2c4e532c32d27e5daa913da069c8e7b0891 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Mon, 19 Apr 2021 10:47:58 +0200 Subject: [PATCH 14/22] Silence false positives for resumable media code --- google/cloud/bigquery/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index b3cee4e50..e1c05ef70 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -37,7 +37,7 @@ except ImportError: # pragma: NO COVER pyarrow = None -from google import resumable_media # pytype: disable=import-error +from google import resumable_media # type: ignore from google.resumable_media.requests import ( # pytype: disable=import-error MultipartUpload, ) From 5f672f9e38559c8ea3dd95e8e976ce9a1347ea69 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Mon, 19 Apr 2021 11:07:49 +0200 Subject: [PATCH 15/22] Add pytype to nox.options.sessions --- noxfile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/noxfile.py b/noxfile.py index 916f73549..f8d2f7ab2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -39,6 +39,7 @@ "lint", "lint_setup_py", "blacken", + "pytype", "docs", ] From 0afc6ca5e8868339adac16dcef741f2f03bb2e50 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 21 Apr 2021 12:30:34 +0200 Subject: [PATCH 16/22] Hide for-type-check-only imports behind a flag --- google/cloud/bigquery/job/base.py | 15 ++++++++------- google/cloud/bigquery/job/query.py | 21 +++++++++++---------- google/cloud/bigquery/table.py | 13 +++++++------ 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/google/cloud/bigquery/job/base.py b/google/cloud/bigquery/job/base.py index c3669bfa2..cdf5fec1e 100644 --- a/google/cloud/bigquery/job/base.py +++ b/google/cloud/bigquery/job/base.py @@ -18,6 +18,7 @@ import copy import http import threading +import typing from google.api_core import exceptions # pytype: disable=import-error import google.api_core.future.polling # pytype: disable=import-error @@ -25,8 +26,8 @@ from google.cloud.bigquery import _helpers from google.cloud.bigquery.retry import DEFAULT_RETRY -# Types needed only for Type Hints -from google.api_core import retry as retries # pytype: disable=import-error +if typing.TYPE_CHECKING: + from google.api_core import retry as retries # pytype: disable=import-error _DONE_STATE = "DONE" @@ -472,7 +473,7 @@ def _begin(self, client=None, retry=DEFAULT_RETRY, timeout=None): self._set_properties(api_response) def exists( - self, client=None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None + self, client=None, retry: "retries.Retry" = DEFAULT_RETRY, timeout: float = None ) -> bool: """API call: test for the existence of the job via a GET request @@ -517,7 +518,7 @@ def exists( return True def reload( - self, client=None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None + self, client=None, retry: "retries.Retry" = DEFAULT_RETRY, timeout: float = None ): """API call: refresh job properties via a GET request. @@ -554,7 +555,7 @@ def reload( self._set_properties(api_response) def cancel( - self, client=None, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None + self, client=None, retry: "retries.Retry" = DEFAULT_RETRY, timeout: float = None ) -> bool: """API call: cancel job via a POST request @@ -623,7 +624,7 @@ def _set_future_result(self): def done( self, - retry: retries.Retry = DEFAULT_RETRY, + retry: "retries.Retry" = DEFAULT_RETRY, timeout: float = None, reload: bool = True, ) -> bool: @@ -650,7 +651,7 @@ def done( return self.state == _DONE_STATE def result( - self, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None + self, retry: "retries.Retry" = DEFAULT_RETRY, timeout: float = None ) -> "_AsyncJob": """Start the job and wait for it to complete and get the result. diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index cac574972..c39c2ab7c 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -17,6 +17,8 @@ import concurrent.futures import copy import re +import typing +from typing import Any, Dict, Union from google.api_core import exceptions # pytype: disable=import-error from google.api_core.future import ( # pytype: disable=import-error @@ -48,11 +50,10 @@ from google.cloud.bigquery.job.base import _JobConfig from google.cloud.bigquery.job.base import _JobReference -# Types needed only for Type Hints -from google.api_core import retry as retries # pytype: disable=import-error -from google.cloud import bigquery_storage # pytype: disable=import-error -from google.cloud.bigquery.table import RowIterator -from typing import Any, Dict, Union +if typing.TYPE_CHECKING: + from google.api_core import retry as retries # pytype: disable=import-error + from google.cloud import bigquery_storage # pytype: disable=import-error + from google.cloud.bigquery.table import RowIterator _CONTAINS_ORDER_BY = re.compile(r"ORDER\s+BY", re.IGNORECASE) @@ -1045,7 +1046,7 @@ def _begin(self, client=None, retry=DEFAULT_RETRY, timeout=None): raise def _reload_query_results( - self, retry: retries.Retry = DEFAULT_RETRY, timeout: float = None + self, retry: "retries.Retry" = DEFAULT_RETRY, timeout: float = None ): """Refresh the cached query results. @@ -1123,10 +1124,10 @@ def result( self, page_size: int = None, max_results: int = None, - retry: retries.Retry = DEFAULT_RETRY, + retry: "retries.Retry" = DEFAULT_RETRY, timeout: float = None, start_index: int = None, - ) -> Union[RowIterator, _EmptyRowIterator]: + ) -> Union["RowIterator", _EmptyRowIterator]: """Start the job and wait for it to complete and get the result. Args: @@ -1207,7 +1208,7 @@ def result( def to_arrow( self, progress_bar_type: str = None, - bqstorage_client: bigquery_storage.BigQueryReadClient = None, + bqstorage_client: "bigquery_storage.BigQueryReadClient" = None, create_bqstorage_client: bool = True, ) -> Any: """[Beta] Create a class:`pyarrow.Table` by loading all pages of a @@ -1275,7 +1276,7 @@ def to_arrow( # changes to table.RowIterator.to_dataframe() def to_dataframe( self, - bqstorage_client: bigquery_storage.BigQueryReadClient = None, + bqstorage_client: "bigquery_storage.BigQueryReadClient" = None, dtypes: Dict[str, Any] = None, progress_bar_type: str = None, create_bqstorage_client: bool = True, diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py index 44b2016b6..f4b4e9ea7 100644 --- a/google/cloud/bigquery/table.py +++ b/google/cloud/bigquery/table.py @@ -22,6 +22,8 @@ import logging import operator import pytz +import typing +from typing import Any, Dict, Iterable, Tuple import warnings try: @@ -47,9 +49,8 @@ from google.cloud.bigquery.external_config import ExternalConfig from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration -# Types needed only for Type Hints -from google.cloud import bigquery_storage # pytype: disable=import-error -from typing import Any, Dict, Iterable, Tuple +if typing.TYPE_CHECKING: + from google.cloud import bigquery_storage # pytype: disable=import-error _LOGGER = logging.getLogger(__name__) @@ -1527,7 +1528,7 @@ def _to_arrow_iterable(self, bqstorage_client=None): def to_arrow( self, progress_bar_type: str = None, - bqstorage_client: bigquery_storage.BigQueryReadClient = None, + bqstorage_client: "bigquery_storage.BigQueryReadClient" = None, create_bqstorage_client: bool = True, ) -> Any: """[Beta] Create a class:`pyarrow.Table` by loading all pages of a @@ -1629,7 +1630,7 @@ def to_arrow( def to_dataframe_iterable( self, - bqstorage_client: bigquery_storage.BigQueryReadClient = None, + bqstorage_client: "bigquery_storage.BigQueryReadClient" = None, dtypes: Dict[str, Any] = None, max_queue_size: int = _pandas_helpers._MAX_QUEUE_SIZE_DEFAULT, ) -> Any: @@ -1704,7 +1705,7 @@ def to_dataframe_iterable( # changes to job.QueryJob.to_dataframe() def to_dataframe( self, - bqstorage_client: bigquery_storage.BigQueryReadClient = None, + bqstorage_client: "bigquery_storage.BigQueryReadClient" = None, dtypes: Dict[str, Any] = None, progress_bar_type: str = None, create_bqstorage_client: bool = True, From 3bdc7d872a122043d07299c5b7b62e10865069d0 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 21 Apr 2021 12:31:19 +0200 Subject: [PATCH 17/22] Remove obsolete skipIf decorator from two tests inspect.signature() was added in Python 3.3, and the library only needs to suppport Python3.6+. --- tests/unit/test_signature_compatibility.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/unit/test_signature_compatibility.py b/tests/unit/test_signature_compatibility.py index 6002ae3e8..e5016b0e5 100644 --- a/tests/unit/test_signature_compatibility.py +++ b/tests/unit/test_signature_compatibility.py @@ -31,20 +31,12 @@ def row_iterator_class(): return RowIterator -@pytest.mark.skipif( - not hasattr(inspect, "signature"), - reason="inspect.signature() is not availalbe in older Python versions", -) def test_to_arrow_method_signatures_match(query_job_class, row_iterator_class): sig = inspect.signature(query_job_class.to_arrow) sig2 = inspect.signature(row_iterator_class.to_arrow) assert sig == sig2 -@pytest.mark.skipif( - not hasattr(inspect, "signature"), - reason="inspect.signature() is not availalbe in older Python versions", -) def test_to_dataframe_method_signatures_match(query_job_class, row_iterator_class): sig = inspect.signature(query_job_class.to_dataframe) sig2 = inspect.signature(row_iterator_class.to_dataframe) From 8466b33dd6e37b5379d1b1ae3da834206e3b787c Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 21 Apr 2021 14:04:39 +0200 Subject: [PATCH 18/22] Install dependencies in pytype session This avoids numerous unnecessary import and module attribute errors, rendering lots of pytype directive comments obsolete. --- google/cloud/bigquery/_helpers.py | 16 ++++------ google/cloud/bigquery/_pandas_helpers.py | 14 ++++---- google/cloud/bigquery/_tqdm_helpers.py | 2 +- google/cloud/bigquery/client.py | 32 +++++++------------ google/cloud/bigquery/dataset.py | 2 +- google/cloud/bigquery/dbapi/cursor.py | 10 +++--- google/cloud/bigquery/job/base.py | 10 +++--- google/cloud/bigquery/job/query.py | 15 +++++---- google/cloud/bigquery/magics/magics.py | 22 ++++++------- google/cloud/bigquery/model.py | 4 +-- .../cloud/bigquery/opentelemetry_tracing.py | 12 +++---- google/cloud/bigquery/retry.py | 4 +-- google/cloud/bigquery/routine/routine.py | 4 --- google/cloud/bigquery/table.py | 10 +++--- noxfile.py | 5 +-- 15 files changed, 68 insertions(+), 94 deletions(-) diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py index f39078e84..daa14b92a 100644 --- a/google/cloud/bigquery/_helpers.py +++ b/google/cloud/bigquery/_helpers.py @@ -19,16 +19,12 @@ import decimal import re -from google.cloud._helpers import UTC # pytype: disable=import-error -from google.cloud._helpers import ( # pytype: disable=import-error - _date_from_iso8601_date, -) -from google.cloud._helpers import ( # pytype: disable=import-error - _datetime_from_microseconds, -) -from google.cloud._helpers import _RFC3339_MICROS # pytype: disable=import-error -from google.cloud._helpers import _RFC3339_NO_FRACTION # pytype: disable=import-error -from google.cloud._helpers import _to_bytes # pytype: disable=import-error +from google.cloud._helpers import UTC +from google.cloud._helpers import _date_from_iso8601_date +from google.cloud._helpers import _datetime_from_microseconds +from google.cloud._helpers import _RFC3339_MICROS +from google.cloud._helpers import _RFC3339_NO_FRACTION +from google.cloud._helpers import _to_bytes _RFC3339_MICROS_NO_ZULU = "%Y-%m-%dT%H:%M:%S.%f" _TIMEONLY_WO_MICROS = "%H:%M:%S" diff --git a/google/cloud/bigquery/_pandas_helpers.py b/google/cloud/bigquery/_pandas_helpers.py index 81882e1f6..e93a99eba 100644 --- a/google/cloud/bigquery/_pandas_helpers.py +++ b/google/cloud/bigquery/_pandas_helpers.py @@ -20,23 +20,21 @@ import queue import warnings -from packaging import version # pytype: disable=import-error +from packaging import version try: - import pandas # pytype: disable=import-error + import pandas except ImportError: # pragma: NO COVER pandas = None try: - import pyarrow # pytype: disable=import-error - import pyarrow.parquet # pytype: disable=import-error + import pyarrow + import pyarrow.parquet except ImportError: # pragma: NO COVER pyarrow = None try: - from google.cloud.bigquery_storage import ( # pytype: disable=import-error - ArrowSerializationOptions, - ) + from google.cloud.bigquery_storage import ArrowSerializationOptions except ImportError: _ARROW_COMPRESSION_SUPPORT = False else: @@ -628,7 +626,7 @@ def _download_table_bqstorage( # Passing a BQ Storage client in implies that the BigQuery Storage library # is available and can be imported. - from google.cloud import bigquery_storage # type: ignore + from google.cloud import bigquery_storage if "$" in table.table_id: raise ValueError( diff --git a/google/cloud/bigquery/_tqdm_helpers.py b/google/cloud/bigquery/_tqdm_helpers.py index b21ad9ed7..2fcf2a981 100644 --- a/google/cloud/bigquery/_tqdm_helpers.py +++ b/google/cloud/bigquery/_tqdm_helpers.py @@ -19,7 +19,7 @@ import warnings try: - import tqdm # pytype: disable=import-error + import tqdm except ImportError: # pragma: NO COVER tqdm = None diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index e1c05ef70..ec436819f 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -33,23 +33,19 @@ import warnings try: - import pyarrow # pytype: disable=import-error + import pyarrow except ImportError: # pragma: NO COVER pyarrow = None from google import resumable_media # type: ignore -from google.resumable_media.requests import ( # pytype: disable=import-error - MultipartUpload, -) -from google.resumable_media.requests import ( # pytype: disable=import-error - ResumableUpload, -) +from google.resumable_media.requests import MultipartUpload +from google.resumable_media.requests import ResumableUpload import google.api_core.client_options -import google.api_core.exceptions as core_exceptions # pytype: disable=import-error -from google.api_core.iam import Policy # pytype: disable=import-error -from google.api_core import page_iterator # pytype: disable=import-error -from google.api_core import retry as retries # pytype: disable=import-error +import google.api_core.exceptions as core_exceptions +from google.api_core.iam import Policy +from google.api_core import page_iterator +from google.api_core import retry as retries import google.cloud._helpers from google.cloud import exceptions # pytype: disable=import-error from google.cloud.client import ClientWithProject # pytype: disable=import-error @@ -201,7 +197,7 @@ def __init__( kw_args = {"client_info": client_info} if client_options: if type(client_options) == dict: - client_options = google.api_core.client_options.from_dict( # pytype: disable=module-attr + client_options = google.api_core.client_options.from_dict( client_options ) if client_options.api_endpoint: @@ -460,7 +456,7 @@ def _create_bqstorage_client(self): A BigQuery Storage API client. """ try: - from google.cloud import bigquery_storage # pytype: disable=import-error + from google.cloud import bigquery_storage except ImportError: warnings.warn( "Cannot create BigQuery Storage client, the dependency " @@ -611,7 +607,7 @@ def create_routine( timeout=timeout, ) return Routine.from_api_repr(api_response) - except core_exceptions.Conflict: # pytype: disable=module-attr + except core_exceptions.Conflict: if not exists_ok: raise return self.get_routine(routine.reference, retry=retry) @@ -2033,14 +2029,10 @@ def list_jobs( "allUsers": all_users, "stateFilter": state_filter, "minCreationTime": _str_or_none( - google.cloud._helpers._millis_from_datetime( # pytype: disable=module-attr - min_creation_time - ) + google.cloud._helpers._millis_from_datetime(min_creation_time) ), "maxCreationTime": _str_or_none( - google.cloud._helpers._millis_from_datetime( # pytype: disable=module-attr - max_creation_time - ) + google.cloud._helpers._millis_from_datetime(max_creation_time) ), "projection": "full", "parentJobId": parent_job, diff --git a/google/cloud/bigquery/dataset.py b/google/cloud/bigquery/dataset.py index ff015d605..21e56f305 100644 --- a/google/cloud/bigquery/dataset.py +++ b/google/cloud/bigquery/dataset.py @@ -18,7 +18,7 @@ import copy -import google.cloud._helpers # type: ignore +import google.cloud._helpers from google.cloud.bigquery import _helpers from google.cloud.bigquery.model import ModelReference diff --git a/google/cloud/bigquery/dbapi/cursor.py b/google/cloud/bigquery/dbapi/cursor.py index bcd408765..ee09158d8 100644 --- a/google/cloud/bigquery/dbapi/cursor.py +++ b/google/cloud/bigquery/dbapi/cursor.py @@ -20,9 +20,7 @@ import logging try: - from google.cloud.bigquery_storage import ( # pytype: disable=import-error - ArrowSerializationOptions, - ) + from google.cloud.bigquery_storage import ArrowSerializationOptions except ImportError: _ARROW_COMPRESSION_SUPPORT = False else: @@ -32,7 +30,7 @@ from google.cloud.bigquery import job from google.cloud.bigquery.dbapi import _helpers from google.cloud.bigquery.dbapi import exceptions -import google.cloud.exceptions # pytype: disable=import-error +import google.cloud.exceptions _LOGGER = logging.getLogger(__name__) @@ -195,7 +193,7 @@ def execute(self, operation, parameters=None, job_id=None, job_config=None): # Wait for the query to finish. try: self._query_job.result() - except google.cloud.exceptions.GoogleCloudError as exc: # pytype: disable=module-attr + except google.cloud.exceptions.GoogleCloudError as exc: raise exceptions.DatabaseError(exc) query_results = self._query_job._query_results @@ -257,7 +255,7 @@ def _bqstorage_fetch(self, bqstorage_client): """ # Hitting this code path with a BQ Storage client instance implies that # bigquery_storage can indeed be imported here without errors. - from google.cloud import bigquery_storage # pytype: disable=import-error + from google.cloud import bigquery_storage table_reference = self._query_job.destination diff --git a/google/cloud/bigquery/job/base.py b/google/cloud/bigquery/job/base.py index cdf5fec1e..50d307b27 100644 --- a/google/cloud/bigquery/job/base.py +++ b/google/cloud/bigquery/job/base.py @@ -20,14 +20,14 @@ import threading import typing -from google.api_core import exceptions # pytype: disable=import-error -import google.api_core.future.polling # pytype: disable=import-error +from google.api_core import exceptions +import google.api_core.future.polling from google.cloud.bigquery import _helpers from google.cloud.bigquery.retry import DEFAULT_RETRY if typing.TYPE_CHECKING: - from google.api_core import retry as retries # pytype: disable=import-error + from google.api_core import retry as retries _DONE_STATE = "DONE" @@ -132,9 +132,7 @@ def _from_api_repr(cls, resource): return job_ref -class _AsyncJob( - google.api_core.future.polling.PollingFuture # pytype: disable=module-attr -): +class _AsyncJob(google.api_core.future.polling.PollingFuture): """Base class for asynchronous jobs. Args: diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index c39c2ab7c..17590dad3 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -20,10 +20,8 @@ import typing from typing import Any, Dict, Union -from google.api_core import exceptions # pytype: disable=import-error -from google.api_core.future import ( # pytype: disable=import-error - polling as polling_future, -) +from google.api_core import exceptions +from google.api_core.future import polling as polling_future import requests from google.cloud.bigquery.dataset import Dataset @@ -51,8 +49,11 @@ from google.cloud.bigquery.job.base import _JobReference if typing.TYPE_CHECKING: - from google.api_core import retry as retries # pytype: disable=import-error - from google.cloud import bigquery_storage # pytype: disable=import-error + # Assumption: type checks are only used by library developers and CI environments + # that have all optional dependencies installed, thus no conditional pyarrow import. + import pyarrow + from google.api_core import retry as retries + from google.cloud import bigquery_storage from google.cloud.bigquery.table import RowIterator @@ -1210,7 +1211,7 @@ def to_arrow( progress_bar_type: str = None, bqstorage_client: "bigquery_storage.BigQueryReadClient" = None, create_bqstorage_client: bool = True, - ) -> Any: + ) -> "pyarrow.Table": """[Beta] Create a class:`pyarrow.Table` by loading all pages of a table or query. diff --git a/google/cloud/bigquery/magics/magics.py b/google/cloud/bigquery/magics/magics.py index 70e4a25c5..6ae7cae12 100644 --- a/google/cloud/bigquery/magics/magics.py +++ b/google/cloud/bigquery/magics/magics.py @@ -156,15 +156,15 @@ from concurrent import futures try: - import IPython # pytype: disable=import-error - from IPython import display # pytype: disable=import-error - from IPython.core import magic_arguments # pytype: disable=import-error + import IPython + from IPython import display + from IPython.core import magic_arguments except ImportError: # pragma: NO COVER raise ImportError("This module can only be loaded in IPython.") -from google.api_core import client_info # pytype: disable=import-error -from google.api_core import client_options # pytype: disable=import-error -from google.api_core.exceptions import NotFound # pytype: disable=import-error +from google.api_core import client_info +from google.api_core import client_options +from google.api_core.exceptions import NotFound import google.auth from google.cloud import bigquery import google.cloud.bigquery.dataset @@ -219,7 +219,7 @@ def credentials(self): /en/latest/user-guide.html#obtaining-credentials """ if self._credentials is None: - self._credentials, _ = google.auth.default() # pytype: disable=module-attr + self._credentials, _ = google.auth.default() return self._credentials @credentials.setter @@ -244,7 +244,7 @@ def project(self): >>> magics.context.project = 'my-project' """ if self._project is None: - _, self._project = google.auth.default() # pytype: disable=module-attr + _, self._project = google.auth.default() return self._project @project.setter @@ -767,7 +767,7 @@ def _make_bqstorage_client(use_bqstorage_api, credentials, client_options): return None try: - from google.cloud import bigquery_storage # pytype: disable=import-error + from google.cloud import bigquery_storage except ImportError as err: customized_error = ImportError( "The default BigQuery Storage API client cannot be used, install " @@ -778,9 +778,7 @@ def _make_bqstorage_client(use_bqstorage_api, credentials, client_options): raise customized_error from err try: - from google.api_core.gapic_v1 import ( # pytype: disable=import-error - client_info as gapic_client_info, - ) + from google.api_core.gapic_v1 import client_info as gapic_client_info except ImportError as err: customized_error = ImportError( "Install the grpcio package to use the BigQuery Storage API." diff --git a/google/cloud/bigquery/model.py b/google/cloud/bigquery/model.py index 5ec19ed99..2d3f6660f 100644 --- a/google/cloud/bigquery/model.py +++ b/google/cloud/bigquery/model.py @@ -20,8 +20,8 @@ from google.protobuf import json_format -import google.cloud._helpers # type: ignore -from google.api_core import datetime_helpers # pytype: disable=import-error +import google.cloud._helpers +from google.api_core import datetime_helpers from google.cloud.bigquery import _helpers from google.cloud.bigquery_v2 import types from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration diff --git a/google/cloud/bigquery/opentelemetry_tracing.py b/google/cloud/bigquery/opentelemetry_tracing.py index d4c914b5c..57f258ac4 100644 --- a/google/cloud/bigquery/opentelemetry_tracing.py +++ b/google/cloud/bigquery/opentelemetry_tracing.py @@ -14,17 +14,13 @@ import logging from contextlib import contextmanager -from google.api_core.exceptions import ( # pytype: disable=import-error - GoogleAPICallError, -) +from google.api_core.exceptions import GoogleAPICallError logger = logging.getLogger(__name__) try: - from opentelemetry import trace # pytype: disable=import-error - from opentelemetry.instrumentation.utils import ( # pytype: disable=import-error - http_status_to_canonical_code, - ) - from opentelemetry.trace.status import Status # pytype: disable=import-error + from opentelemetry import trace + from opentelemetry.instrumentation.utils import http_status_to_canonical_code + from opentelemetry.trace.status import Status HAS_OPENTELEMETRY = True _warned_telemetry = True diff --git a/google/cloud/bigquery/retry.py b/google/cloud/bigquery/retry.py index 878f5492b..20a8e7b13 100644 --- a/google/cloud/bigquery/retry.py +++ b/google/cloud/bigquery/retry.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from google.api_core import exceptions # pytype: disable=import-error -from google.api_core import retry # pytype: disable=import-error +from google.api_core import exceptions +from google.api_core import retry import requests.exceptions diff --git a/google/cloud/bigquery/routine/routine.py b/google/cloud/bigquery/routine/routine.py index 670fdf580..bbc0a7693 100644 --- a/google/cloud/bigquery/routine/routine.py +++ b/google/cloud/bigquery/routine/routine.py @@ -119,14 +119,12 @@ def created(self): Read-only. """ - # pytype: disable=module-attr value = self._properties.get(self._PROPERTY_TO_API_FIELD["created"]) if value is not None and value != 0: # value will be in milliseconds. return google.cloud._helpers._datetime_from_microseconds( 1000.0 * float(value) ) - # pytype: enable=module-attr @property def modified(self): @@ -135,14 +133,12 @@ def modified(self): Read-only. """ - # pytype: disable=module-attr value = self._properties.get(self._PROPERTY_TO_API_FIELD["modified"]) if value is not None and value != 0: # value will be in milliseconds. return google.cloud._helpers._datetime_from_microseconds( 1000.0 * float(value) ) - # pytype: enable=module-attr @property def language(self): diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py index f4b4e9ea7..259ca1226 100644 --- a/google/cloud/bigquery/table.py +++ b/google/cloud/bigquery/table.py @@ -27,19 +27,19 @@ import warnings try: - import pandas # pytype: disable=import-error + import pandas except ImportError: # pragma: NO COVER pandas = None try: - import pyarrow # pytype: disable=import-error + import pyarrow except ImportError: # pragma: NO COVER pyarrow = None import google.api_core.exceptions -from google.api_core.page_iterator import HTTPIterator # pytype: disable=import-error +from google.api_core.page_iterator import HTTPIterator -import google.cloud._helpers # type: ignore +import google.cloud._helpers from google.cloud.bigquery import _helpers from google.cloud.bigquery import _pandas_helpers from google.cloud.bigquery.schema import _build_schema_resource @@ -50,7 +50,7 @@ from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration if typing.TYPE_CHECKING: - from google.cloud import bigquery_storage # pytype: disable=import-error + from google.cloud import bigquery_storage _LOGGER = logging.getLogger(__name__) diff --git a/noxfile.py b/noxfile.py index f8d2f7ab2..5d4bc292d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -94,8 +94,9 @@ def unit(session): @nox.session(python="3.8") def pytype(session): - """Run pytype - """ + """Run type checks.""" + session.install("-e", ".[all]") + session.install("ipython") session.install(PYTYPE_VERSION) session.run("pytype") From dfd431739fa327071a48e5c786e4a377924f5274 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 21 Apr 2021 14:27:17 +0200 Subject: [PATCH 19/22] Be more specific about to_dataframe()'s return type --- google/cloud/bigquery/job/query.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index 17590dad3..466a5759f 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -50,7 +50,8 @@ if typing.TYPE_CHECKING: # Assumption: type checks are only used by library developers and CI environments - # that have all optional dependencies installed, thus no conditional pyarrow import. + # that have all optional dependencies installed, thus no conditional imports. + import pandas import pyarrow from google.api_core import retry as retries from google.cloud import bigquery_storage @@ -1282,7 +1283,7 @@ def to_dataframe( progress_bar_type: str = None, create_bqstorage_client: bool = True, date_as_object: bool = True, - ) -> Any: + ) -> "pandas.DataFrame": """Return a pandas DataFrame from a QueryJob Args: From 3a449d2aa495a89b086ab0a15638804705bc6146 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 21 Apr 2021 14:31:44 +0200 Subject: [PATCH 20/22] Add missing return type for _get_query_results() --- google/cloud/bigquery/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index ec436819f..45ab342d0 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -1657,7 +1657,7 @@ def _get_query_results( timeout_ms: int = None, location: str = None, timeout: float = None, - ): + ) -> _QueryResults: """Get the query results object for a query job. Args: From 80820f7ce742c14ef935cd63948fa63f7cc3d71d Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 21 Apr 2021 15:12:32 +0200 Subject: [PATCH 21/22] Be more specific about pandas/pyarrow return types --- google/cloud/bigquery/table.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py index 259ca1226..b678d867e 100644 --- a/google/cloud/bigquery/table.py +++ b/google/cloud/bigquery/table.py @@ -50,6 +50,10 @@ from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration if typing.TYPE_CHECKING: + # Unconditionally import optional dependencies again to tell pytype that + # they are not None. + import pandas + import pyarrow from google.cloud import bigquery_storage @@ -1530,7 +1534,7 @@ def to_arrow( progress_bar_type: str = None, bqstorage_client: "bigquery_storage.BigQueryReadClient" = None, create_bqstorage_client: bool = True, - ) -> Any: + ) -> "pyarrow.Table": """[Beta] Create a class:`pyarrow.Table` by loading all pages of a table or query. @@ -1633,7 +1637,7 @@ def to_dataframe_iterable( bqstorage_client: "bigquery_storage.BigQueryReadClient" = None, dtypes: Dict[str, Any] = None, max_queue_size: int = _pandas_helpers._MAX_QUEUE_SIZE_DEFAULT, - ) -> Any: + ) -> "pandas.DataFrame": """Create an iterable of pandas DataFrames, to process the table as a stream. Args: @@ -1710,7 +1714,7 @@ def to_dataframe( progress_bar_type: str = None, create_bqstorage_client: bool = True, date_as_object: bool = True, - ) -> Any: + ) -> "pandas.DataFrame": """Create a pandas DataFrame by loading all pages of a query. Args: @@ -1838,7 +1842,7 @@ def to_arrow( progress_bar_type=None, bqstorage_client=None, create_bqstorage_client=True, - ) -> Any: + ) -> "pyarrow.Table": """[Beta] Create an empty class:`pyarrow.Table`. Args: @@ -1860,7 +1864,7 @@ def to_dataframe( progress_bar_type=None, create_bqstorage_client=True, date_as_object=True, - ) -> Any: + ) -> "pandas.DataFrame": """Create an empty dataframe. Args: From 055d9f5d3ef887edcc9d93536abbb11564c0bf84 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 21 Apr 2021 15:33:21 +0200 Subject: [PATCH 22/22] Exclude typing-only imports from coverage checks --- google/cloud/bigquery/job/base.py | 2 +- google/cloud/bigquery/job/query.py | 2 +- google/cloud/bigquery/table.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/google/cloud/bigquery/job/base.py b/google/cloud/bigquery/job/base.py index 50d307b27..20ad81c0b 100644 --- a/google/cloud/bigquery/job/base.py +++ b/google/cloud/bigquery/job/base.py @@ -26,7 +26,7 @@ from google.cloud.bigquery import _helpers from google.cloud.bigquery.retry import DEFAULT_RETRY -if typing.TYPE_CHECKING: +if typing.TYPE_CHECKING: # pragma: NO COVER from google.api_core import retry as retries diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index 466a5759f..f52f9c621 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -48,7 +48,7 @@ from google.cloud.bigquery.job.base import _JobConfig from google.cloud.bigquery.job.base import _JobReference -if typing.TYPE_CHECKING: +if typing.TYPE_CHECKING: # pragma: NO COVER # Assumption: type checks are only used by library developers and CI environments # that have all optional dependencies installed, thus no conditional imports. import pandas diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py index b678d867e..5ab649a25 100644 --- a/google/cloud/bigquery/table.py +++ b/google/cloud/bigquery/table.py @@ -49,9 +49,9 @@ from google.cloud.bigquery.external_config import ExternalConfig from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration -if typing.TYPE_CHECKING: +if typing.TYPE_CHECKING: # pragma: NO COVER # Unconditionally import optional dependencies again to tell pytype that - # they are not None. + # they are not None, avoiding false "no attribute" errors. import pandas import pyarrow from google.cloud import bigquery_storage