diff --git a/docs/bigquery_storage_v1/big_query_write.rst b/docs/bigquery_storage_v1/big_query_write.rst new file mode 100644 index 00000000..e0c869ca --- /dev/null +++ b/docs/bigquery_storage_v1/big_query_write.rst @@ -0,0 +1,6 @@ +BigQueryWrite +------------------------------- + +.. automodule:: google.cloud.bigquery_storage_v1.services.big_query_write + :members: + :inherited-members: diff --git a/docs/bigquery_storage_v1/services.rst b/docs/bigquery_storage_v1/services.rst index 3f6cff3d..50f35459 100644 --- a/docs/bigquery_storage_v1/services.rst +++ b/docs/bigquery_storage_v1/services.rst @@ -4,3 +4,4 @@ Services for Google Cloud Bigquery Storage v1 API :maxdepth: 2 big_query_read + big_query_write diff --git a/google/cloud/bigquery_storage/__init__.py b/google/cloud/bigquery_storage/__init__.py index c11d4870..af605f0a 100644 --- a/google/cloud/bigquery_storage/__init__.py +++ b/google/cloud/bigquery_storage/__init__.py @@ -15,6 +15,12 @@ # from google.cloud.bigquery_storage_v1 import BigQueryReadClient +from google.cloud.bigquery_storage_v1.services.big_query_write.client import ( + BigQueryWriteClient, +) +from google.cloud.bigquery_storage_v1.services.big_query_write.async_client import ( + BigQueryWriteAsyncClient, +) from google.cloud.bigquery_storage_v1 import gapic_types as types from google.cloud.bigquery_storage_v1 import __version__ @@ -23,19 +29,41 @@ from google.cloud.bigquery_storage_v1.types.arrow import ArrowSerializationOptions from google.cloud.bigquery_storage_v1.types.avro import AvroRows from google.cloud.bigquery_storage_v1.types.avro import AvroSchema +from google.cloud.bigquery_storage_v1.types.protobuf import ProtoRows +from google.cloud.bigquery_storage_v1.types.protobuf import ProtoSchema +from google.cloud.bigquery_storage_v1.types.storage import AppendRowsRequest +from google.cloud.bigquery_storage_v1.types.storage import AppendRowsResponse +from google.cloud.bigquery_storage_v1.types.storage import ( + BatchCommitWriteStreamsRequest, +) +from google.cloud.bigquery_storage_v1.types.storage import ( + BatchCommitWriteStreamsResponse, +) from google.cloud.bigquery_storage_v1.types.storage import CreateReadSessionRequest +from google.cloud.bigquery_storage_v1.types.storage import CreateWriteStreamRequest +from google.cloud.bigquery_storage_v1.types.storage import FinalizeWriteStreamRequest +from google.cloud.bigquery_storage_v1.types.storage import FinalizeWriteStreamResponse +from google.cloud.bigquery_storage_v1.types.storage import FlushRowsRequest +from google.cloud.bigquery_storage_v1.types.storage import FlushRowsResponse +from google.cloud.bigquery_storage_v1.types.storage import GetWriteStreamRequest from google.cloud.bigquery_storage_v1.types.storage import ReadRowsRequest from google.cloud.bigquery_storage_v1.types.storage import ReadRowsResponse from google.cloud.bigquery_storage_v1.types.storage import SplitReadStreamRequest from google.cloud.bigquery_storage_v1.types.storage import SplitReadStreamResponse +from google.cloud.bigquery_storage_v1.types.storage import StorageError from google.cloud.bigquery_storage_v1.types.storage import StreamStats from google.cloud.bigquery_storage_v1.types.storage import ThrottleState from google.cloud.bigquery_storage_v1.types.stream import ReadSession from google.cloud.bigquery_storage_v1.types.stream import ReadStream +from google.cloud.bigquery_storage_v1.types.stream import WriteStream from google.cloud.bigquery_storage_v1.types.stream import DataFormat +from google.cloud.bigquery_storage_v1.types.table import TableFieldSchema +from google.cloud.bigquery_storage_v1.types.table import TableSchema __all__ = ( "BigQueryReadClient", + "BigQueryWriteClient", + "BigQueryWriteAsyncClient", "__version__", "types", "ArrowRecordBatch", @@ -43,14 +71,30 @@ "ArrowSerializationOptions", "AvroRows", "AvroSchema", + "ProtoRows", + "ProtoSchema", + "AppendRowsRequest", + "AppendRowsResponse", + "BatchCommitWriteStreamsRequest", + "BatchCommitWriteStreamsResponse", "CreateReadSessionRequest", + "CreateWriteStreamRequest", + "FinalizeWriteStreamRequest", + "FinalizeWriteStreamResponse", + "FlushRowsRequest", + "FlushRowsResponse", + "GetWriteStreamRequest", "ReadRowsRequest", "ReadRowsResponse", "SplitReadStreamRequest", "SplitReadStreamResponse", + "StorageError", "StreamStats", "ThrottleState", "ReadSession", "ReadStream", + "WriteStream", "DataFormat", + "TableFieldSchema", + "TableSchema", ) diff --git a/google/cloud/bigquery_storage_v1/gapic_metadata.json b/google/cloud/bigquery_storage_v1/gapic_metadata.json index 3311e568..4fc72131 100644 --- a/google/cloud/bigquery_storage_v1/gapic_metadata.json +++ b/google/cloud/bigquery_storage_v1/gapic_metadata.json @@ -48,6 +48,80 @@ } } } + }, + "BigQueryWrite": { + "clients": { + "grpc": { + "libraryClient": "BigQueryWriteClient", + "rpcs": { + "AppendRows": { + "methods": [ + "append_rows" + ] + }, + "BatchCommitWriteStreams": { + "methods": [ + "batch_commit_write_streams" + ] + }, + "CreateWriteStream": { + "methods": [ + "create_write_stream" + ] + }, + "FinalizeWriteStream": { + "methods": [ + "finalize_write_stream" + ] + }, + "FlushRows": { + "methods": [ + "flush_rows" + ] + }, + "GetWriteStream": { + "methods": [ + "get_write_stream" + ] + } + } + }, + "grpc-async": { + "libraryClient": "BigQueryWriteAsyncClient", + "rpcs": { + "AppendRows": { + "methods": [ + "append_rows" + ] + }, + "BatchCommitWriteStreams": { + "methods": [ + "batch_commit_write_streams" + ] + }, + "CreateWriteStream": { + "methods": [ + "create_write_stream" + ] + }, + "FinalizeWriteStream": { + "methods": [ + "finalize_write_stream" + ] + }, + "FlushRows": { + "methods": [ + "flush_rows" + ] + }, + "GetWriteStream": { + "methods": [ + "get_write_stream" + ] + } + } + } + } } } } diff --git a/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py b/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py index 4b201f2e..a7d74d46 100644 --- a/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py +++ b/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py @@ -197,7 +197,7 @@ async def create_read_session( number of pre-filtered rows, so some filters can lead to lopsided assignments. - Read sessions automatically expire 24 hours after they + Read sessions automatically expire 6 hours after they are created and do not require manual clean-up by the caller. @@ -441,17 +441,7 @@ async def split_read_stream( # and friendly error handling. rpc = gapic_v1.method_async.wrap_method( self._client._transport.split_read_stream, - default_retry=retries.Retry( - initial=0.1, - maximum=60.0, - multiplier=1.3, - predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, + default_timeout=None, client_info=DEFAULT_CLIENT_INFO, ) diff --git a/google/cloud/bigquery_storage_v1/services/big_query_read/client.py b/google/cloud/bigquery_storage_v1/services/big_query_read/client.py index 8279d2cb..80f8fb8c 100644 --- a/google/cloud/bigquery_storage_v1/services/big_query_read/client.py +++ b/google/cloud/bigquery_storage_v1/services/big_query_read/client.py @@ -418,7 +418,7 @@ def create_read_session( number of pre-filtered rows, so some filters can lead to lopsided assignments. - Read sessions automatically expire 24 hours after they + Read sessions automatically expire 6 hours after they are created and do not require manual clean-up by the caller. diff --git a/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py b/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py index af7f5390..65da10ff 100644 --- a/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py +++ b/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py @@ -189,19 +189,7 @@ def _prep_wrapped_messages(self, client_info): client_info=client_info, ), self.split_read_stream: gapic_v1.method.wrap_method( - self.split_read_stream, - default_retry=retries.Retry( - initial=0.1, - maximum=60.0, - multiplier=1.3, - predicate=retries.if_exception_type( - core_exceptions.DeadlineExceeded, - core_exceptions.ServiceUnavailable, - ), - deadline=600.0, - ), - default_timeout=600.0, - client_info=client_info, + self.split_read_stream, default_timeout=None, client_info=client_info, ), } diff --git a/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py b/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py index 6cb890e9..1ab6a270 100644 --- a/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py +++ b/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py @@ -253,7 +253,7 @@ def create_read_session( number of pre-filtered rows, so some filters can lead to lopsided assignments. - Read sessions automatically expire 24 hours after they + Read sessions automatically expire 6 hours after they are created and do not require manual clean-up by the caller. diff --git a/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py b/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py index fd5ecad0..c138a63d 100644 --- a/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py +++ b/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py @@ -256,7 +256,7 @@ def create_read_session( number of pre-filtered rows, so some filters can lead to lopsided assignments. - Read sessions automatically expire 24 hours after they + Read sessions automatically expire 6 hours after they are created and do not require manual clean-up by the caller. diff --git a/google/cloud/bigquery_storage_v1/services/big_query_write/__init__.py b/google/cloud/bigquery_storage_v1/services/big_query_write/__init__.py new file mode 100644 index 00000000..baf8bc92 --- /dev/null +++ b/google/cloud/bigquery_storage_v1/services/big_query_write/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .client import BigQueryWriteClient +from .async_client import BigQueryWriteAsyncClient + +__all__ = ( + "BigQueryWriteClient", + "BigQueryWriteAsyncClient", +) diff --git a/google/cloud/bigquery_storage_v1/services/big_query_write/async_client.py b/google/cloud/bigquery_storage_v1/services/big_query_write/async_client.py new file mode 100644 index 00000000..0f3753b4 --- /dev/null +++ b/google/cloud/bigquery_storage_v1/services/big_query_write/async_client.py @@ -0,0 +1,714 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +import functools +import re +from typing import ( + Dict, + AsyncIterable, + Awaitable, + AsyncIterator, + Sequence, + Tuple, + Type, + Union, +) +import pkg_resources + +import google.api_core.client_options as ClientOptions # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import retry as retries # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from google.cloud.bigquery_storage_v1.types import table +from google.protobuf import timestamp_pb2 # type: ignore +from google.rpc import status_pb2 # type: ignore +from .transports.base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO +from .transports.grpc_asyncio import BigQueryWriteGrpcAsyncIOTransport +from .client import BigQueryWriteClient + + +class BigQueryWriteAsyncClient: + """BigQuery Write API. + The Write API can be used to write data to BigQuery. + For supplementary information about the Write API, see: + https://cloud.google.com/bigquery/docs/write-api + """ + + _client: BigQueryWriteClient + + DEFAULT_ENDPOINT = BigQueryWriteClient.DEFAULT_ENDPOINT + DEFAULT_MTLS_ENDPOINT = BigQueryWriteClient.DEFAULT_MTLS_ENDPOINT + + table_path = staticmethod(BigQueryWriteClient.table_path) + parse_table_path = staticmethod(BigQueryWriteClient.parse_table_path) + write_stream_path = staticmethod(BigQueryWriteClient.write_stream_path) + parse_write_stream_path = staticmethod(BigQueryWriteClient.parse_write_stream_path) + common_billing_account_path = staticmethod( + BigQueryWriteClient.common_billing_account_path + ) + parse_common_billing_account_path = staticmethod( + BigQueryWriteClient.parse_common_billing_account_path + ) + common_folder_path = staticmethod(BigQueryWriteClient.common_folder_path) + parse_common_folder_path = staticmethod( + BigQueryWriteClient.parse_common_folder_path + ) + common_organization_path = staticmethod( + BigQueryWriteClient.common_organization_path + ) + parse_common_organization_path = staticmethod( + BigQueryWriteClient.parse_common_organization_path + ) + common_project_path = staticmethod(BigQueryWriteClient.common_project_path) + parse_common_project_path = staticmethod( + BigQueryWriteClient.parse_common_project_path + ) + common_location_path = staticmethod(BigQueryWriteClient.common_location_path) + parse_common_location_path = staticmethod( + BigQueryWriteClient.parse_common_location_path + ) + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteAsyncClient: The constructed client. + """ + return BigQueryWriteClient.from_service_account_info.__func__(BigQueryWriteAsyncClient, info, *args, **kwargs) # type: ignore + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteAsyncClient: The constructed client. + """ + return BigQueryWriteClient.from_service_account_file.__func__(BigQueryWriteAsyncClient, filename, *args, **kwargs) # type: ignore + + from_service_account_json = from_service_account_file + + @property + def transport(self) -> BigQueryWriteTransport: + """Returns the transport used by the client instance. + + Returns: + BigQueryWriteTransport: The transport used by the client instance. + """ + return self._client.transport + + get_transport_class = functools.partial( + type(BigQueryWriteClient).get_transport_class, type(BigQueryWriteClient) + ) + + def __init__( + self, + *, + credentials: ga_credentials.Credentials = None, + transport: Union[str, BigQueryWriteTransport] = "grpc_asyncio", + client_options: ClientOptions = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the big query write client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Union[str, ~.BigQueryWriteTransport]): The + transport to use. If set to None, a transport is chosen + automatically. + client_options (ClientOptions): Custom options for the client. It + won't take effect if a ``transport`` instance is provided. + (1) The ``api_endpoint`` property can be used to override the + default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT + environment variable can also be used to override the endpoint: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto switch to the + default mTLS endpoint if client certificate is present, this is + the default value). However, the ``api_endpoint`` property takes + precedence if provided. + (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide client certificate for mutual TLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + """ + self._client = BigQueryWriteClient( + credentials=credentials, + transport=transport, + client_options=client_options, + client_info=client_info, + ) + + async def create_write_stream( + self, + request: storage.CreateWriteStreamRequest = None, + *, + parent: str = None, + write_stream: stream.WriteStream = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.WriteStream: + r"""Creates a write stream to the given table. Additionally, every + table has a special stream named '_default' to which data can be + written. This stream doesn't need to be created using + CreateWriteStream. It is a stream that can be used + simultaneously by any number of clients. Data written to this + stream is considered committed as soon as an acknowledgement is + received. + + Args: + request (:class:`google.cloud.bigquery_storage_v1.types.CreateWriteStreamRequest`): + The request object. Request message for + `CreateWriteStream`. + parent (:class:`str`): + Required. Reference to the table to which the stream + belongs, in the format of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + write_stream (:class:`google.cloud.bigquery_storage_v1.types.WriteStream`): + Required. Stream to be created. + This corresponds to the ``write_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.WriteStream: + Information about a single stream + that gets data inside the storage + system. + + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, write_stream]) + if request is not None and has_flattened_params: + raise ValueError( + "If the `request` argument is set, then none of " + "the individual field arguments should be set." + ) + + request = storage.CreateWriteStreamRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if write_stream is not None: + request.write_stream = write_stream + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.create_write_stream, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), + ) + + # Send the request. + response = await rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Done; return the response. + return response + + def append_rows( + self, + requests: AsyncIterator[storage.AppendRowsRequest] = None, + *, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Awaitable[AsyncIterable[storage.AppendRowsResponse]]: + r"""Appends data to the given stream. + + If ``offset`` is specified, the ``offset`` is checked against + the end of stream. The server returns ``OUT_OF_RANGE`` in + ``AppendRowsResponse`` if an attempt is made to append to an + offset beyond the current end of the stream or + ``ALREADY_EXISTS`` if user provides an ``offset`` that has + already been written to. User can retry with adjusted offset + within the same RPC connection. If ``offset`` is not specified, + append happens at the end of the stream. + + The response contains an optional offset at which the append + happened. No offset information will be returned for appends to + a default stream. + + Responses are received in the same order in which requests are + sent. There will be one response for each successful inserted + request. Responses may optionally embed error information if the + originating AppendRequest was not successfully processed. + + The specifics of when successfully appended data is made visible + to the table are governed by the type of stream: + + - For COMMITTED streams (which includes the default stream), + data is visible immediately upon successful append. + + - For BUFFERED streams, data is made visible via a subsequent + ``FlushRows`` rpc which advances a cursor to a newer offset + in the stream. + + - For PENDING streams, data is not made visible until the + stream itself is finalized (via the ``FinalizeWriteStream`` + rpc), and the stream is explicitly committed via the + ``BatchCommitWriteStreams`` rpc. + + Args: + requests (AsyncIterator[`google.cloud.bigquery_storage_v1.types.AppendRowsRequest`]): + The request object AsyncIterator. Request message for `AppendRows`. + Due to the nature of AppendRows being a bidirectional + streaming RPC, certain parts of the AppendRowsRequest + need only be specified for the first request sent each + time the gRPC network connection is opened/reopened. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + AsyncIterable[google.cloud.bigquery_storage_v1.types.AppendRowsResponse]: + Response message for AppendRows. + """ + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.append_rows, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=86400.0, + ), + default_timeout=86400.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + (gapic_v1.routing_header.to_grpc_metadata(()),) + + # Send the request. + response = rpc(requests, retry=retry, timeout=timeout, metadata=metadata,) + + # Done; return the response. + return response + + async def get_write_stream( + self, + request: storage.GetWriteStreamRequest = None, + *, + name: str = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.WriteStream: + r"""Gets information about a write stream. + + Args: + request (:class:`google.cloud.bigquery_storage_v1.types.GetWriteStreamRequest`): + The request object. Request message for + `GetWriteStreamRequest`. + name (:class:`str`): + Required. Name of the stream to get, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.WriteStream: + Information about a single stream + that gets data inside the storage + system. + + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([name]) + if request is not None and has_flattened_params: + raise ValueError( + "If the `request` argument is set, then none of " + "the individual field arguments should be set." + ) + + request = storage.GetWriteStreamRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if name is not None: + request.name = name + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.get_write_stream, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), + ) + + # Send the request. + response = await rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Done; return the response. + return response + + async def finalize_write_stream( + self, + request: storage.FinalizeWriteStreamRequest = None, + *, + name: str = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.FinalizeWriteStreamResponse: + r"""Finalize a write stream so that no new data can be appended to + the stream. Finalize is not supported on the '_default' stream. + + Args: + request (:class:`google.cloud.bigquery_storage_v1.types.FinalizeWriteStreamRequest`): + The request object. Request message for invoking + `FinalizeWriteStream`. + name (:class:`str`): + Required. Name of the stream to finalize, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.FinalizeWriteStreamResponse: + Response message for FinalizeWriteStream. + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([name]) + if request is not None and has_flattened_params: + raise ValueError( + "If the `request` argument is set, then none of " + "the individual field arguments should be set." + ) + + request = storage.FinalizeWriteStreamRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if name is not None: + request.name = name + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.finalize_write_stream, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), + ) + + # Send the request. + response = await rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Done; return the response. + return response + + async def batch_commit_write_streams( + self, + request: storage.BatchCommitWriteStreamsRequest = None, + *, + parent: str = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.BatchCommitWriteStreamsResponse: + r"""Atomically commits a group of ``PENDING`` streams that belong to + the same ``parent`` table. + + Streams must be finalized before commit and cannot be committed + multiple times. Once a stream is committed, data in the stream + becomes available for read operations. + + Args: + request (:class:`google.cloud.bigquery_storage_v1.types.BatchCommitWriteStreamsRequest`): + The request object. Request message for + `BatchCommitWriteStreams`. + parent (:class:`str`): + Required. Parent table that all the streams should + belong to, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.BatchCommitWriteStreamsResponse: + Response message for BatchCommitWriteStreams. + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent]) + if request is not None and has_flattened_params: + raise ValueError( + "If the `request` argument is set, then none of " + "the individual field arguments should be set." + ) + + request = storage.BatchCommitWriteStreamsRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.batch_commit_write_streams, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), + ) + + # Send the request. + response = await rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Done; return the response. + return response + + async def flush_rows( + self, + request: storage.FlushRowsRequest = None, + *, + write_stream: str = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.FlushRowsResponse: + r"""Flushes rows to a BUFFERED stream. + + If users are appending rows to BUFFERED stream, flush operation + is required in order for the rows to become available for + reading. A Flush operation flushes up to any previously flushed + offset in a BUFFERED stream, to the offset specified in the + request. + + Flush is not supported on the \_default stream, since it is not + BUFFERED. + + Args: + request (:class:`google.cloud.bigquery_storage_v1.types.FlushRowsRequest`): + The request object. Request message for `FlushRows`. + write_stream (:class:`str`): + Required. The stream that is the + target of the flush operation. + + This corresponds to the ``write_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.FlushRowsResponse: + Respond message for FlushRows. + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([write_stream]) + if request is not None and has_flattened_params: + raise ValueError( + "If the `request` argument is set, then none of " + "the individual field arguments should be set." + ) + + request = storage.FlushRowsRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if write_stream is not None: + request.write_stream = write_stream + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.flush_rows, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata( + (("write_stream", request.write_stream),) + ), + ) + + # Send the request. + response = await rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Done; return the response. + return response + + +try: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=pkg_resources.get_distribution( + "google-cloud-bigquery-storage", + ).version, + ) +except pkg_resources.DistributionNotFound: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() + + +__all__ = ("BigQueryWriteAsyncClient",) diff --git a/google/cloud/bigquery_storage_v1/services/big_query_write/client.py b/google/cloud/bigquery_storage_v1/services/big_query_write/client.py new file mode 100644 index 00000000..09233410 --- /dev/null +++ b/google/cloud/bigquery_storage_v1/services/big_query_write/client.py @@ -0,0 +1,855 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from distutils import util +import os +import re +from typing import ( + Callable, + Dict, + Optional, + Iterable, + Iterator, + Sequence, + Tuple, + Type, + Union, +) +import pkg_resources + +from google.api_core import client_options as client_options_lib # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import retry as retries # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport import mtls # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore +from google.auth.exceptions import MutualTLSChannelError # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from google.cloud.bigquery_storage_v1.types import table +from google.protobuf import timestamp_pb2 # type: ignore +from google.rpc import status_pb2 # type: ignore +from .transports.base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO +from .transports.grpc import BigQueryWriteGrpcTransport +from .transports.grpc_asyncio import BigQueryWriteGrpcAsyncIOTransport + + +class BigQueryWriteClientMeta(type): + """Metaclass for the BigQueryWrite client. + + This provides class-level methods for building and retrieving + support objects (e.g. transport) without polluting the client instance + objects. + """ + + _transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryWriteTransport]] + _transport_registry["grpc"] = BigQueryWriteGrpcTransport + _transport_registry["grpc_asyncio"] = BigQueryWriteGrpcAsyncIOTransport + + def get_transport_class(cls, label: str = None,) -> Type[BigQueryWriteTransport]: + """Returns an appropriate transport class. + + Args: + label: The name of the desired transport. If none is + provided, then the first transport in the registry is used. + + Returns: + The transport class to use. + """ + # If a specific transport is requested, return that one. + if label: + return cls._transport_registry[label] + + # No transport is requested; return the default (that is, the first one + # in the dictionary). + return next(iter(cls._transport_registry.values())) + + +class BigQueryWriteClient(metaclass=BigQueryWriteClientMeta): + """BigQuery Write API. + The Write API can be used to write data to BigQuery. + For supplementary information about the Write API, see: + https://cloud.google.com/bigquery/docs/write-api + """ + + @staticmethod + def _get_default_mtls_endpoint(api_endpoint): + """Converts api endpoint to mTLS endpoint. + + Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to + "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. + Args: + api_endpoint (Optional[str]): the api endpoint to convert. + Returns: + str: converted mTLS api endpoint. + """ + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + DEFAULT_ENDPOINT = "bigquerystorage.googleapis.com" + DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore + DEFAULT_ENDPOINT + ) + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_info(info) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_file(filename) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + from_service_account_json = from_service_account_file + + @property + def transport(self) -> BigQueryWriteTransport: + """Returns the transport used by the client instance. + + Returns: + BigQueryWriteTransport: The transport used by the client + instance. + """ + return self._transport + + @staticmethod + def table_path(project: str, dataset: str, table: str,) -> str: + """Returns a fully-qualified table string.""" + return "projects/{project}/datasets/{dataset}/tables/{table}".format( + project=project, dataset=dataset, table=table, + ) + + @staticmethod + def parse_table_path(path: str) -> Dict[str, str]: + """Parses a table path into its component segments.""" + m = re.match( + r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P.+?)$", + path, + ) + return m.groupdict() if m else {} + + @staticmethod + def write_stream_path(project: str, dataset: str, table: str, stream: str,) -> str: + """Returns a fully-qualified write_stream string.""" + return "projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}".format( + project=project, dataset=dataset, table=table, stream=stream, + ) + + @staticmethod + def parse_write_stream_path(path: str) -> Dict[str, str]: + """Parses a write_stream path into its component segments.""" + m = re.match( + r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P
.+?)/streams/(?P.+?)$", + path, + ) + return m.groupdict() if m else {} + + @staticmethod + def common_billing_account_path(billing_account: str,) -> str: + """Returns a fully-qualified billing_account string.""" + return "billingAccounts/{billing_account}".format( + billing_account=billing_account, + ) + + @staticmethod + def parse_common_billing_account_path(path: str) -> Dict[str, str]: + """Parse a billing_account path into its component segments.""" + m = re.match(r"^billingAccounts/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_folder_path(folder: str,) -> str: + """Returns a fully-qualified folder string.""" + return "folders/{folder}".format(folder=folder,) + + @staticmethod + def parse_common_folder_path(path: str) -> Dict[str, str]: + """Parse a folder path into its component segments.""" + m = re.match(r"^folders/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_organization_path(organization: str,) -> str: + """Returns a fully-qualified organization string.""" + return "organizations/{organization}".format(organization=organization,) + + @staticmethod + def parse_common_organization_path(path: str) -> Dict[str, str]: + """Parse a organization path into its component segments.""" + m = re.match(r"^organizations/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_project_path(project: str,) -> str: + """Returns a fully-qualified project string.""" + return "projects/{project}".format(project=project,) + + @staticmethod + def parse_common_project_path(path: str) -> Dict[str, str]: + """Parse a project path into its component segments.""" + m = re.match(r"^projects/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_location_path(project: str, location: str,) -> str: + """Returns a fully-qualified location string.""" + return "projects/{project}/locations/{location}".format( + project=project, location=location, + ) + + @staticmethod + def parse_common_location_path(path: str) -> Dict[str, str]: + """Parse a location path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path) + return m.groupdict() if m else {} + + def __init__( + self, + *, + credentials: Optional[ga_credentials.Credentials] = None, + transport: Union[str, BigQueryWriteTransport, None] = None, + client_options: Optional[client_options_lib.ClientOptions] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the big query write client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Union[str, BigQueryWriteTransport]): The + transport to use. If set to None, a transport is chosen + automatically. + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. It won't take effect if a ``transport`` instance is provided. + (1) The ``api_endpoint`` property can be used to override the + default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT + environment variable can also be used to override the endpoint: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto switch to the + default mTLS endpoint if client certificate is present, this is + the default value). However, the ``api_endpoint`` property takes + precedence if provided. + (2) If GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide client certificate for mutual TLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + """ + if isinstance(client_options, dict): + client_options = client_options_lib.from_dict(client_options) + if client_options is None: + client_options = client_options_lib.ClientOptions() + + # Create SSL credentials for mutual TLS if needed. + use_client_cert = bool( + util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false")) + ) + + client_cert_source_func = None + is_mtls = False + if use_client_cert: + if client_options.client_cert_source: + is_mtls = True + client_cert_source_func = client_options.client_cert_source + else: + is_mtls = mtls.has_default_client_cert_source() + if is_mtls: + client_cert_source_func = mtls.default_client_cert_source() + else: + client_cert_source_func = None + + # Figure out which api endpoint to use. + if client_options.api_endpoint is not None: + api_endpoint = client_options.api_endpoint + else: + use_mtls_env = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") + if use_mtls_env == "never": + api_endpoint = self.DEFAULT_ENDPOINT + elif use_mtls_env == "always": + api_endpoint = self.DEFAULT_MTLS_ENDPOINT + elif use_mtls_env == "auto": + if is_mtls: + api_endpoint = self.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = self.DEFAULT_ENDPOINT + else: + raise MutualTLSChannelError( + "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted " + "values: never, auto, always" + ) + + # Save or instantiate the transport. + # Ordinarily, we provide the transport, but allowing a custom transport + # instance provides an extensibility point for unusual situations. + if isinstance(transport, BigQueryWriteTransport): + # transport is a BigQueryWriteTransport instance. + if credentials or client_options.credentials_file: + raise ValueError( + "When providing a transport instance, " + "provide its credentials directly." + ) + if client_options.scopes: + raise ValueError( + "When providing a transport instance, provide its scopes " + "directly." + ) + self._transport = transport + else: + Transport = type(self).get_transport_class(transport) + self._transport = Transport( + credentials=credentials, + credentials_file=client_options.credentials_file, + host=api_endpoint, + scopes=client_options.scopes, + client_cert_source_for_mtls=client_cert_source_func, + quota_project_id=client_options.quota_project_id, + client_info=client_info, + always_use_jwt_access=( + Transport == type(self).get_transport_class("grpc") + or Transport == type(self).get_transport_class("grpc_asyncio") + ), + ) + + def create_write_stream( + self, + request: storage.CreateWriteStreamRequest = None, + *, + parent: str = None, + write_stream: stream.WriteStream = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.WriteStream: + r"""Creates a write stream to the given table. Additionally, every + table has a special stream named '_default' to which data can be + written. This stream doesn't need to be created using + CreateWriteStream. It is a stream that can be used + simultaneously by any number of clients. Data written to this + stream is considered committed as soon as an acknowledgement is + received. + + Args: + request (google.cloud.bigquery_storage_v1.types.CreateWriteStreamRequest): + The request object. Request message for + `CreateWriteStream`. + parent (str): + Required. Reference to the table to which the stream + belongs, in the format of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + write_stream (google.cloud.bigquery_storage_v1.types.WriteStream): + Required. Stream to be created. + This corresponds to the ``write_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.WriteStream: + Information about a single stream + that gets data inside the storage + system. + + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, write_stream]) + if request is not None and has_flattened_params: + raise ValueError( + "If the `request` argument is set, then none of " + "the individual field arguments should be set." + ) + + # Minor optimization to avoid making a copy if the user passes + # in a storage.CreateWriteStreamRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, storage.CreateWriteStreamRequest): + request = storage.CreateWriteStreamRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if write_stream is not None: + request.write_stream = write_stream + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.create_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), + ) + + # Send the request. + response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Done; return the response. + return response + + def append_rows( + self, + requests: Iterator[storage.AppendRowsRequest] = None, + *, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Iterable[storage.AppendRowsResponse]: + r"""Appends data to the given stream. + + If ``offset`` is specified, the ``offset`` is checked against + the end of stream. The server returns ``OUT_OF_RANGE`` in + ``AppendRowsResponse`` if an attempt is made to append to an + offset beyond the current end of the stream or + ``ALREADY_EXISTS`` if user provides an ``offset`` that has + already been written to. User can retry with adjusted offset + within the same RPC connection. If ``offset`` is not specified, + append happens at the end of the stream. + + The response contains an optional offset at which the append + happened. No offset information will be returned for appends to + a default stream. + + Responses are received in the same order in which requests are + sent. There will be one response for each successful inserted + request. Responses may optionally embed error information if the + originating AppendRequest was not successfully processed. + + The specifics of when successfully appended data is made visible + to the table are governed by the type of stream: + + - For COMMITTED streams (which includes the default stream), + data is visible immediately upon successful append. + + - For BUFFERED streams, data is made visible via a subsequent + ``FlushRows`` rpc which advances a cursor to a newer offset + in the stream. + + - For PENDING streams, data is not made visible until the + stream itself is finalized (via the ``FinalizeWriteStream`` + rpc), and the stream is explicitly committed via the + ``BatchCommitWriteStreams`` rpc. + + Args: + requests (Iterator[google.cloud.bigquery_storage_v1.types.AppendRowsRequest]): + The request object iterator. Request message for `AppendRows`. + Due to the nature of AppendRows being a bidirectional + streaming RPC, certain parts of the AppendRowsRequest + need only be specified for the first request sent each + time the gRPC network connection is opened/reopened. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + Iterable[google.cloud.bigquery_storage_v1.types.AppendRowsResponse]: + Response message for AppendRows. + """ + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.append_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + (gapic_v1.routing_header.to_grpc_metadata(()),) + + # Send the request. + response = rpc(requests, retry=retry, timeout=timeout, metadata=metadata,) + + # Done; return the response. + return response + + def get_write_stream( + self, + request: storage.GetWriteStreamRequest = None, + *, + name: str = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.WriteStream: + r"""Gets information about a write stream. + + Args: + request (google.cloud.bigquery_storage_v1.types.GetWriteStreamRequest): + The request object. Request message for + `GetWriteStreamRequest`. + name (str): + Required. Name of the stream to get, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.WriteStream: + Information about a single stream + that gets data inside the storage + system. + + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([name]) + if request is not None and has_flattened_params: + raise ValueError( + "If the `request` argument is set, then none of " + "the individual field arguments should be set." + ) + + # Minor optimization to avoid making a copy if the user passes + # in a storage.GetWriteStreamRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, storage.GetWriteStreamRequest): + request = storage.GetWriteStreamRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if name is not None: + request.name = name + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.get_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), + ) + + # Send the request. + response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Done; return the response. + return response + + def finalize_write_stream( + self, + request: storage.FinalizeWriteStreamRequest = None, + *, + name: str = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.FinalizeWriteStreamResponse: + r"""Finalize a write stream so that no new data can be appended to + the stream. Finalize is not supported on the '_default' stream. + + Args: + request (google.cloud.bigquery_storage_v1.types.FinalizeWriteStreamRequest): + The request object. Request message for invoking + `FinalizeWriteStream`. + name (str): + Required. Name of the stream to finalize, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.FinalizeWriteStreamResponse: + Response message for FinalizeWriteStream. + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([name]) + if request is not None and has_flattened_params: + raise ValueError( + "If the `request` argument is set, then none of " + "the individual field arguments should be set." + ) + + # Minor optimization to avoid making a copy if the user passes + # in a storage.FinalizeWriteStreamRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, storage.FinalizeWriteStreamRequest): + request = storage.FinalizeWriteStreamRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if name is not None: + request.name = name + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.finalize_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata((("name", request.name),)), + ) + + # Send the request. + response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Done; return the response. + return response + + def batch_commit_write_streams( + self, + request: storage.BatchCommitWriteStreamsRequest = None, + *, + parent: str = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.BatchCommitWriteStreamsResponse: + r"""Atomically commits a group of ``PENDING`` streams that belong to + the same ``parent`` table. + + Streams must be finalized before commit and cannot be committed + multiple times. Once a stream is committed, data in the stream + becomes available for read operations. + + Args: + request (google.cloud.bigquery_storage_v1.types.BatchCommitWriteStreamsRequest): + The request object. Request message for + `BatchCommitWriteStreams`. + parent (str): + Required. Parent table that all the streams should + belong to, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.BatchCommitWriteStreamsResponse: + Response message for BatchCommitWriteStreams. + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent]) + if request is not None and has_flattened_params: + raise ValueError( + "If the `request` argument is set, then none of " + "the individual field arguments should be set." + ) + + # Minor optimization to avoid making a copy if the user passes + # in a storage.BatchCommitWriteStreamsRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, storage.BatchCommitWriteStreamsRequest): + request = storage.BatchCommitWriteStreamsRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[ + self._transport.batch_commit_write_streams + ] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata((("parent", request.parent),)), + ) + + # Send the request. + response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Done; return the response. + return response + + def flush_rows( + self, + request: storage.FlushRowsRequest = None, + *, + write_stream: str = None, + retry: retries.Retry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.FlushRowsResponse: + r"""Flushes rows to a BUFFERED stream. + + If users are appending rows to BUFFERED stream, flush operation + is required in order for the rows to become available for + reading. A Flush operation flushes up to any previously flushed + offset in a BUFFERED stream, to the offset specified in the + request. + + Flush is not supported on the \_default stream, since it is not + BUFFERED. + + Args: + request (google.cloud.bigquery_storage_v1.types.FlushRowsRequest): + The request object. Request message for `FlushRows`. + write_stream (str): + Required. The stream that is the + target of the flush operation. + + This corresponds to the ``write_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.FlushRowsResponse: + Respond message for FlushRows. + """ + # Create or coerce a protobuf request object. + # Sanity check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([write_stream]) + if request is not None and has_flattened_params: + raise ValueError( + "If the `request` argument is set, then none of " + "the individual field arguments should be set." + ) + + # Minor optimization to avoid making a copy if the user passes + # in a storage.FlushRowsRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, storage.FlushRowsRequest): + request = storage.FlushRowsRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if write_stream is not None: + request.write_stream = write_stream + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.flush_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata( + (("write_stream", request.write_stream),) + ), + ) + + # Send the request. + response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Done; return the response. + return response + + +try: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=pkg_resources.get_distribution( + "google-cloud-bigquery-storage", + ).version, + ) +except pkg_resources.DistributionNotFound: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() + + +__all__ = ("BigQueryWriteClient",) diff --git a/google/cloud/bigquery_storage_v1/services/big_query_write/transports/__init__.py b/google/cloud/bigquery_storage_v1/services/big_query_write/transports/__init__.py new file mode 100644 index 00000000..0153b19a --- /dev/null +++ b/google/cloud/bigquery_storage_v1/services/big_query_write/transports/__init__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from typing import Dict, Type + +from .base import BigQueryWriteTransport +from .grpc import BigQueryWriteGrpcTransport +from .grpc_asyncio import BigQueryWriteGrpcAsyncIOTransport + + +# Compile a registry of transports. +_transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryWriteTransport]] +_transport_registry["grpc"] = BigQueryWriteGrpcTransport +_transport_registry["grpc_asyncio"] = BigQueryWriteGrpcAsyncIOTransport + +__all__ = ( + "BigQueryWriteTransport", + "BigQueryWriteGrpcTransport", + "BigQueryWriteGrpcAsyncIOTransport", +) diff --git a/google/cloud/bigquery_storage_v1/services/big_query_write/transports/base.py b/google/cloud/bigquery_storage_v1/services/big_query_write/transports/base.py new file mode 100644 index 00000000..1ecbb93f --- /dev/null +++ b/google/cloud/bigquery_storage_v1/services/big_query_write/transports/base.py @@ -0,0 +1,314 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import abc +from typing import Awaitable, Callable, Dict, Optional, Sequence, Union +import packaging.version +import pkg_resources + +import google.auth # type: ignore +import google.api_core # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore +from google.api_core import gapic_v1 # type: ignore +from google.api_core import retry as retries # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream + +try: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( + gapic_version=pkg_resources.get_distribution( + "google-cloud-bigquery-storage", + ).version, + ) +except pkg_resources.DistributionNotFound: + DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo() + +try: + # google.auth.__version__ was added in 1.26.0 + _GOOGLE_AUTH_VERSION = google.auth.__version__ +except AttributeError: + try: # try pkg_resources if it is available + _GOOGLE_AUTH_VERSION = pkg_resources.get_distribution("google-auth").version + except pkg_resources.DistributionNotFound: # pragma: NO COVER + _GOOGLE_AUTH_VERSION = None + + +class BigQueryWriteTransport(abc.ABC): + """Abstract transport class for BigQueryWrite.""" + + AUTH_SCOPES = ( + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/bigquery.insertdata", + "https://www.googleapis.com/auth/cloud-platform", + ) + + DEFAULT_HOST: str = "bigquerystorage.googleapis.com" + + def __init__( + self, + *, + host: str = DEFAULT_HOST, + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + **kwargs, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A list of scopes. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + """ + # Save the hostname. Default to port 443 (HTTPS) if none is specified. + if ":" not in host: + host += ":443" + self._host = host + + scopes_kwargs = self._get_scopes_kwargs(self._host, scopes) + + # Save the scopes. + self._scopes = scopes + + # If no credentials are provided, then determine the appropriate + # defaults. + if credentials and credentials_file: + raise core_exceptions.DuplicateCredentialArgs( + "'credentials_file' and 'credentials' are mutually exclusive" + ) + + if credentials_file is not None: + credentials, _ = google.auth.load_credentials_from_file( + credentials_file, **scopes_kwargs, quota_project_id=quota_project_id + ) + + elif credentials is None: + credentials, _ = google.auth.default( + **scopes_kwargs, quota_project_id=quota_project_id + ) + + # If the credentials is service account credentials, then always try to use self signed JWT. + if ( + always_use_jwt_access + and isinstance(credentials, service_account.Credentials) + and hasattr(service_account.Credentials, "with_always_use_jwt_access") + ): + credentials = credentials.with_always_use_jwt_access(True) + + # Save the credentials. + self._credentials = credentials + + # TODO(busunkim): This method is in the base transport + # to avoid duplicating code across the transport classes. These functions + # should be deleted once the minimum required versions of google-auth is increased. + + # TODO: Remove this function once google-auth >= 1.25.0 is required + @classmethod + def _get_scopes_kwargs( + cls, host: str, scopes: Optional[Sequence[str]] + ) -> Dict[str, Optional[Sequence[str]]]: + """Returns scopes kwargs to pass to google-auth methods depending on the google-auth version""" + + scopes_kwargs = {} + + if _GOOGLE_AUTH_VERSION and ( + packaging.version.parse(_GOOGLE_AUTH_VERSION) + >= packaging.version.parse("1.25.0") + ): + scopes_kwargs = {"scopes": scopes, "default_scopes": cls.AUTH_SCOPES} + else: + scopes_kwargs = {"scopes": scopes or cls.AUTH_SCOPES} + + return scopes_kwargs + + def _prep_wrapped_messages(self, client_info): + # Precompute the wrapped methods. + self._wrapped_methods = { + self.create_write_stream: gapic_v1.method.wrap_method( + self.create_write_stream, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.append_rows: gapic_v1.method.wrap_method( + self.append_rows, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=86400.0, + ), + default_timeout=86400.0, + client_info=client_info, + ), + self.get_write_stream: gapic_v1.method.wrap_method( + self.get_write_stream, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.finalize_write_stream: gapic_v1.method.wrap_method( + self.finalize_write_stream, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.batch_commit_write_streams: gapic_v1.method.wrap_method( + self.batch_commit_write_streams, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.flush_rows: gapic_v1.method.wrap_method( + self.flush_rows, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + } + + @property + def create_write_stream( + self, + ) -> Callable[ + [storage.CreateWriteStreamRequest], + Union[stream.WriteStream, Awaitable[stream.WriteStream]], + ]: + raise NotImplementedError() + + @property + def append_rows( + self, + ) -> Callable[ + [storage.AppendRowsRequest], + Union[storage.AppendRowsResponse, Awaitable[storage.AppendRowsResponse]], + ]: + raise NotImplementedError() + + @property + def get_write_stream( + self, + ) -> Callable[ + [storage.GetWriteStreamRequest], + Union[stream.WriteStream, Awaitable[stream.WriteStream]], + ]: + raise NotImplementedError() + + @property + def finalize_write_stream( + self, + ) -> Callable[ + [storage.FinalizeWriteStreamRequest], + Union[ + storage.FinalizeWriteStreamResponse, + Awaitable[storage.FinalizeWriteStreamResponse], + ], + ]: + raise NotImplementedError() + + @property + def batch_commit_write_streams( + self, + ) -> Callable[ + [storage.BatchCommitWriteStreamsRequest], + Union[ + storage.BatchCommitWriteStreamsResponse, + Awaitable[storage.BatchCommitWriteStreamsResponse], + ], + ]: + raise NotImplementedError() + + @property + def flush_rows( + self, + ) -> Callable[ + [storage.FlushRowsRequest], + Union[storage.FlushRowsResponse, Awaitable[storage.FlushRowsResponse]], + ]: + raise NotImplementedError() + + +__all__ = ("BigQueryWriteTransport",) diff --git a/google/cloud/bigquery_storage_v1/services/big_query_write/transports/grpc.py b/google/cloud/bigquery_storage_v1/services/big_query_write/transports/grpc.py new file mode 100644 index 00000000..68b3b076 --- /dev/null +++ b/google/cloud/bigquery_storage_v1/services/big_query_write/transports/grpc.py @@ -0,0 +1,448 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import warnings +from typing import Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import grpc_helpers # type: ignore +from google.api_core import gapic_v1 # type: ignore +import google.auth # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore + +import grpc # type: ignore + +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from .base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO + + +class BigQueryWriteGrpcTransport(BigQueryWriteTransport): + """gRPC backend transport for BigQueryWrite. + + BigQuery Write API. + The Write API can be used to write data to BigQuery. + For supplementary information about the Write API, see: + https://cloud.google.com/bigquery/docs/write-api + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + + _stubs: Dict[str, Callable] + + def __init__( + self, + *, + host: str = "bigquerystorage.googleapis.com", + credentials: ga_credentials.Credentials = None, + credentials_file: str = None, + scopes: Sequence[str] = None, + channel: grpc.Channel = None, + api_mtls_endpoint: str = None, + client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, + ssl_channel_credentials: grpc.ChannelCredentials = None, + client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if ``channel`` is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if ``channel`` is provided. + scopes (Optional(Sequence[str])): A list of scopes. This argument is + ignored if ``channel`` is provided. + channel (Optional[grpc.Channel]): A ``Channel`` instance through + which to make calls. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or applicatin default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for grpc channel. It is ignored if ``channel`` is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure mutual TLS channel. It is + ignored if ``channel`` or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if channel: + # Ignore credentials if a channel was passed. + credentials = False + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + ) + + if not self._grpc_channel: + self._grpc_channel = type(self).create_channel( + self._host, + credentials=self._credentials, + credentials_file=credentials_file, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) + + @classmethod + def create_channel( + cls, + host: str = "bigquerystorage.googleapis.com", + credentials: ga_credentials.Credentials = None, + credentials_file: str = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs, + ) -> grpc.Channel: + """Create and return a gRPC channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + grpc.Channel: A gRPC channel object. + + Raises: + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + + return grpc_helpers.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs, + ) + + @property + def grpc_channel(self) -> grpc.Channel: + """Return the channel designed to connect to this service. + """ + return self._grpc_channel + + @property + def create_write_stream( + self, + ) -> Callable[[storage.CreateWriteStreamRequest], stream.WriteStream]: + r"""Return a callable for the create write stream method over gRPC. + + Creates a write stream to the given table. Additionally, every + table has a special stream named '_default' to which data can be + written. This stream doesn't need to be created using + CreateWriteStream. It is a stream that can be used + simultaneously by any number of clients. Data written to this + stream is considered committed as soon as an acknowledgement is + received. + + Returns: + Callable[[~.CreateWriteStreamRequest], + ~.WriteStream]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "create_write_stream" not in self._stubs: + self._stubs["create_write_stream"] = self.grpc_channel.unary_unary( + "/google.cloud.bigquery.storage.v1.BigQueryWrite/CreateWriteStream", + request_serializer=storage.CreateWriteStreamRequest.serialize, + response_deserializer=stream.WriteStream.deserialize, + ) + return self._stubs["create_write_stream"] + + @property + def append_rows( + self, + ) -> Callable[[storage.AppendRowsRequest], storage.AppendRowsResponse]: + r"""Return a callable for the append rows method over gRPC. + + Appends data to the given stream. + + If ``offset`` is specified, the ``offset`` is checked against + the end of stream. The server returns ``OUT_OF_RANGE`` in + ``AppendRowsResponse`` if an attempt is made to append to an + offset beyond the current end of the stream or + ``ALREADY_EXISTS`` if user provides an ``offset`` that has + already been written to. User can retry with adjusted offset + within the same RPC connection. If ``offset`` is not specified, + append happens at the end of the stream. + + The response contains an optional offset at which the append + happened. No offset information will be returned for appends to + a default stream. + + Responses are received in the same order in which requests are + sent. There will be one response for each successful inserted + request. Responses may optionally embed error information if the + originating AppendRequest was not successfully processed. + + The specifics of when successfully appended data is made visible + to the table are governed by the type of stream: + + - For COMMITTED streams (which includes the default stream), + data is visible immediately upon successful append. + + - For BUFFERED streams, data is made visible via a subsequent + ``FlushRows`` rpc which advances a cursor to a newer offset + in the stream. + + - For PENDING streams, data is not made visible until the + stream itself is finalized (via the ``FinalizeWriteStream`` + rpc), and the stream is explicitly committed via the + ``BatchCommitWriteStreams`` rpc. + + Returns: + Callable[[~.AppendRowsRequest], + ~.AppendRowsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "append_rows" not in self._stubs: + self._stubs["append_rows"] = self.grpc_channel.stream_stream( + "/google.cloud.bigquery.storage.v1.BigQueryWrite/AppendRows", + request_serializer=storage.AppendRowsRequest.serialize, + response_deserializer=storage.AppendRowsResponse.deserialize, + ) + return self._stubs["append_rows"] + + @property + def get_write_stream( + self, + ) -> Callable[[storage.GetWriteStreamRequest], stream.WriteStream]: + r"""Return a callable for the get write stream method over gRPC. + + Gets information about a write stream. + + Returns: + Callable[[~.GetWriteStreamRequest], + ~.WriteStream]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "get_write_stream" not in self._stubs: + self._stubs["get_write_stream"] = self.grpc_channel.unary_unary( + "/google.cloud.bigquery.storage.v1.BigQueryWrite/GetWriteStream", + request_serializer=storage.GetWriteStreamRequest.serialize, + response_deserializer=stream.WriteStream.deserialize, + ) + return self._stubs["get_write_stream"] + + @property + def finalize_write_stream( + self, + ) -> Callable[ + [storage.FinalizeWriteStreamRequest], storage.FinalizeWriteStreamResponse + ]: + r"""Return a callable for the finalize write stream method over gRPC. + + Finalize a write stream so that no new data can be appended to + the stream. Finalize is not supported on the '_default' stream. + + Returns: + Callable[[~.FinalizeWriteStreamRequest], + ~.FinalizeWriteStreamResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "finalize_write_stream" not in self._stubs: + self._stubs["finalize_write_stream"] = self.grpc_channel.unary_unary( + "/google.cloud.bigquery.storage.v1.BigQueryWrite/FinalizeWriteStream", + request_serializer=storage.FinalizeWriteStreamRequest.serialize, + response_deserializer=storage.FinalizeWriteStreamResponse.deserialize, + ) + return self._stubs["finalize_write_stream"] + + @property + def batch_commit_write_streams( + self, + ) -> Callable[ + [storage.BatchCommitWriteStreamsRequest], + storage.BatchCommitWriteStreamsResponse, + ]: + r"""Return a callable for the batch commit write streams method over gRPC. + + Atomically commits a group of ``PENDING`` streams that belong to + the same ``parent`` table. + + Streams must be finalized before commit and cannot be committed + multiple times. Once a stream is committed, data in the stream + becomes available for read operations. + + Returns: + Callable[[~.BatchCommitWriteStreamsRequest], + ~.BatchCommitWriteStreamsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "batch_commit_write_streams" not in self._stubs: + self._stubs["batch_commit_write_streams"] = self.grpc_channel.unary_unary( + "/google.cloud.bigquery.storage.v1.BigQueryWrite/BatchCommitWriteStreams", + request_serializer=storage.BatchCommitWriteStreamsRequest.serialize, + response_deserializer=storage.BatchCommitWriteStreamsResponse.deserialize, + ) + return self._stubs["batch_commit_write_streams"] + + @property + def flush_rows( + self, + ) -> Callable[[storage.FlushRowsRequest], storage.FlushRowsResponse]: + r"""Return a callable for the flush rows method over gRPC. + + Flushes rows to a BUFFERED stream. + + If users are appending rows to BUFFERED stream, flush operation + is required in order for the rows to become available for + reading. A Flush operation flushes up to any previously flushed + offset in a BUFFERED stream, to the offset specified in the + request. + + Flush is not supported on the \_default stream, since it is not + BUFFERED. + + Returns: + Callable[[~.FlushRowsRequest], + ~.FlushRowsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "flush_rows" not in self._stubs: + self._stubs["flush_rows"] = self.grpc_channel.unary_unary( + "/google.cloud.bigquery.storage.v1.BigQueryWrite/FlushRows", + request_serializer=storage.FlushRowsRequest.serialize, + response_deserializer=storage.FlushRowsResponse.deserialize, + ) + return self._stubs["flush_rows"] + + +__all__ = ("BigQueryWriteGrpcTransport",) diff --git a/google/cloud/bigquery_storage_v1/services/big_query_write/transports/grpc_asyncio.py b/google/cloud/bigquery_storage_v1/services/big_query_write/transports/grpc_asyncio.py new file mode 100644 index 00000000..7202e0e3 --- /dev/null +++ b/google/cloud/bigquery_storage_v1/services/big_query_write/transports/grpc_asyncio.py @@ -0,0 +1,452 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import warnings +from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import gapic_v1 # type: ignore +from google.api_core import grpc_helpers_async # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore +import packaging.version + +import grpc # type: ignore +from grpc.experimental import aio # type: ignore + +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from .base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO +from .grpc import BigQueryWriteGrpcTransport + + +class BigQueryWriteGrpcAsyncIOTransport(BigQueryWriteTransport): + """gRPC AsyncIO backend transport for BigQueryWrite. + + BigQuery Write API. + The Write API can be used to write data to BigQuery. + For supplementary information about the Write API, see: + https://cloud.google.com/bigquery/docs/write-api + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + + _grpc_channel: aio.Channel + _stubs: Dict[str, Callable] = {} + + @classmethod + def create_channel( + cls, + host: str = "bigquerystorage.googleapis.com", + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs, + ) -> aio.Channel: + """Create and return a gRPC AsyncIO channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if ``channel`` is provided. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + aio.Channel: A gRPC AsyncIO channel object. + """ + + return grpc_helpers_async.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs, + ) + + def __init__( + self, + *, + host: str = "bigquerystorage.googleapis.com", + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + channel: aio.Channel = None, + api_mtls_endpoint: str = None, + client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, + ssl_channel_credentials: grpc.ChannelCredentials = None, + client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, + quota_project_id=None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to. + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if ``channel`` is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if ``channel`` is provided. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + channel (Optional[aio.Channel]): A ``Channel`` instance through + which to make calls. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or applicatin default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for grpc channel. It is ignored if ``channel`` is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure mutual TLS channel. It is + ignored if ``channel`` or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if channel: + # Ignore credentials if a channel was passed. + credentials = False + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + ) + + if not self._grpc_channel: + self._grpc_channel = type(self).create_channel( + self._host, + credentials=self._credentials, + credentials_file=credentials_file, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) + + @property + def grpc_channel(self) -> aio.Channel: + """Create the channel designed to connect to this service. + + This property caches on the instance; repeated calls return + the same channel. + """ + # Return the channel from cache. + return self._grpc_channel + + @property + def create_write_stream( + self, + ) -> Callable[[storage.CreateWriteStreamRequest], Awaitable[stream.WriteStream]]: + r"""Return a callable for the create write stream method over gRPC. + + Creates a write stream to the given table. Additionally, every + table has a special stream named '_default' to which data can be + written. This stream doesn't need to be created using + CreateWriteStream. It is a stream that can be used + simultaneously by any number of clients. Data written to this + stream is considered committed as soon as an acknowledgement is + received. + + Returns: + Callable[[~.CreateWriteStreamRequest], + Awaitable[~.WriteStream]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "create_write_stream" not in self._stubs: + self._stubs["create_write_stream"] = self.grpc_channel.unary_unary( + "/google.cloud.bigquery.storage.v1.BigQueryWrite/CreateWriteStream", + request_serializer=storage.CreateWriteStreamRequest.serialize, + response_deserializer=stream.WriteStream.deserialize, + ) + return self._stubs["create_write_stream"] + + @property + def append_rows( + self, + ) -> Callable[[storage.AppendRowsRequest], Awaitable[storage.AppendRowsResponse]]: + r"""Return a callable for the append rows method over gRPC. + + Appends data to the given stream. + + If ``offset`` is specified, the ``offset`` is checked against + the end of stream. The server returns ``OUT_OF_RANGE`` in + ``AppendRowsResponse`` if an attempt is made to append to an + offset beyond the current end of the stream or + ``ALREADY_EXISTS`` if user provides an ``offset`` that has + already been written to. User can retry with adjusted offset + within the same RPC connection. If ``offset`` is not specified, + append happens at the end of the stream. + + The response contains an optional offset at which the append + happened. No offset information will be returned for appends to + a default stream. + + Responses are received in the same order in which requests are + sent. There will be one response for each successful inserted + request. Responses may optionally embed error information if the + originating AppendRequest was not successfully processed. + + The specifics of when successfully appended data is made visible + to the table are governed by the type of stream: + + - For COMMITTED streams (which includes the default stream), + data is visible immediately upon successful append. + + - For BUFFERED streams, data is made visible via a subsequent + ``FlushRows`` rpc which advances a cursor to a newer offset + in the stream. + + - For PENDING streams, data is not made visible until the + stream itself is finalized (via the ``FinalizeWriteStream`` + rpc), and the stream is explicitly committed via the + ``BatchCommitWriteStreams`` rpc. + + Returns: + Callable[[~.AppendRowsRequest], + Awaitable[~.AppendRowsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "append_rows" not in self._stubs: + self._stubs["append_rows"] = self.grpc_channel.stream_stream( + "/google.cloud.bigquery.storage.v1.BigQueryWrite/AppendRows", + request_serializer=storage.AppendRowsRequest.serialize, + response_deserializer=storage.AppendRowsResponse.deserialize, + ) + return self._stubs["append_rows"] + + @property + def get_write_stream( + self, + ) -> Callable[[storage.GetWriteStreamRequest], Awaitable[stream.WriteStream]]: + r"""Return a callable for the get write stream method over gRPC. + + Gets information about a write stream. + + Returns: + Callable[[~.GetWriteStreamRequest], + Awaitable[~.WriteStream]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "get_write_stream" not in self._stubs: + self._stubs["get_write_stream"] = self.grpc_channel.unary_unary( + "/google.cloud.bigquery.storage.v1.BigQueryWrite/GetWriteStream", + request_serializer=storage.GetWriteStreamRequest.serialize, + response_deserializer=stream.WriteStream.deserialize, + ) + return self._stubs["get_write_stream"] + + @property + def finalize_write_stream( + self, + ) -> Callable[ + [storage.FinalizeWriteStreamRequest], + Awaitable[storage.FinalizeWriteStreamResponse], + ]: + r"""Return a callable for the finalize write stream method over gRPC. + + Finalize a write stream so that no new data can be appended to + the stream. Finalize is not supported on the '_default' stream. + + Returns: + Callable[[~.FinalizeWriteStreamRequest], + Awaitable[~.FinalizeWriteStreamResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "finalize_write_stream" not in self._stubs: + self._stubs["finalize_write_stream"] = self.grpc_channel.unary_unary( + "/google.cloud.bigquery.storage.v1.BigQueryWrite/FinalizeWriteStream", + request_serializer=storage.FinalizeWriteStreamRequest.serialize, + response_deserializer=storage.FinalizeWriteStreamResponse.deserialize, + ) + return self._stubs["finalize_write_stream"] + + @property + def batch_commit_write_streams( + self, + ) -> Callable[ + [storage.BatchCommitWriteStreamsRequest], + Awaitable[storage.BatchCommitWriteStreamsResponse], + ]: + r"""Return a callable for the batch commit write streams method over gRPC. + + Atomically commits a group of ``PENDING`` streams that belong to + the same ``parent`` table. + + Streams must be finalized before commit and cannot be committed + multiple times. Once a stream is committed, data in the stream + becomes available for read operations. + + Returns: + Callable[[~.BatchCommitWriteStreamsRequest], + Awaitable[~.BatchCommitWriteStreamsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "batch_commit_write_streams" not in self._stubs: + self._stubs["batch_commit_write_streams"] = self.grpc_channel.unary_unary( + "/google.cloud.bigquery.storage.v1.BigQueryWrite/BatchCommitWriteStreams", + request_serializer=storage.BatchCommitWriteStreamsRequest.serialize, + response_deserializer=storage.BatchCommitWriteStreamsResponse.deserialize, + ) + return self._stubs["batch_commit_write_streams"] + + @property + def flush_rows( + self, + ) -> Callable[[storage.FlushRowsRequest], Awaitable[storage.FlushRowsResponse]]: + r"""Return a callable for the flush rows method over gRPC. + + Flushes rows to a BUFFERED stream. + + If users are appending rows to BUFFERED stream, flush operation + is required in order for the rows to become available for + reading. A Flush operation flushes up to any previously flushed + offset in a BUFFERED stream, to the offset specified in the + request. + + Flush is not supported on the \_default stream, since it is not + BUFFERED. + + Returns: + Callable[[~.FlushRowsRequest], + Awaitable[~.FlushRowsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "flush_rows" not in self._stubs: + self._stubs["flush_rows"] = self.grpc_channel.unary_unary( + "/google.cloud.bigquery.storage.v1.BigQueryWrite/FlushRows", + request_serializer=storage.FlushRowsRequest.serialize, + response_deserializer=storage.FlushRowsResponse.deserialize, + ) + return self._stubs["flush_rows"] + + +__all__ = ("BigQueryWriteGrpcAsyncIOTransport",) diff --git a/google/cloud/bigquery_storage_v1/types/__init__.py b/google/cloud/bigquery_storage_v1/types/__init__.py index 3e0b90e6..903d2077 100644 --- a/google/cloud/bigquery_storage_v1/types/__init__.py +++ b/google/cloud/bigquery_storage_v1/types/__init__.py @@ -22,12 +22,27 @@ AvroRows, AvroSchema, ) +from .protobuf import ( + ProtoRows, + ProtoSchema, +) from .storage import ( + AppendRowsRequest, + AppendRowsResponse, + BatchCommitWriteStreamsRequest, + BatchCommitWriteStreamsResponse, CreateReadSessionRequest, + CreateWriteStreamRequest, + FinalizeWriteStreamRequest, + FinalizeWriteStreamResponse, + FlushRowsRequest, + FlushRowsResponse, + GetWriteStreamRequest, ReadRowsRequest, ReadRowsResponse, SplitReadStreamRequest, SplitReadStreamResponse, + StorageError, StreamStats, ThrottleState, ) @@ -35,8 +50,13 @@ DataFormat, ReadSession, ReadStream, + WriteStream, DataFormat, ) +from .table import ( + TableFieldSchema, + TableSchema, +) __all__ = ( "ArrowRecordBatch", @@ -44,15 +64,31 @@ "ArrowSerializationOptions", "AvroRows", "AvroSchema", + "ProtoRows", + "ProtoSchema", + "AppendRowsRequest", + "AppendRowsResponse", + "BatchCommitWriteStreamsRequest", + "BatchCommitWriteStreamsResponse", "CreateReadSessionRequest", + "CreateWriteStreamRequest", + "FinalizeWriteStreamRequest", + "FinalizeWriteStreamResponse", + "FlushRowsRequest", + "FlushRowsResponse", + "GetWriteStreamRequest", "ReadRowsRequest", "ReadRowsResponse", "SplitReadStreamRequest", "SplitReadStreamResponse", + "StorageError", "StreamStats", "ThrottleState", "DataFormat", "ReadSession", "ReadStream", + "WriteStream", "DataFormat", + "TableFieldSchema", + "TableSchema", ) diff --git a/google/cloud/bigquery_storage_v1/types/protobuf.py b/google/cloud/bigquery_storage_v1/types/protobuf.py new file mode 100644 index 00000000..c5dab2d3 --- /dev/null +++ b/google/cloud/bigquery_storage_v1/types/protobuf.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import proto # type: ignore + +from google.protobuf import descriptor_pb2 # type: ignore + + +__protobuf__ = proto.module( + package="google.cloud.bigquery.storage.v1", manifest={"ProtoSchema", "ProtoRows",}, +) + + +class ProtoSchema(proto.Message): + r"""ProtoSchema describes the schema of the serialized protocol + buffer data rows. + + Attributes: + proto_descriptor (google.protobuf.descriptor_pb2.DescriptorProto): + Descriptor for input message. The provided descriptor must + be self contained, such that data rows sent can be fully + decoded using only the single descriptor. For data rows that + are compositions of multiple independent messages, this + means the descriptor may need to be transformed to only use + nested types: + https://developers.google.com/protocol-buffers/docs/proto#nested + + For additional information for how proto types and values + map onto BigQuery see: + https://cloud.google.com/bigquery/docs/write-api#data_type_conversions + """ + + proto_descriptor = proto.Field( + proto.MESSAGE, number=1, message=descriptor_pb2.DescriptorProto, + ) + + +class ProtoRows(proto.Message): + r""" + Attributes: + serialized_rows (Sequence[bytes]): + A sequence of rows serialized as a Protocol + Buffer. + See https://developers.google.com/protocol- + buffers/docs/overview for more information on + deserializing this field. + """ + + serialized_rows = proto.RepeatedField(proto.BYTES, number=1,) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/bigquery_storage_v1/types/storage.py b/google/cloud/bigquery_storage_v1/types/storage.py index b5a3ffca..8299a175 100644 --- a/google/cloud/bigquery_storage_v1/types/storage.py +++ b/google/cloud/bigquery_storage_v1/types/storage.py @@ -17,7 +17,12 @@ from google.cloud.bigquery_storage_v1.types import arrow from google.cloud.bigquery_storage_v1.types import avro +from google.cloud.bigquery_storage_v1.types import protobuf from google.cloud.bigquery_storage_v1.types import stream +from google.cloud.bigquery_storage_v1.types import table +from google.protobuf import timestamp_pb2 # type: ignore +from google.protobuf import wrappers_pb2 # type: ignore +from google.rpc import status_pb2 # type: ignore __protobuf__ = proto.module( @@ -30,6 +35,17 @@ "ReadRowsResponse", "SplitReadStreamRequest", "SplitReadStreamResponse", + "CreateWriteStreamRequest", + "AppendRowsRequest", + "AppendRowsResponse", + "GetWriteStreamRequest", + "BatchCommitWriteStreamsRequest", + "BatchCommitWriteStreamsResponse", + "FinalizeWriteStreamRequest", + "FinalizeWriteStreamResponse", + "FlushRowsRequest", + "FlushRowsResponse", + "StorageError", }, ) @@ -90,7 +106,7 @@ class ThrottleState(proto.Message): class StreamStats(proto.Message): - r"""Estimated stream statistics for a given Stream. + r"""Estimated stream statistics for a given read Stream. Attributes: progress (google.cloud.bigquery_storage_v1.types.StreamStats.Progress): Represents the progress of the current @@ -206,4 +222,269 @@ class SplitReadStreamResponse(proto.Message): remainder_stream = proto.Field(proto.MESSAGE, number=2, message=stream.ReadStream,) +class CreateWriteStreamRequest(proto.Message): + r"""Request message for ``CreateWriteStream``. + Attributes: + parent (str): + Required. Reference to the table to which the stream + belongs, in the format of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + write_stream (google.cloud.bigquery_storage_v1.types.WriteStream): + Required. Stream to be created. + """ + + parent = proto.Field(proto.STRING, number=1,) + write_stream = proto.Field(proto.MESSAGE, number=2, message=stream.WriteStream,) + + +class AppendRowsRequest(proto.Message): + r"""Request message for ``AppendRows``. + + Due to the nature of AppendRows being a bidirectional streaming RPC, + certain parts of the AppendRowsRequest need only be specified for + the first request sent each time the gRPC network connection is + opened/reopened. + + Attributes: + write_stream (str): + Required. The write_stream identifies the target of the + append operation, and only needs to be specified as part of + the first request on the gRPC connection. If provided for + subsequent requests, it must match the value of the first + request. + + For explicitly created write streams, the format is: + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{id}`` + + For the special default stream, the format is: + ``projects/{project}/datasets/{dataset}/tables/{table}/_default``. + offset (google.protobuf.wrappers_pb2.Int64Value): + If present, the write is only performed if the next append + offset is same as the provided value. If not present, the + write is performed at the current end of stream. Specifying + a value for this field is not allowed when calling + AppendRows for the '_default' stream. + proto_rows (google.cloud.bigquery_storage_v1.types.AppendRowsRequest.ProtoData): + Rows in proto format. + trace_id (str): + Id set by client to annotate its identity. + Only initial request setting is respected. + """ + + class ProtoData(proto.Message): + r"""ProtoData contains the data rows and schema when constructing + append requests. + + Attributes: + writer_schema (google.cloud.bigquery_storage_v1.types.ProtoSchema): + Proto schema used to serialize the data. + This value only needs to be provided as part of + the first request on a gRPC network connection, + and will be ignored for subsequent requests on + the connection. + rows (google.cloud.bigquery_storage_v1.types.ProtoRows): + Serialized row data in protobuf message + format. Currently, the backend expects the + serialized rows to adhere to proto2 semantics + when appending rows, particularly with respect + to how default values are encoded. + """ + + writer_schema = proto.Field( + proto.MESSAGE, number=1, message=protobuf.ProtoSchema, + ) + rows = proto.Field(proto.MESSAGE, number=2, message=protobuf.ProtoRows,) + + write_stream = proto.Field(proto.STRING, number=1,) + offset = proto.Field(proto.MESSAGE, number=2, message=wrappers_pb2.Int64Value,) + proto_rows = proto.Field(proto.MESSAGE, number=4, oneof="rows", message=ProtoData,) + trace_id = proto.Field(proto.STRING, number=6,) + + +class AppendRowsResponse(proto.Message): + r"""Response message for ``AppendRows``. + Attributes: + append_result (google.cloud.bigquery_storage_v1.types.AppendRowsResponse.AppendResult): + Result if the append is successful. + error (google.rpc.status_pb2.Status): + Error returned when problems were encountered. If present, + it indicates rows were not accepted into the system. Users + can retry or continue with other append requests within the + same connection. + + Additional information about error signalling: + + ALREADY_EXISTS: Happens when an append specified an offset, + and the backend already has received data at this offset. + Typically encountered in retry scenarios, and can be + ignored. + + OUT_OF_RANGE: Returned when the specified offset in the + stream is beyond the current end of the stream. + + INVALID_ARGUMENT: Indicates a malformed request or data. + + ABORTED: Request processing is aborted because of prior + failures. The request can be retried if previous failure is + addressed. + + INTERNAL: Indicates server side error(s) that can be + retried. + updated_schema (google.cloud.bigquery_storage_v1.types.TableSchema): + If backend detects a schema update, pass it + to user so that user can use it to input new + type of message. It will be empty when no schema + updates have occurred. + """ + + class AppendResult(proto.Message): + r"""AppendResult is returned for successful append requests. + Attributes: + offset (google.protobuf.wrappers_pb2.Int64Value): + The row offset at which the last append + occurred. The offset will not be set if + appending using default streams. + """ + + offset = proto.Field(proto.MESSAGE, number=1, message=wrappers_pb2.Int64Value,) + + append_result = proto.Field( + proto.MESSAGE, number=1, oneof="response", message=AppendResult, + ) + error = proto.Field( + proto.MESSAGE, number=2, oneof="response", message=status_pb2.Status, + ) + updated_schema = proto.Field(proto.MESSAGE, number=3, message=table.TableSchema,) + + +class GetWriteStreamRequest(proto.Message): + r"""Request message for ``GetWriteStreamRequest``. + Attributes: + name (str): + Required. Name of the stream to get, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + """ + + name = proto.Field(proto.STRING, number=1,) + + +class BatchCommitWriteStreamsRequest(proto.Message): + r"""Request message for ``BatchCommitWriteStreams``. + Attributes: + parent (str): + Required. Parent table that all the streams should belong + to, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + write_streams (Sequence[str]): + Required. The group of streams that will be + committed atomically. + """ + + parent = proto.Field(proto.STRING, number=1,) + write_streams = proto.RepeatedField(proto.STRING, number=2,) + + +class BatchCommitWriteStreamsResponse(proto.Message): + r"""Response message for ``BatchCommitWriteStreams``. + Attributes: + commit_time (google.protobuf.timestamp_pb2.Timestamp): + The time at which streams were committed in microseconds + granularity. This field will only exist when there are no + stream errors. **Note** if this field is not set, it means + the commit was not successful. + stream_errors (Sequence[google.cloud.bigquery_storage_v1.types.StorageError]): + Stream level error if commit failed. Only + streams with error will be in the list. + If empty, there is no error and all streams are + committed successfully. If non empty, certain + streams have errors and ZERO stream is committed + due to atomicity guarantee. + """ + + commit_time = proto.Field(proto.MESSAGE, number=1, message=timestamp_pb2.Timestamp,) + stream_errors = proto.RepeatedField( + proto.MESSAGE, number=2, message="StorageError", + ) + + +class FinalizeWriteStreamRequest(proto.Message): + r"""Request message for invoking ``FinalizeWriteStream``. + Attributes: + name (str): + Required. Name of the stream to finalize, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + """ + + name = proto.Field(proto.STRING, number=1,) + + +class FinalizeWriteStreamResponse(proto.Message): + r"""Response message for ``FinalizeWriteStream``. + Attributes: + row_count (int): + Number of rows in the finalized stream. + """ + + row_count = proto.Field(proto.INT64, number=1,) + + +class FlushRowsRequest(proto.Message): + r"""Request message for ``FlushRows``. + Attributes: + write_stream (str): + Required. The stream that is the target of + the flush operation. + offset (google.protobuf.wrappers_pb2.Int64Value): + Ending offset of the flush operation. Rows + before this offset(including this offset) will + be flushed. + """ + + write_stream = proto.Field(proto.STRING, number=1,) + offset = proto.Field(proto.MESSAGE, number=2, message=wrappers_pb2.Int64Value,) + + +class FlushRowsResponse(proto.Message): + r"""Respond message for ``FlushRows``. + Attributes: + offset (int): + The rows before this offset (including this + offset) are flushed. + """ + + offset = proto.Field(proto.INT64, number=1,) + + +class StorageError(proto.Message): + r"""Structured custom BigQuery Storage error message. The error + can be attached as error details in the returned rpc Status. In + particular, the use of error codes allows more structured error + handling, and reduces the need to evaluate unstructured error + text strings. + + Attributes: + code (google.cloud.bigquery_storage_v1.types.StorageError.StorageErrorCode): + BigQuery Storage specific error code. + entity (str): + Name of the failed entity. + error_message (str): + Message that describes the error. + """ + + class StorageErrorCode(proto.Enum): + r"""Error code for ``StorageError``.""" + STORAGE_ERROR_CODE_UNSPECIFIED = 0 + TABLE_NOT_FOUND = 1 + STREAM_ALREADY_COMMITTED = 2 + STREAM_NOT_FOUND = 3 + INVALID_STREAM_TYPE = 4 + INVALID_STREAM_STATE = 5 + STREAM_FINALIZED = 6 + SCHEMA_MISMATCH_EXTRA_FIELDS = 7 + + code = proto.Field(proto.ENUM, number=1, enum=StorageErrorCode,) + entity = proto.Field(proto.STRING, number=2,) + error_message = proto.Field(proto.STRING, number=3,) + + __all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/bigquery_storage_v1/types/stream.py b/google/cloud/bigquery_storage_v1/types/stream.py index 1117dbc4..a4311e3a 100644 --- a/google/cloud/bigquery_storage_v1/types/stream.py +++ b/google/cloud/bigquery_storage_v1/types/stream.py @@ -17,12 +17,13 @@ from google.cloud.bigquery_storage_v1.types import arrow from google.cloud.bigquery_storage_v1.types import avro +from google.cloud.bigquery_storage_v1.types import table as gcbs_table from google.protobuf import timestamp_pb2 # type: ignore __protobuf__ = proto.module( package="google.cloud.bigquery.storage.v1", - manifest={"DataFormat", "ReadSession", "ReadStream",}, + manifest={"DataFormat", "ReadSession", "ReadStream", "WriteStream",}, ) @@ -68,6 +69,12 @@ class ReadSession(proto.Message): this list being unpopulated, in that case, the user will need to use a List method to get the streams instead, which is not yet available. + estimated_total_bytes_scanned (int): + Output only. An estimate on the number of + bytes this session will scan when all streams + are completely consumed. This estimate is based + on metadata from the table which might be + incomplete or stale. """ class TableModifiers(proto.Message): @@ -128,6 +135,7 @@ class TableReadOptions(proto.Message): table_modifiers = proto.Field(proto.MESSAGE, number=7, message=TableModifiers,) read_options = proto.Field(proto.MESSAGE, number=8, message=TableReadOptions,) streams = proto.RepeatedField(proto.MESSAGE, number=10, message="ReadStream",) + estimated_total_bytes_scanned = proto.Field(proto.INT64, number=12,) class ReadStream(proto.Message): @@ -144,4 +152,44 @@ class ReadStream(proto.Message): name = proto.Field(proto.STRING, number=1,) +class WriteStream(proto.Message): + r"""Information about a single stream that gets data inside the + storage system. + + Attributes: + name (str): + Output only. Name of the stream, in the form + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + type_ (google.cloud.bigquery_storage_v1.types.WriteStream.Type): + Immutable. Type of the stream. + create_time (google.protobuf.timestamp_pb2.Timestamp): + Output only. Create time of the stream. For the \_default + stream, this is the creation_time of the table. + commit_time (google.protobuf.timestamp_pb2.Timestamp): + Output only. Commit time of the stream. If a stream is of + ``COMMITTED`` type, then it will have a commit_time same as + ``create_time``. If the stream is of ``PENDING`` type, empty + commit_time means it is not committed. + table_schema (google.cloud.bigquery_storage_v1.types.TableSchema): + Output only. The schema of the destination table. It is only + returned in ``CreateWriteStream`` response. Caller should + generate data that's compatible with this schema to send in + initial ``AppendRowsRequest``. The table schema could go out + of date during the life time of the stream. + """ + + class Type(proto.Enum): + r"""Type enum of the stream.""" + TYPE_UNSPECIFIED = 0 + COMMITTED = 1 + PENDING = 2 + BUFFERED = 3 + + name = proto.Field(proto.STRING, number=1,) + type_ = proto.Field(proto.ENUM, number=2, enum=Type,) + create_time = proto.Field(proto.MESSAGE, number=3, message=timestamp_pb2.Timestamp,) + commit_time = proto.Field(proto.MESSAGE, number=4, message=timestamp_pb2.Timestamp,) + table_schema = proto.Field(proto.MESSAGE, number=5, message=gcbs_table.TableSchema,) + + __all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/bigquery_storage_v1/types/table.py b/google/cloud/bigquery_storage_v1/types/table.py new file mode 100644 index 00000000..295f3b50 --- /dev/null +++ b/google/cloud/bigquery_storage_v1/types/table.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import proto # type: ignore + + +__protobuf__ = proto.module( + package="google.cloud.bigquery.storage.v1", + manifest={"TableSchema", "TableFieldSchema",}, +) + + +class TableSchema(proto.Message): + r"""Schema of a table. + Attributes: + fields (Sequence[google.cloud.bigquery_storage_v1.types.TableFieldSchema]): + Describes the fields in a table. + """ + + fields = proto.RepeatedField(proto.MESSAGE, number=1, message="TableFieldSchema",) + + +class TableFieldSchema(proto.Message): + r"""TableFieldSchema defines a single field/column within a table + schema. + + Attributes: + name (str): + Required. The field name. The name must contain only letters + (a-z, A-Z), numbers (0-9), or underscores (_), and must + start with a letter or underscore. The maximum length is 128 + characters. + type_ (google.cloud.bigquery_storage_v1.types.TableFieldSchema.Type): + Required. The field data type. + mode (google.cloud.bigquery_storage_v1.types.TableFieldSchema.Mode): + Optional. The field mode. The default value + is NULLABLE. + fields (Sequence[google.cloud.bigquery_storage_v1.types.TableFieldSchema]): + Optional. Describes the nested schema fields + if the type property is set to STRUCT. + description (str): + Optional. The field description. The maximum + length is 1,024 characters. + max_length (int): + Optional. Maximum length of values of this field for STRINGS + or BYTES. + + If max_length is not specified, no maximum length constraint + is imposed on this field. + + If type = "STRING", then max_length represents the maximum + UTF-8 length of strings in this field. + + If type = "BYTES", then max_length represents the maximum + number of bytes in this field. + + It is invalid to set this field if type is not "STRING" or + "BYTES". + precision (int): + Optional. Precision (maximum number of total digits in base + 10) and scale (maximum number of digits in the fractional + part in base 10) constraints for values of this field for + NUMERIC or BIGNUMERIC. + + It is invalid to set precision or scale if type is not + "NUMERIC" or "BIGNUMERIC". + + If precision and scale are not specified, no value range + constraint is imposed on this field insofar as values are + permitted by the type. + + Values of this NUMERIC or BIGNUMERIC field must be in this + range when: + + - Precision (P) and scale (S) are specified: [-10^(P-S) + + 10^(-S), 10^(P-S) - 10^(-S)] + - Precision (P) is specified but not scale (and thus scale + is interpreted to be equal to zero): [-10^P + 1, 10^P - + 1]. + + Acceptable values for precision and scale if both are + specified: + + - If type = "NUMERIC": 1 <= precision - scale <= 29 and 0 + <= scale <= 9. + - If type = "BIGNUMERIC": 1 <= precision - scale <= 38 and + 0 <= scale <= 38. + + Acceptable values for precision if only precision is + specified but not scale (and thus scale is interpreted to be + equal to zero): + + - If type = "NUMERIC": 1 <= precision <= 29. + - If type = "BIGNUMERIC": 1 <= precision <= 38. + + If scale is specified but not precision, then it is invalid. + scale (int): + Optional. See documentation for precision. + """ + + class Type(proto.Enum): + r"""""" + TYPE_UNSPECIFIED = 0 + STRING = 1 + INT64 = 2 + DOUBLE = 3 + STRUCT = 4 + BYTES = 5 + BOOL = 6 + TIMESTAMP = 7 + DATE = 8 + TIME = 9 + DATETIME = 10 + GEOGRAPHY = 11 + NUMERIC = 12 + BIGNUMERIC = 13 + INTERVAL = 14 + JSON = 15 + + class Mode(proto.Enum): + r"""""" + MODE_UNSPECIFIED = 0 + NULLABLE = 1 + REQUIRED = 2 + REPEATED = 3 + + name = proto.Field(proto.STRING, number=1,) + type_ = proto.Field(proto.ENUM, number=2, enum=Type,) + mode = proto.Field(proto.ENUM, number=3, enum=Mode,) + fields = proto.RepeatedField(proto.MESSAGE, number=4, message="TableFieldSchema",) + description = proto.Field(proto.STRING, number=6,) + max_length = proto.Field(proto.INT64, number=7,) + precision = proto.Field(proto.INT64, number=8,) + scale = proto.Field(proto.INT64, number=9,) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/scripts/fixup_bigquery_storage_v1_keywords.py b/scripts/fixup_bigquery_storage_v1_keywords.py index 107af58b..9ba710dc 100644 --- a/scripts/fixup_bigquery_storage_v1_keywords.py +++ b/scripts/fixup_bigquery_storage_v1_keywords.py @@ -39,7 +39,13 @@ def partition( class bigquery_storageCallTransformer(cst.CSTTransformer): CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { + 'append_rows': ('write_stream', 'offset', 'proto_rows', 'trace_id', ), + 'batch_commit_write_streams': ('parent', 'write_streams', ), 'create_read_session': ('parent', 'read_session', 'max_stream_count', ), + 'create_write_stream': ('parent', 'write_stream', ), + 'finalize_write_stream': ('name', ), + 'flush_rows': ('write_stream', 'offset', ), + 'get_write_stream': ('name', ), 'read_rows': ('read_stream', 'offset', ), 'split_read_stream': ('name', 'fraction', ), } diff --git a/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py b/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py index a4d56468..9d2ee3cb 100644 --- a/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py +++ b/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py @@ -506,6 +506,7 @@ def test_create_read_session( name="name_value", data_format=stream.DataFormat.AVRO, table="table_value", + estimated_total_bytes_scanned=3076, avro_schema=avro.AvroSchema(schema="schema_value"), ) response = client.create_read_session(request) @@ -520,6 +521,7 @@ def test_create_read_session( assert response.name == "name_value" assert response.data_format == stream.DataFormat.AVRO assert response.table == "table_value" + assert response.estimated_total_bytes_scanned == 3076 def test_create_read_session_from_dict(): @@ -565,6 +567,7 @@ async def test_create_read_session_async( name="name_value", data_format=stream.DataFormat.AVRO, table="table_value", + estimated_total_bytes_scanned=3076, ) ) response = await client.create_read_session(request) @@ -579,6 +582,7 @@ async def test_create_read_session_async( assert response.name == "name_value" assert response.data_format == stream.DataFormat.AVRO assert response.table == "table_value" + assert response.estimated_total_bytes_scanned == 3076 @pytest.mark.asyncio diff --git a/tests/unit/gapic/bigquery_storage_v1/test_big_query_write.py b/tests/unit/gapic/bigquery_storage_v1/test_big_query_write.py new file mode 100644 index 00000000..28d36177 --- /dev/null +++ b/tests/unit/gapic/bigquery_storage_v1/test_big_query_write.py @@ -0,0 +1,2283 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import mock +import packaging.version + +import grpc +from grpc.experimental import aio +import math +import pytest +from proto.marshal.rules.dates import DurationRule, TimestampRule + + +from google.api_core import client_options +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import grpc_helpers +from google.api_core import grpc_helpers_async +from google.auth import credentials as ga_credentials +from google.auth.exceptions import MutualTLSChannelError +from google.cloud.bigquery_storage_v1.services.big_query_write import ( + BigQueryWriteAsyncClient, +) +from google.cloud.bigquery_storage_v1.services.big_query_write import ( + BigQueryWriteClient, +) +from google.cloud.bigquery_storage_v1.services.big_query_write import transports +from google.cloud.bigquery_storage_v1.services.big_query_write.transports.base import ( + _GOOGLE_AUTH_VERSION, +) +from google.cloud.bigquery_storage_v1.types import protobuf +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from google.cloud.bigquery_storage_v1.types import table +from google.oauth2 import service_account +from google.protobuf import descriptor_pb2 # type: ignore +from google.protobuf import timestamp_pb2 # type: ignore +from google.protobuf import wrappers_pb2 # type: ignore +from google.rpc import status_pb2 # type: ignore +import google.auth + + +# TODO(busunkim): Once google-auth >= 1.25.0 is required transitively +# through google-api-core: +# - Delete the auth "less than" test cases +# - Delete these pytest markers (Make the "greater than or equal to" tests the default). +requires_google_auth_lt_1_25_0 = pytest.mark.skipif( + packaging.version.parse(_GOOGLE_AUTH_VERSION) >= packaging.version.parse("1.25.0"), + reason="This test requires google-auth < 1.25.0", +) +requires_google_auth_gte_1_25_0 = pytest.mark.skipif( + packaging.version.parse(_GOOGLE_AUTH_VERSION) < packaging.version.parse("1.25.0"), + reason="This test requires google-auth >= 1.25.0", +) + + +def client_cert_source_callback(): + return b"cert bytes", b"key bytes" + + +# If default endpoint is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint(client): + return ( + "foo.googleapis.com" + if ("localhost" in client.DEFAULT_ENDPOINT) + else client.DEFAULT_ENDPOINT + ) + + +def test__get_default_mtls_endpoint(): + api_endpoint = "example.googleapis.com" + api_mtls_endpoint = "example.mtls.googleapis.com" + sandbox_endpoint = "example.sandbox.googleapis.com" + sandbox_mtls_endpoint = "example.mtls.sandbox.googleapis.com" + non_googleapi = "api.example.com" + + assert BigQueryWriteClient._get_default_mtls_endpoint(None) is None + assert ( + BigQueryWriteClient._get_default_mtls_endpoint(api_endpoint) + == api_mtls_endpoint + ) + assert ( + BigQueryWriteClient._get_default_mtls_endpoint(api_mtls_endpoint) + == api_mtls_endpoint + ) + assert ( + BigQueryWriteClient._get_default_mtls_endpoint(sandbox_endpoint) + == sandbox_mtls_endpoint + ) + assert ( + BigQueryWriteClient._get_default_mtls_endpoint(sandbox_mtls_endpoint) + == sandbox_mtls_endpoint + ) + assert ( + BigQueryWriteClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi + ) + + +@pytest.mark.parametrize( + "client_class", [BigQueryWriteClient, BigQueryWriteAsyncClient,] +) +def test_big_query_write_client_from_service_account_info(client_class): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object( + service_account.Credentials, "from_service_account_info" + ) as factory: + factory.return_value = creds + info = {"valid": True} + client = client_class.from_service_account_info(info) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == "bigquerystorage.googleapis.com:443" + + +@pytest.mark.parametrize( + "transport_class,transport_name", + [ + (transports.BigQueryWriteGrpcTransport, "grpc"), + (transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio"), + ], +) +def test_big_query_write_client_service_account_always_use_jwt( + transport_class, transport_name +): + with mock.patch.object( + service_account.Credentials, "with_always_use_jwt_access", create=True + ) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=True) + use_jwt.assert_called_once_with(True) + + with mock.patch.object( + service_account.Credentials, "with_always_use_jwt_access", create=True + ) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=False) + use_jwt.assert_not_called() + + +@pytest.mark.parametrize( + "client_class", [BigQueryWriteClient, BigQueryWriteAsyncClient,] +) +def test_big_query_write_client_from_service_account_file(client_class): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object( + service_account.Credentials, "from_service_account_file" + ) as factory: + factory.return_value = creds + client = client_class.from_service_account_file("dummy/file/path.json") + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + client = client_class.from_service_account_json("dummy/file/path.json") + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == "bigquerystorage.googleapis.com:443" + + +def test_big_query_write_client_get_transport_class(): + transport = BigQueryWriteClient.get_transport_class() + available_transports = [ + transports.BigQueryWriteGrpcTransport, + ] + assert transport in available_transports + + transport = BigQueryWriteClient.get_transport_class("grpc") + assert transport == transports.BigQueryWriteGrpcTransport + + +@pytest.mark.parametrize( + "client_class,transport_class,transport_name", + [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc"), + ( + BigQueryWriteAsyncClient, + transports.BigQueryWriteGrpcAsyncIOTransport, + "grpc_asyncio", + ), + ], +) +@mock.patch.object( + BigQueryWriteClient, + "DEFAULT_ENDPOINT", + modify_default_endpoint(BigQueryWriteClient), +) +@mock.patch.object( + BigQueryWriteAsyncClient, + "DEFAULT_ENDPOINT", + modify_default_endpoint(BigQueryWriteAsyncClient), +) +def test_big_query_write_client_client_options( + client_class, transport_class, transport_name +): + # Check that if channel is provided we won't create a new one. + with mock.patch.object(BigQueryWriteClient, "get_transport_class") as gtc: + transport = transport_class(credentials=ga_credentials.AnonymousCredentials()) + client = client_class(transport=transport) + gtc.assert_not_called() + + # Check that if channel is provided via str we will create a new one. + with mock.patch.object(BigQueryWriteClient, "get_transport_class") as gtc: + client = client_class(transport=transport_name) + gtc.assert_called() + + # Check the case api_endpoint is provided. + options = client_options.ClientOptions(api_endpoint="squid.clam.whelk") + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_MTLS_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError): + client = client_class() + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"} + ): + with pytest.raises(ValueError): + client = client_class() + + # Check the case quota_project_id is provided + options = client_options.ClientOptions(quota_project_id="octopus") + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id="octopus", + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +@pytest.mark.parametrize( + "client_class,transport_class,transport_name,use_client_cert_env", + [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc", "true"), + ( + BigQueryWriteAsyncClient, + transports.BigQueryWriteGrpcAsyncIOTransport, + "grpc_asyncio", + "true", + ), + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc", "false"), + ( + BigQueryWriteAsyncClient, + transports.BigQueryWriteGrpcAsyncIOTransport, + "grpc_asyncio", + "false", + ), + ], +) +@mock.patch.object( + BigQueryWriteClient, + "DEFAULT_ENDPOINT", + modify_default_endpoint(BigQueryWriteClient), +) +@mock.patch.object( + BigQueryWriteAsyncClient, + "DEFAULT_ENDPOINT", + modify_default_endpoint(BigQueryWriteAsyncClient), +) +@mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) +def test_big_query_write_client_mtls_env_auto( + client_class, transport_class, transport_name, use_client_cert_env +): + # This tests the endpoint autoswitch behavior. Endpoint is autoswitched to the default + # mtls endpoint, if GOOGLE_API_USE_CLIENT_CERTIFICATE is "true" and client cert exists. + + # Check the case client_cert_source is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env} + ): + options = client_options.ClientOptions( + client_cert_source=client_cert_source_callback + ) + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + + if use_client_cert_env == "false": + expected_client_cert_source = None + expected_host = client.DEFAULT_ENDPOINT + else: + expected_client_cert_source = client_cert_source_callback + expected_host = client.DEFAULT_MTLS_ENDPOINT + + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case ADC client cert is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env} + ): + with mock.patch.object(transport_class, "__init__") as patched: + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=True, + ): + with mock.patch( + "google.auth.transport.mtls.default_client_cert_source", + return_value=client_cert_source_callback, + ): + if use_client_cert_env == "false": + expected_host = client.DEFAULT_ENDPOINT + expected_client_cert_source = None + else: + expected_host = client.DEFAULT_MTLS_ENDPOINT + expected_client_cert_source = client_cert_source_callback + + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # Check the case client_cert_source and ADC client cert are not provided. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env} + ): + with mock.patch.object(transport_class, "__init__") as patched: + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=False, + ): + patched.return_value = None + client = client_class() + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +@pytest.mark.parametrize( + "client_class,transport_class,transport_name", + [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc"), + ( + BigQueryWriteAsyncClient, + transports.BigQueryWriteGrpcAsyncIOTransport, + "grpc_asyncio", + ), + ], +) +def test_big_query_write_client_client_options_scopes( + client_class, transport_class, transport_name +): + # Check the case scopes are provided. + options = client_options.ClientOptions(scopes=["1", "2"],) + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=["1", "2"], + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +@pytest.mark.parametrize( + "client_class,transport_class,transport_name", + [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc"), + ( + BigQueryWriteAsyncClient, + transports.BigQueryWriteGrpcAsyncIOTransport, + "grpc_asyncio", + ), + ], +) +def test_big_query_write_client_client_options_credentials_file( + client_class, transport_class, transport_name +): + # Check the case credentials file is provided. + options = client_options.ClientOptions(credentials_file="credentials.json") + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +def test_big_query_write_client_client_options_from_dict(): + with mock.patch( + "google.cloud.bigquery_storage_v1.services.big_query_write.transports.BigQueryWriteGrpcTransport.__init__" + ) as grpc_transport: + grpc_transport.return_value = None + client = BigQueryWriteClient( + client_options={"api_endpoint": "squid.clam.whelk"} + ) + grpc_transport.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + +def test_create_write_stream( + transport: str = "grpc", request_type=storage.CreateWriteStreamRequest +): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), "__call__" + ) as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream( + name="name_value", type_=stream.WriteStream.Type.COMMITTED, + ) + response = client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == storage.CreateWriteStreamRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.WriteStream) + assert response.name == "name_value" + assert response.type_ == stream.WriteStream.Type.COMMITTED + + +def test_create_write_stream_from_dict(): + test_create_write_stream(request_type=dict) + + +def test_create_write_stream_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), "__call__" + ) as call: + client.create_write_stream() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.CreateWriteStreamRequest() + + +@pytest.mark.asyncio +async def test_create_write_stream_async( + transport: str = "grpc_asyncio", request_type=storage.CreateWriteStreamRequest +): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), "__call__" + ) as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + stream.WriteStream( + name="name_value", type_=stream.WriteStream.Type.COMMITTED, + ) + ) + response = await client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == storage.CreateWriteStreamRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.WriteStream) + assert response.name == "name_value" + assert response.type_ == stream.WriteStream.Type.COMMITTED + + +@pytest.mark.asyncio +async def test_create_write_stream_async_from_dict(): + await test_create_write_stream_async(request_type=dict) + + +def test_create_write_stream_field_headers(): + client = BigQueryWriteClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.CreateWriteStreamRequest() + + request.parent = "parent/value" + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), "__call__" + ) as call: + call.return_value = stream.WriteStream() + client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ("x-goog-request-params", "parent=parent/value",) in kw["metadata"] + + +@pytest.mark.asyncio +async def test_create_write_stream_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.CreateWriteStreamRequest() + + request.parent = "parent/value" + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), "__call__" + ) as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) + await client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ("x-goog-request-params", "parent=parent/value",) in kw["metadata"] + + +def test_create_write_stream_flattened(): + client = BigQueryWriteClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), "__call__" + ) as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.create_write_stream( + parent="parent_value", write_stream=stream.WriteStream(name="name_value"), + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0].parent == "parent_value" + assert args[0].write_stream == stream.WriteStream(name="name_value") + + +def test_create_write_stream_flattened_error(): + client = BigQueryWriteClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.create_write_stream( + storage.CreateWriteStreamRequest(), + parent="parent_value", + write_stream=stream.WriteStream(name="name_value"), + ) + + +@pytest.mark.asyncio +async def test_create_write_stream_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), "__call__" + ) as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.create_write_stream( + parent="parent_value", write_stream=stream.WriteStream(name="name_value"), + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0].parent == "parent_value" + assert args[0].write_stream == stream.WriteStream(name="name_value") + + +@pytest.mark.asyncio +async def test_create_write_stream_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.create_write_stream( + storage.CreateWriteStreamRequest(), + parent="parent_value", + write_stream=stream.WriteStream(name="name_value"), + ) + + +def test_append_rows(transport: str = "grpc", request_type=storage.AppendRowsRequest): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + requests = [request] + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.append_rows), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.AppendRowsResponse()]) + response = client.append_rows(iter(requests)) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert next(args[0]) == request + + # Establish that the response is the type that we expect. + for message in response: + assert isinstance(message, storage.AppendRowsResponse) + + +def test_append_rows_from_dict(): + test_append_rows(request_type=dict) + + +@pytest.mark.asyncio +async def test_append_rows_async( + transport: str = "grpc_asyncio", request_type=storage.AppendRowsRequest +): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + requests = [request] + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.append_rows), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = mock.Mock(aio.StreamStreamCall, autospec=True) + call.return_value.read = mock.AsyncMock( + side_effect=[storage.AppendRowsResponse()] + ) + response = await client.append_rows(iter(requests)) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert next(args[0]) == request + + # Establish that the response is the type that we expect. + message = await response.read() + assert isinstance(message, storage.AppendRowsResponse) + + +@pytest.mark.asyncio +async def test_append_rows_async_from_dict(): + await test_append_rows_async(request_type=dict) + + +def test_get_write_stream( + transport: str = "grpc", request_type=storage.GetWriteStreamRequest +): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.get_write_stream), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream( + name="name_value", type_=stream.WriteStream.Type.COMMITTED, + ) + response = client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == storage.GetWriteStreamRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.WriteStream) + assert response.name == "name_value" + assert response.type_ == stream.WriteStream.Type.COMMITTED + + +def test_get_write_stream_from_dict(): + test_get_write_stream(request_type=dict) + + +def test_get_write_stream_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.get_write_stream), "__call__") as call: + client.get_write_stream() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.GetWriteStreamRequest() + + +@pytest.mark.asyncio +async def test_get_write_stream_async( + transport: str = "grpc_asyncio", request_type=storage.GetWriteStreamRequest +): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.get_write_stream), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + stream.WriteStream( + name="name_value", type_=stream.WriteStream.Type.COMMITTED, + ) + ) + response = await client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == storage.GetWriteStreamRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.WriteStream) + assert response.name == "name_value" + assert response.type_ == stream.WriteStream.Type.COMMITTED + + +@pytest.mark.asyncio +async def test_get_write_stream_async_from_dict(): + await test_get_write_stream_async(request_type=dict) + + +def test_get_write_stream_field_headers(): + client = BigQueryWriteClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.GetWriteStreamRequest() + + request.name = "name/value" + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.get_write_stream), "__call__") as call: + call.return_value = stream.WriteStream() + client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ("x-goog-request-params", "name=name/value",) in kw["metadata"] + + +@pytest.mark.asyncio +async def test_get_write_stream_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.GetWriteStreamRequest() + + request.name = "name/value" + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.get_write_stream), "__call__") as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) + await client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ("x-goog-request-params", "name=name/value",) in kw["metadata"] + + +def test_get_write_stream_flattened(): + client = BigQueryWriteClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.get_write_stream), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.get_write_stream(name="name_value",) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0].name == "name_value" + + +def test_get_write_stream_flattened_error(): + client = BigQueryWriteClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.get_write_stream( + storage.GetWriteStreamRequest(), name="name_value", + ) + + +@pytest.mark.asyncio +async def test_get_write_stream_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.get_write_stream), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.get_write_stream(name="name_value",) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0].name == "name_value" + + +@pytest.mark.asyncio +async def test_get_write_stream_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.get_write_stream( + storage.GetWriteStreamRequest(), name="name_value", + ) + + +def test_finalize_write_stream( + transport: str = "grpc", request_type=storage.FinalizeWriteStreamRequest +): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), "__call__" + ) as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FinalizeWriteStreamResponse(row_count=992,) + response = client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == storage.FinalizeWriteStreamRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.FinalizeWriteStreamResponse) + assert response.row_count == 992 + + +def test_finalize_write_stream_from_dict(): + test_finalize_write_stream(request_type=dict) + + +def test_finalize_write_stream_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), "__call__" + ) as call: + client.finalize_write_stream() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.FinalizeWriteStreamRequest() + + +@pytest.mark.asyncio +async def test_finalize_write_stream_async( + transport: str = "grpc_asyncio", request_type=storage.FinalizeWriteStreamRequest +): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), "__call__" + ) as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + storage.FinalizeWriteStreamResponse(row_count=992,) + ) + response = await client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == storage.FinalizeWriteStreamRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.FinalizeWriteStreamResponse) + assert response.row_count == 992 + + +@pytest.mark.asyncio +async def test_finalize_write_stream_async_from_dict(): + await test_finalize_write_stream_async(request_type=dict) + + +def test_finalize_write_stream_field_headers(): + client = BigQueryWriteClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.FinalizeWriteStreamRequest() + + request.name = "name/value" + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), "__call__" + ) as call: + call.return_value = storage.FinalizeWriteStreamResponse() + client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ("x-goog-request-params", "name=name/value",) in kw["metadata"] + + +@pytest.mark.asyncio +async def test_finalize_write_stream_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.FinalizeWriteStreamRequest() + + request.name = "name/value" + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), "__call__" + ) as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + storage.FinalizeWriteStreamResponse() + ) + await client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ("x-goog-request-params", "name=name/value",) in kw["metadata"] + + +def test_finalize_write_stream_flattened(): + client = BigQueryWriteClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), "__call__" + ) as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FinalizeWriteStreamResponse() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.finalize_write_stream(name="name_value",) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0].name == "name_value" + + +def test_finalize_write_stream_flattened_error(): + client = BigQueryWriteClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.finalize_write_stream( + storage.FinalizeWriteStreamRequest(), name="name_value", + ) + + +@pytest.mark.asyncio +async def test_finalize_write_stream_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), "__call__" + ) as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FinalizeWriteStreamResponse() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + storage.FinalizeWriteStreamResponse() + ) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.finalize_write_stream(name="name_value",) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0].name == "name_value" + + +@pytest.mark.asyncio +async def test_finalize_write_stream_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.finalize_write_stream( + storage.FinalizeWriteStreamRequest(), name="name_value", + ) + + +def test_batch_commit_write_streams( + transport: str = "grpc", request_type=storage.BatchCommitWriteStreamsRequest +): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), "__call__" + ) as call: + # Designate an appropriate return value for the call. + call.return_value = storage.BatchCommitWriteStreamsResponse() + response = client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == storage.BatchCommitWriteStreamsRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.BatchCommitWriteStreamsResponse) + + +def test_batch_commit_write_streams_from_dict(): + test_batch_commit_write_streams(request_type=dict) + + +def test_batch_commit_write_streams_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), "__call__" + ) as call: + client.batch_commit_write_streams() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.BatchCommitWriteStreamsRequest() + + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_async( + transport: str = "grpc_asyncio", request_type=storage.BatchCommitWriteStreamsRequest +): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), "__call__" + ) as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + storage.BatchCommitWriteStreamsResponse() + ) + response = await client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == storage.BatchCommitWriteStreamsRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.BatchCommitWriteStreamsResponse) + + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_async_from_dict(): + await test_batch_commit_write_streams_async(request_type=dict) + + +def test_batch_commit_write_streams_field_headers(): + client = BigQueryWriteClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.BatchCommitWriteStreamsRequest() + + request.parent = "parent/value" + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), "__call__" + ) as call: + call.return_value = storage.BatchCommitWriteStreamsResponse() + client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ("x-goog-request-params", "parent=parent/value",) in kw["metadata"] + + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.BatchCommitWriteStreamsRequest() + + request.parent = "parent/value" + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), "__call__" + ) as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + storage.BatchCommitWriteStreamsResponse() + ) + await client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ("x-goog-request-params", "parent=parent/value",) in kw["metadata"] + + +def test_batch_commit_write_streams_flattened(): + client = BigQueryWriteClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), "__call__" + ) as call: + # Designate an appropriate return value for the call. + call.return_value = storage.BatchCommitWriteStreamsResponse() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.batch_commit_write_streams(parent="parent_value",) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0].parent == "parent_value" + + +def test_batch_commit_write_streams_flattened_error(): + client = BigQueryWriteClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.batch_commit_write_streams( + storage.BatchCommitWriteStreamsRequest(), parent="parent_value", + ) + + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), "__call__" + ) as call: + # Designate an appropriate return value for the call. + call.return_value = storage.BatchCommitWriteStreamsResponse() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + storage.BatchCommitWriteStreamsResponse() + ) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.batch_commit_write_streams(parent="parent_value",) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0].parent == "parent_value" + + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.batch_commit_write_streams( + storage.BatchCommitWriteStreamsRequest(), parent="parent_value", + ) + + +def test_flush_rows(transport: str = "grpc", request_type=storage.FlushRowsRequest): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.flush_rows), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FlushRowsResponse(offset=647,) + response = client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == storage.FlushRowsRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.FlushRowsResponse) + assert response.offset == 647 + + +def test_flush_rows_from_dict(): + test_flush_rows(request_type=dict) + + +def test_flush_rows_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.flush_rows), "__call__") as call: + client.flush_rows() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.FlushRowsRequest() + + +@pytest.mark.asyncio +async def test_flush_rows_async( + transport: str = "grpc_asyncio", request_type=storage.FlushRowsRequest +): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.flush_rows), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + storage.FlushRowsResponse(offset=647,) + ) + response = await client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == storage.FlushRowsRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.FlushRowsResponse) + assert response.offset == 647 + + +@pytest.mark.asyncio +async def test_flush_rows_async_from_dict(): + await test_flush_rows_async(request_type=dict) + + +def test_flush_rows_field_headers(): + client = BigQueryWriteClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.FlushRowsRequest() + + request.write_stream = "write_stream/value" + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.flush_rows), "__call__") as call: + call.return_value = storage.FlushRowsResponse() + client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ("x-goog-request-params", "write_stream=write_stream/value",) in kw[ + "metadata" + ] + + +@pytest.mark.asyncio +async def test_flush_rows_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.FlushRowsRequest() + + request.write_stream = "write_stream/value" + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.flush_rows), "__call__") as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + storage.FlushRowsResponse() + ) + await client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ("x-goog-request-params", "write_stream=write_stream/value",) in kw[ + "metadata" + ] + + +def test_flush_rows_flattened(): + client = BigQueryWriteClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.flush_rows), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FlushRowsResponse() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.flush_rows(write_stream="write_stream_value",) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0].write_stream == "write_stream_value" + + +def test_flush_rows_flattened_error(): + client = BigQueryWriteClient(credentials=ga_credentials.AnonymousCredentials(),) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.flush_rows( + storage.FlushRowsRequest(), write_stream="write_stream_value", + ) + + +@pytest.mark.asyncio +async def test_flush_rows_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.flush_rows), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FlushRowsResponse() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + storage.FlushRowsResponse() + ) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.flush_rows(write_stream="write_stream_value",) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0].write_stream == "write_stream_value" + + +@pytest.mark.asyncio +async def test_flush_rows_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.flush_rows( + storage.FlushRowsRequest(), write_stream="write_stream_value", + ) + + +def test_credentials_transport_error(): + # It is an error to provide credentials and a transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # It is an error to provide a credentials file and a transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryWriteClient( + client_options={"credentials_file": "credentials.json"}, + transport=transport, + ) + + # It is an error to provide scopes and a transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryWriteClient( + client_options={"scopes": ["1", "2"]}, transport=transport, + ) + + +def test_transport_instance(): + # A client may be instantiated with a custom transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + client = BigQueryWriteClient(transport=transport) + assert client.transport is transport + + +def test_transport_get_channel(): + # A client may be instantiated with a custom transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + + transport = transports.BigQueryWriteGrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryWriteGrpcTransport, + transports.BigQueryWriteGrpcAsyncIOTransport, + ], +) +def test_transport_adc(transport_class): + # Test default credentials are used if not provided. + with mock.patch.object(google.auth, "default") as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class() + adc.assert_called_once() + + +def test_transport_grpc_default(): + # A client should use the gRPC transport by default. + client = BigQueryWriteClient(credentials=ga_credentials.AnonymousCredentials(),) + assert isinstance(client.transport, transports.BigQueryWriteGrpcTransport,) + + +def test_big_query_write_base_transport_error(): + # Passing both a credentials object and credentials_file should raise an error + with pytest.raises(core_exceptions.DuplicateCredentialArgs): + transport = transports.BigQueryWriteTransport( + credentials=ga_credentials.AnonymousCredentials(), + credentials_file="credentials.json", + ) + + +def test_big_query_write_base_transport(): + # Instantiate the base transport. + with mock.patch( + "google.cloud.bigquery_storage_v1.services.big_query_write.transports.BigQueryWriteTransport.__init__" + ) as Transport: + Transport.return_value = None + transport = transports.BigQueryWriteTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Every method on the transport should just blindly + # raise NotImplementedError. + methods = ( + "create_write_stream", + "append_rows", + "get_write_stream", + "finalize_write_stream", + "batch_commit_write_streams", + "flush_rows", + ) + for method in methods: + with pytest.raises(NotImplementedError): + getattr(transport, method)(request=object()) + + +@requires_google_auth_gte_1_25_0 +def test_big_query_write_base_transport_with_credentials_file(): + # Instantiate the base transport with a credentials file + with mock.patch.object( + google.auth, "load_credentials_from_file", autospec=True + ) as load_creds, mock.patch( + "google.cloud.bigquery_storage_v1.services.big_query_write.transports.BigQueryWriteTransport._prep_wrapped_messages" + ) as Transport: + Transport.return_value = None + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryWriteTransport( + credentials_file="credentials.json", quota_project_id="octopus", + ) + load_creds.assert_called_once_with( + "credentials.json", + scopes=None, + default_scopes=( + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/bigquery.insertdata", + "https://www.googleapis.com/auth/cloud-platform", + ), + quota_project_id="octopus", + ) + + +@requires_google_auth_lt_1_25_0 +def test_big_query_write_base_transport_with_credentials_file_old_google_auth(): + # Instantiate the base transport with a credentials file + with mock.patch.object( + google.auth, "load_credentials_from_file", autospec=True + ) as load_creds, mock.patch( + "google.cloud.bigquery_storage_v1.services.big_query_write.transports.BigQueryWriteTransport._prep_wrapped_messages" + ) as Transport: + Transport.return_value = None + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryWriteTransport( + credentials_file="credentials.json", quota_project_id="octopus", + ) + load_creds.assert_called_once_with( + "credentials.json", + scopes=( + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/bigquery.insertdata", + "https://www.googleapis.com/auth/cloud-platform", + ), + quota_project_id="octopus", + ) + + +def test_big_query_write_base_transport_with_adc(): + # Test the default credentials are used if credentials and credentials_file are None. + with mock.patch.object(google.auth, "default", autospec=True) as adc, mock.patch( + "google.cloud.bigquery_storage_v1.services.big_query_write.transports.BigQueryWriteTransport._prep_wrapped_messages" + ) as Transport: + Transport.return_value = None + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryWriteTransport() + adc.assert_called_once() + + +@requires_google_auth_gte_1_25_0 +def test_big_query_write_auth_adc(): + # If no credentials are provided, we should use ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + BigQueryWriteClient() + adc.assert_called_once_with( + scopes=None, + default_scopes=( + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/bigquery.insertdata", + "https://www.googleapis.com/auth/cloud-platform", + ), + quota_project_id=None, + ) + + +@requires_google_auth_lt_1_25_0 +def test_big_query_write_auth_adc_old_google_auth(): + # If no credentials are provided, we should use ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + BigQueryWriteClient() + adc.assert_called_once_with( + scopes=( + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/bigquery.insertdata", + "https://www.googleapis.com/auth/cloud-platform", + ), + quota_project_id=None, + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryWriteGrpcTransport, + transports.BigQueryWriteGrpcAsyncIOTransport, + ], +) +@requires_google_auth_gte_1_25_0 +def test_big_query_write_transport_auth_adc(transport_class): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class(quota_project_id="octopus", scopes=["1", "2"]) + adc.assert_called_once_with( + scopes=["1", "2"], + default_scopes=( + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/bigquery.insertdata", + "https://www.googleapis.com/auth/cloud-platform", + ), + quota_project_id="octopus", + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryWriteGrpcTransport, + transports.BigQueryWriteGrpcAsyncIOTransport, + ], +) +@requires_google_auth_lt_1_25_0 +def test_big_query_write_transport_auth_adc_old_google_auth(transport_class): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class(quota_project_id="octopus") + adc.assert_called_once_with( + scopes=( + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/bigquery.insertdata", + "https://www.googleapis.com/auth/cloud-platform", + ), + quota_project_id="octopus", + ) + + +@pytest.mark.parametrize( + "transport_class,grpc_helpers", + [ + (transports.BigQueryWriteGrpcTransport, grpc_helpers), + (transports.BigQueryWriteGrpcAsyncIOTransport, grpc_helpers_async), + ], +) +def test_big_query_write_transport_create_channel(transport_class, grpc_helpers): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object( + google.auth, "default", autospec=True + ) as adc, mock.patch.object( + grpc_helpers, "create_channel", autospec=True + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + adc.return_value = (creds, None) + transport_class(quota_project_id="octopus", scopes=["1", "2"]) + + create_channel.assert_called_with( + "bigquerystorage.googleapis.com:443", + credentials=creds, + credentials_file=None, + quota_project_id="octopus", + default_scopes=( + "https://www.googleapis.com/auth/bigquery", + "https://www.googleapis.com/auth/bigquery.insertdata", + "https://www.googleapis.com/auth/cloud-platform", + ), + scopes=["1", "2"], + default_host="bigquerystorage.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryWriteGrpcTransport, + transports.BigQueryWriteGrpcAsyncIOTransport, + ], +) +def test_big_query_write_grpc_transport_client_cert_source_for_mtls(transport_class): + cred = ga_credentials.AnonymousCredentials() + + # Check ssl_channel_credentials is used if provided. + with mock.patch.object(transport_class, "create_channel") as mock_create_channel: + mock_ssl_channel_creds = mock.Mock() + transport_class( + host="squid.clam.whelk", + credentials=cred, + ssl_channel_credentials=mock_ssl_channel_creds, + ) + mock_create_channel.assert_called_once_with( + "squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_channel_creds, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Check if ssl_channel_credentials is not provided, then client_cert_source_for_mtls + # is used. + with mock.patch.object(transport_class, "create_channel", return_value=mock.Mock()): + with mock.patch("grpc.ssl_channel_credentials") as mock_ssl_cred: + transport_class( + credentials=cred, + client_cert_source_for_mtls=client_cert_source_callback, + ) + expected_cert, expected_key = client_cert_source_callback() + mock_ssl_cred.assert_called_once_with( + certificate_chain=expected_cert, private_key=expected_key + ) + + +def test_big_query_write_host_no_port(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions( + api_endpoint="bigquerystorage.googleapis.com" + ), + ) + assert client.transport._host == "bigquerystorage.googleapis.com:443" + + +def test_big_query_write_host_with_port(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions( + api_endpoint="bigquerystorage.googleapis.com:8000" + ), + ) + assert client.transport._host == "bigquerystorage.googleapis.com:8000" + + +def test_big_query_write_grpc_transport_channel(): + channel = grpc.secure_channel("http://localhost/", grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.BigQueryWriteGrpcTransport( + host="squid.clam.whelk", channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +def test_big_query_write_grpc_asyncio_transport_channel(): + channel = aio.secure_channel("http://localhost/", grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.BigQueryWriteGrpcAsyncIOTransport( + host="squid.clam.whelk", channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryWriteGrpcTransport, + transports.BigQueryWriteGrpcAsyncIOTransport, + ], +) +def test_big_query_write_transport_channel_mtls_with_client_cert_source( + transport_class, +): + with mock.patch( + "grpc.ssl_channel_credentials", autospec=True + ) as grpc_ssl_channel_cred: + with mock.patch.object( + transport_class, "create_channel" + ) as grpc_create_channel: + mock_ssl_cred = mock.Mock() + grpc_ssl_channel_cred.return_value = mock_ssl_cred + + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + + cred = ga_credentials.AnonymousCredentials() + with pytest.warns(DeprecationWarning): + with mock.patch.object(google.auth, "default") as adc: + adc.return_value = (cred, None) + transport = transport_class( + host="squid.clam.whelk", + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=client_cert_source_callback, + ) + adc.assert_called_once() + + grpc_ssl_channel_cred.assert_called_once_with( + certificate_chain=b"cert bytes", private_key=b"key bytes" + ) + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + assert transport._ssl_channel_credentials == mock_ssl_cred + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryWriteGrpcTransport, + transports.BigQueryWriteGrpcAsyncIOTransport, + ], +) +def test_big_query_write_transport_channel_mtls_with_adc(transport_class): + mock_ssl_cred = mock.Mock() + with mock.patch.multiple( + "google.auth.transport.grpc.SslCredentials", + __init__=mock.Mock(return_value=None), + ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred), + ): + with mock.patch.object( + transport_class, "create_channel" + ) as grpc_create_channel: + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + mock_cred = mock.Mock() + + with pytest.warns(DeprecationWarning): + transport = transport_class( + host="squid.clam.whelk", + credentials=mock_cred, + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=None, + ) + + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=mock_cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + + +def test_table_path(): + project = "squid" + dataset = "clam" + table = "whelk" + expected = "projects/{project}/datasets/{dataset}/tables/{table}".format( + project=project, dataset=dataset, table=table, + ) + actual = BigQueryWriteClient.table_path(project, dataset, table) + assert expected == actual + + +def test_parse_table_path(): + expected = { + "project": "octopus", + "dataset": "oyster", + "table": "nudibranch", + } + path = BigQueryWriteClient.table_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_table_path(path) + assert expected == actual + + +def test_write_stream_path(): + project = "cuttlefish" + dataset = "mussel" + table = "winkle" + stream = "nautilus" + expected = "projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}".format( + project=project, dataset=dataset, table=table, stream=stream, + ) + actual = BigQueryWriteClient.write_stream_path(project, dataset, table, stream) + assert expected == actual + + +def test_parse_write_stream_path(): + expected = { + "project": "scallop", + "dataset": "abalone", + "table": "squid", + "stream": "clam", + } + path = BigQueryWriteClient.write_stream_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_write_stream_path(path) + assert expected == actual + + +def test_common_billing_account_path(): + billing_account = "whelk" + expected = "billingAccounts/{billing_account}".format( + billing_account=billing_account, + ) + actual = BigQueryWriteClient.common_billing_account_path(billing_account) + assert expected == actual + + +def test_parse_common_billing_account_path(): + expected = { + "billing_account": "octopus", + } + path = BigQueryWriteClient.common_billing_account_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_billing_account_path(path) + assert expected == actual + + +def test_common_folder_path(): + folder = "oyster" + expected = "folders/{folder}".format(folder=folder,) + actual = BigQueryWriteClient.common_folder_path(folder) + assert expected == actual + + +def test_parse_common_folder_path(): + expected = { + "folder": "nudibranch", + } + path = BigQueryWriteClient.common_folder_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_folder_path(path) + assert expected == actual + + +def test_common_organization_path(): + organization = "cuttlefish" + expected = "organizations/{organization}".format(organization=organization,) + actual = BigQueryWriteClient.common_organization_path(organization) + assert expected == actual + + +def test_parse_common_organization_path(): + expected = { + "organization": "mussel", + } + path = BigQueryWriteClient.common_organization_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_organization_path(path) + assert expected == actual + + +def test_common_project_path(): + project = "winkle" + expected = "projects/{project}".format(project=project,) + actual = BigQueryWriteClient.common_project_path(project) + assert expected == actual + + +def test_parse_common_project_path(): + expected = { + "project": "nautilus", + } + path = BigQueryWriteClient.common_project_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_project_path(path) + assert expected == actual + + +def test_common_location_path(): + project = "scallop" + location = "abalone" + expected = "projects/{project}/locations/{location}".format( + project=project, location=location, + ) + actual = BigQueryWriteClient.common_location_path(project, location) + assert expected == actual + + +def test_parse_common_location_path(): + expected = { + "project": "squid", + "location": "clam", + } + path = BigQueryWriteClient.common_location_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_location_path(path) + assert expected == actual + + +def test_client_withDEFAULT_CLIENT_INFO(): + client_info = gapic_v1.client_info.ClientInfo() + + with mock.patch.object( + transports.BigQueryWriteTransport, "_prep_wrapped_messages" + ) as prep: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), client_info=client_info, + ) + prep.assert_called_once_with(client_info) + + with mock.patch.object( + transports.BigQueryWriteTransport, "_prep_wrapped_messages" + ) as prep: + transport_class = BigQueryWriteClient.get_transport_class() + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials(), client_info=client_info, + ) + prep.assert_called_once_with(client_info)