diff --git a/google/cloud/datastore_admin_v1/__init__.py b/google/cloud/datastore_admin_v1/__init__.py index 89cac8e1..70a79c07 100644 --- a/google/cloud/datastore_admin_v1/__init__.py +++ b/google/cloud/datastore_admin_v1/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +15,11 @@ # from .services.datastore_admin import DatastoreAdminClient +from .services.datastore_admin import DatastoreAdminAsyncClient + from .types.datastore_admin import CommonMetadata +from .types.datastore_admin import CreateIndexRequest +from .types.datastore_admin import DeleteIndexRequest from .types.datastore_admin import EntityFilter from .types.datastore_admin import ExportEntitiesMetadata from .types.datastore_admin import ExportEntitiesRequest @@ -27,13 +30,16 @@ from .types.datastore_admin import IndexOperationMetadata from .types.datastore_admin import ListIndexesRequest from .types.datastore_admin import ListIndexesResponse -from .types.datastore_admin import OperationType from .types.datastore_admin import Progress +from .types.datastore_admin import OperationType from .types.index import Index - __all__ = ( + "DatastoreAdminAsyncClient", "CommonMetadata", + "CreateIndexRequest", + "DatastoreAdminClient", + "DeleteIndexRequest", "EntityFilter", "ExportEntitiesMetadata", "ExportEntitiesRequest", @@ -47,5 +53,4 @@ "ListIndexesResponse", "OperationType", "Progress", - "DatastoreAdminClient", ) diff --git a/google/cloud/datastore_admin_v1/gapic_metadata.json b/google/cloud/datastore_admin_v1/gapic_metadata.json new file mode 100644 index 00000000..8df5d474 --- /dev/null +++ b/google/cloud/datastore_admin_v1/gapic_metadata.json @@ -0,0 +1,83 @@ + { + "comment": "This file maps proto services/RPCs to the corresponding library clients/methods", + "language": "python", + "libraryPackage": "google.cloud.datastore_admin_v1", + "protoPackage": "google.datastore.admin.v1", + "schema": "1.0", + "services": { + "DatastoreAdmin": { + "clients": { + "grpc": { + "libraryClient": "DatastoreAdminClient", + "rpcs": { + "CreateIndex": { + "methods": [ + "create_index" + ] + }, + "DeleteIndex": { + "methods": [ + "delete_index" + ] + }, + "ExportEntities": { + "methods": [ + "export_entities" + ] + }, + "GetIndex": { + "methods": [ + "get_index" + ] + }, + "ImportEntities": { + "methods": [ + "import_entities" + ] + }, + "ListIndexes": { + "methods": [ + "list_indexes" + ] + } + } + }, + "grpc-async": { + "libraryClient": "DatastoreAdminAsyncClient", + "rpcs": { + "CreateIndex": { + "methods": [ + "create_index" + ] + }, + "DeleteIndex": { + "methods": [ + "delete_index" + ] + }, + "ExportEntities": { + "methods": [ + "export_entities" + ] + }, + "GetIndex": { + "methods": [ + "get_index" + ] + }, + "ImportEntities": { + "methods": [ + "import_entities" + ] + }, + "ListIndexes": { + "methods": [ + "list_indexes" + ] + } + } + } + } + } + } +} diff --git a/google/cloud/datastore_admin_v1/services/__init__.py b/google/cloud/datastore_admin_v1/services/__init__.py index 42ffdf2b..4de65971 100644 --- a/google/cloud/datastore_admin_v1/services/__init__.py +++ b/google/cloud/datastore_admin_v1/services/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/google/cloud/datastore_admin_v1/services/datastore_admin/__init__.py b/google/cloud/datastore_admin_v1/services/datastore_admin/__init__.py index a004406b..951a69a9 100644 --- a/google/cloud/datastore_admin_v1/services/datastore_admin/__init__.py +++ b/google/cloud/datastore_admin_v1/services/datastore_admin/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # - from .client import DatastoreAdminClient from .async_client import DatastoreAdminAsyncClient diff --git a/google/cloud/datastore_admin_v1/services/datastore_admin/async_client.py b/google/cloud/datastore_admin_v1/services/datastore_admin/async_client.py index 0cd7d99e..e1d24d16 100644 --- a/google/cloud/datastore_admin_v1/services/datastore_admin/async_client.py +++ b/google/cloud/datastore_admin_v1/services/datastore_admin/async_client.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,27 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. # - from collections import OrderedDict import functools import re from typing import Dict, Sequence, Tuple, Type, Union import pkg_resources -import google.api_core.client_options as ClientOptions # type: ignore -from google.api_core import exceptions # type: ignore +from google.api_core.client_options import ClientOptions # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore from google.api_core import gapic_v1 # type: ignore from google.api_core import retry as retries # type: ignore -from google.auth import credentials # type: ignore +from google.auth import credentials as ga_credentials # type: ignore from google.oauth2 import service_account # type: ignore +OptionalRetry = Union[retries.Retry, object] + from google.api_core import operation # type: ignore from google.api_core import operation_async # type: ignore from google.cloud.datastore_admin_v1.services.datastore_admin import pagers from google.cloud.datastore_admin_v1.types import datastore_admin from google.cloud.datastore_admin_v1.types import index -from google.protobuf import empty_pb2 as empty # type: ignore - +from google.protobuf import empty_pb2 # type: ignore from .transports.base import DatastoreAdminTransport, DEFAULT_CLIENT_INFO from .transports.grpc_asyncio import DatastoreAdminGrpcAsyncIOTransport from .client import DatastoreAdminClient @@ -112,35 +111,61 @@ class DatastoreAdminAsyncClient: parse_common_billing_account_path = staticmethod( DatastoreAdminClient.parse_common_billing_account_path ) - common_folder_path = staticmethod(DatastoreAdminClient.common_folder_path) parse_common_folder_path = staticmethod( DatastoreAdminClient.parse_common_folder_path ) - common_organization_path = staticmethod( DatastoreAdminClient.common_organization_path ) parse_common_organization_path = staticmethod( DatastoreAdminClient.parse_common_organization_path ) - common_project_path = staticmethod(DatastoreAdminClient.common_project_path) parse_common_project_path = staticmethod( DatastoreAdminClient.parse_common_project_path ) - common_location_path = staticmethod(DatastoreAdminClient.common_location_path) parse_common_location_path = staticmethod( DatastoreAdminClient.parse_common_location_path ) - from_service_account_file = DatastoreAdminClient.from_service_account_file + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + DatastoreAdminAsyncClient: The constructed client. + """ + return DatastoreAdminClient.from_service_account_info.__func__(DatastoreAdminAsyncClient, info, *args, **kwargs) # type: ignore + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + DatastoreAdminAsyncClient: The constructed client. + """ + return DatastoreAdminClient.from_service_account_file.__func__(DatastoreAdminAsyncClient, filename, *args, **kwargs) # type: ignore + from_service_account_json = from_service_account_file @property def transport(self) -> DatastoreAdminTransport: - """Return the transport used by the client instance. + """Returns the transport used by the client instance. Returns: DatastoreAdminTransport: The transport used by the client instance. @@ -154,12 +179,12 @@ def transport(self) -> DatastoreAdminTransport: def __init__( self, *, - credentials: credentials.Credentials = None, + credentials: ga_credentials.Credentials = None, transport: Union[str, DatastoreAdminTransport] = "grpc_asyncio", client_options: ClientOptions = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, ) -> None: - """Instantiate the datastore admin client. + """Instantiates the datastore admin client. Args: credentials (Optional[google.auth.credentials.Credentials]): The @@ -191,7 +216,6 @@ def __init__( google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport creation failed for any reason. """ - self._client = DatastoreAdminClient( credentials=credentials, transport=transport, @@ -201,13 +225,13 @@ def __init__( async def export_entities( self, - request: datastore_admin.ExportEntitiesRequest = None, + request: Union[datastore_admin.ExportEntitiesRequest, dict] = None, *, project_id: str = None, labels: Sequence[datastore_admin.ExportEntitiesRequest.LabelsEntry] = None, entity_filter: datastore_admin.EntityFilter = None, output_url_prefix: str = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> operation_async.AsyncOperation: @@ -223,23 +247,25 @@ async def export_entities( Google Cloud Storage. Args: - request (:class:`~.datastore_admin.ExportEntitiesRequest`): + request (Union[google.cloud.datastore_admin_v1.types.ExportEntitiesRequest, dict]): The request object. The request for [google.datastore.admin.v1.DatastoreAdmin.ExportEntities][google.datastore.admin.v1.DatastoreAdmin.ExportEntities]. project_id (:class:`str`): Required. Project ID against which to make the request. + This corresponds to the ``project_id`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - labels (:class:`Sequence[~.datastore_admin.ExportEntitiesRequest.LabelsEntry]`): + labels (:class:`Sequence[google.cloud.datastore_admin_v1.types.ExportEntitiesRequest.LabelsEntry]`): Client-assigned labels. This corresponds to the ``labels`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - entity_filter (:class:`~.datastore_admin.EntityFilter`): + entity_filter (:class:`google.cloud.datastore_admin_v1.types.EntityFilter`): Description of what data from the project is included in the export. + This corresponds to the ``entity_filter`` field on the ``request`` instance; if ``request`` is provided, this should not be set. @@ -268,10 +294,10 @@ async def export_entities( By nesting the data files deeper, the same Cloud Storage bucket can be used in multiple ExportEntities operations without conflict. + This corresponds to the ``output_url_prefix`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -279,13 +305,11 @@ async def export_entities( sent along with the request as metadata. Returns: - ~.operation_async.AsyncOperation: + google.api_core.operation_async.AsyncOperation: An object representing a long-running operation. - The result type for the operation will be - :class:``~.datastore_admin.ExportEntitiesResponse``: The - response for - [google.datastore.admin.v1.DatastoreAdmin.ExportEntities][google.datastore.admin.v1.DatastoreAdmin.ExportEntities]. + The result type for the operation will be :class:`google.cloud.datastore_admin_v1.types.ExportEntitiesResponse` The response for + [google.datastore.admin.v1.DatastoreAdmin.ExportEntities][google.datastore.admin.v1.DatastoreAdmin.ExportEntities]. """ # Create or coerce a protobuf request object. @@ -304,7 +328,6 @@ async def export_entities( # If we have keyword arguments corresponding to fields on the # request, apply these. - if project_id is not None: request.project_id = project_id if entity_filter is not None: @@ -339,13 +362,13 @@ async def export_entities( async def import_entities( self, - request: datastore_admin.ImportEntitiesRequest = None, + request: Union[datastore_admin.ImportEntitiesRequest, dict] = None, *, project_id: str = None, labels: Sequence[datastore_admin.ImportEntitiesRequest.LabelsEntry] = None, input_url: str = None, entity_filter: datastore_admin.EntityFilter = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> operation_async.AsyncOperation: @@ -358,16 +381,17 @@ async def import_entities( imported to Cloud Datastore. Args: - request (:class:`~.datastore_admin.ImportEntitiesRequest`): + request (Union[google.cloud.datastore_admin_v1.types.ImportEntitiesRequest, dict]): The request object. The request for [google.datastore.admin.v1.DatastoreAdmin.ImportEntities][google.datastore.admin.v1.DatastoreAdmin.ImportEntities]. project_id (:class:`str`): Required. Project ID against which to make the request. + This corresponds to the ``project_id`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - labels (:class:`Sequence[~.datastore_admin.ImportEntitiesRequest.LabelsEntry]`): + labels (:class:`Sequence[google.cloud.datastore_admin_v1.types.ImportEntitiesRequest.LabelsEntry]`): Client-assigned labels. This corresponds to the ``labels`` field on the ``request`` instance; if ``request`` is provided, this @@ -388,20 +412,21 @@ async def import_entities( For more information, see [google.datastore.admin.v1.ExportEntitiesResponse.output_url][google.datastore.admin.v1.ExportEntitiesResponse.output_url]. + This corresponds to the ``input_url`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - entity_filter (:class:`~.datastore_admin.EntityFilter`): + entity_filter (:class:`google.cloud.datastore_admin_v1.types.EntityFilter`): Optionally specify which kinds/namespaces are to be imported. If provided, the list must be a subset of the EntityFilter used in creating the export, otherwise a FAILED_PRECONDITION error will be returned. If no filter is specified then all entities from the export are imported. + This corresponds to the ``entity_filter`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -409,24 +434,22 @@ async def import_entities( sent along with the request as metadata. Returns: - ~.operation_async.AsyncOperation: + google.api_core.operation_async.AsyncOperation: An object representing a long-running operation. - The result type for the operation will be - :class:``~.empty.Empty``: A generic empty message that - you can re-use to avoid defining duplicated empty - messages in your APIs. A typical example is to use it as - the request or the response type of an API method. For - instance: + The result type for the operation will be :class:`google.protobuf.empty_pb2.Empty` A generic empty message that you can re-use to avoid defining duplicated + empty messages in your APIs. A typical example is to + use it as the request or the response type of an API + method. For instance: - :: + service Foo { + rpc Bar(google.protobuf.Empty) returns + (google.protobuf.Empty); - service Foo { - rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); - } + } - The JSON representation for ``Empty`` is empty JSON - object ``{}``. + The JSON representation for Empty is empty JSON + object {}. """ # Create or coerce a protobuf request object. @@ -443,7 +466,6 @@ async def import_entities( # If we have keyword arguments corresponding to fields on the # request, apply these. - if project_id is not None: request.project_id = project_id if input_url is not None: @@ -469,28 +491,162 @@ async def import_entities( response = operation_async.from_gapic( response, self._client._transport.operations_client, - empty.Empty, + empty_pb2.Empty, metadata_type=datastore_admin.ImportEntitiesMetadata, ) # Done; return the response. return response + async def create_index( + self, + request: Union[datastore_admin.CreateIndexRequest, dict] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> operation_async.AsyncOperation: + r"""Creates the specified index. A newly created index's initial + state is ``CREATING``. On completion of the returned + [google.longrunning.Operation][google.longrunning.Operation], + the state will be ``READY``. If the index already exists, the + call will return an ``ALREADY_EXISTS`` status. + + During index creation, the process could result in an error, in + which case the index will move to the ``ERROR`` state. The + process can be recovered by fixing the data that caused the + error, removing the index with + [delete][google.datastore.admin.v1.DatastoreAdmin.DeleteIndex], + then re-creating the index with [create] + [google.datastore.admin.v1.DatastoreAdmin.CreateIndex]. + + Indexes with a single property cannot be created. + + Args: + request (Union[google.cloud.datastore_admin_v1.types.CreateIndexRequest, dict]): + The request object. The request for + [google.datastore.admin.v1.DatastoreAdmin.CreateIndex][google.datastore.admin.v1.DatastoreAdmin.CreateIndex]. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.api_core.operation_async.AsyncOperation: + An object representing a long-running operation. + + The result type for the operation will be + :class:`google.cloud.datastore_admin_v1.types.Index` + Datastore composite index definition. + + """ + # Create or coerce a protobuf request object. + request = datastore_admin.CreateIndexRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.create_index, + default_timeout=60.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Send the request. + response = await rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Wrap the response in an operation future. + response = operation_async.from_gapic( + response, + self._client._transport.operations_client, + index.Index, + metadata_type=datastore_admin.IndexOperationMetadata, + ) + + # Done; return the response. + return response + + async def delete_index( + self, + request: Union[datastore_admin.DeleteIndexRequest, dict] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> operation_async.AsyncOperation: + r"""Deletes an existing index. An index can only be deleted if it is + in a ``READY`` or ``ERROR`` state. On successful execution of + the request, the index will be in a ``DELETING`` + [state][google.datastore.admin.v1.Index.State]. And on + completion of the returned + [google.longrunning.Operation][google.longrunning.Operation], + the index will be removed. + + During index deletion, the process could result in an error, in + which case the index will move to the ``ERROR`` state. The + process can be recovered by fixing the data that caused the + error, followed by calling + [delete][google.datastore.admin.v1.DatastoreAdmin.DeleteIndex] + again. + + Args: + request (Union[google.cloud.datastore_admin_v1.types.DeleteIndexRequest, dict]): + The request object. The request for + [google.datastore.admin.v1.DatastoreAdmin.DeleteIndex][google.datastore.admin.v1.DatastoreAdmin.DeleteIndex]. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.api_core.operation_async.AsyncOperation: + An object representing a long-running operation. + + The result type for the operation will be + :class:`google.cloud.datastore_admin_v1.types.Index` + Datastore composite index definition. + + """ + # Create or coerce a protobuf request object. + request = datastore_admin.DeleteIndexRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = gapic_v1.method_async.wrap_method( + self._client._transport.delete_index, + default_timeout=60.0, + client_info=DEFAULT_CLIENT_INFO, + ) + + # Send the request. + response = await rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Wrap the response in an operation future. + response = operation_async.from_gapic( + response, + self._client._transport.operations_client, + index.Index, + metadata_type=datastore_admin.IndexOperationMetadata, + ) + + # Done; return the response. + return response + async def get_index( self, - request: datastore_admin.GetIndexRequest = None, + request: Union[datastore_admin.GetIndexRequest, dict] = None, *, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> index.Index: r"""Gets an index. Args: - request (:class:`~.datastore_admin.GetIndexRequest`): + request (Union[google.cloud.datastore_admin_v1.types.GetIndexRequest, dict]): The request object. The request for [google.datastore.admin.v1.DatastoreAdmin.GetIndex][google.datastore.admin.v1.DatastoreAdmin.GetIndex]. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -498,11 +654,10 @@ async def get_index( sent along with the request as metadata. Returns: - ~.index.Index: - A minimal index definition. + google.cloud.datastore_admin_v1.types.Index: + Datastore composite index definition. """ # Create or coerce a protobuf request object. - request = datastore_admin.GetIndexRequest(request) # Wrap the RPC method; this adds retry and timeout information, @@ -514,8 +669,10 @@ async def get_index( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( - exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, ), + deadline=60.0, ), default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, @@ -529,9 +686,9 @@ async def get_index( async def list_indexes( self, - request: datastore_admin.ListIndexesRequest = None, + request: Union[datastore_admin.ListIndexesRequest, dict] = None, *, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> pagers.ListIndexesAsyncPager: @@ -541,10 +698,9 @@ async def list_indexes( results. Args: - request (:class:`~.datastore_admin.ListIndexesRequest`): + request (Union[google.cloud.datastore_admin_v1.types.ListIndexesRequest, dict]): The request object. The request for [google.datastore.admin.v1.DatastoreAdmin.ListIndexes][google.datastore.admin.v1.DatastoreAdmin.ListIndexes]. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -552,16 +708,15 @@ async def list_indexes( sent along with the request as metadata. Returns: - ~.pagers.ListIndexesAsyncPager: + google.cloud.datastore_admin_v1.services.datastore_admin.pagers.ListIndexesAsyncPager: The response for - [google.datastore.admin.v1.DatastoreAdmin.ListIndexes][google.datastore.admin.v1.DatastoreAdmin.ListIndexes]. + [google.datastore.admin.v1.DatastoreAdmin.ListIndexes][google.datastore.admin.v1.DatastoreAdmin.ListIndexes]. Iterating over this object will yield results and resolve additional pages automatically. """ # Create or coerce a protobuf request object. - request = datastore_admin.ListIndexesRequest(request) # Wrap the RPC method; this adds retry and timeout information, @@ -573,8 +728,10 @@ async def list_indexes( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( - exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, ), + deadline=60.0, ), default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, @@ -592,6 +749,12 @@ async def list_indexes( # Done; return the response. return response + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + await self.transport.close() + try: DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( diff --git a/google/cloud/datastore_admin_v1/services/datastore_admin/client.py b/google/cloud/datastore_admin_v1/services/datastore_admin/client.py index a9756759..b8ca70c4 100644 --- a/google/cloud/datastore_admin_v1/services/datastore_admin/client.py +++ b/google/cloud/datastore_admin_v1/services/datastore_admin/client.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,31 +13,31 @@ # See the License for the specific language governing permissions and # limitations under the License. # - from collections import OrderedDict from distutils import util import os import re -from typing import Callable, Dict, Optional, Sequence, Tuple, Type, Union +from typing import Dict, Optional, Sequence, Tuple, Type, Union import pkg_resources from google.api_core import client_options as client_options_lib # type: ignore -from google.api_core import exceptions # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore from google.api_core import gapic_v1 # type: ignore from google.api_core import retry as retries # type: ignore -from google.auth import credentials # type: ignore +from google.auth import credentials as ga_credentials # type: ignore from google.auth.transport import mtls # type: ignore from google.auth.transport.grpc import SslCredentials # type: ignore from google.auth.exceptions import MutualTLSChannelError # type: ignore from google.oauth2 import service_account # type: ignore +OptionalRetry = Union[retries.Retry, object] + from google.api_core import operation # type: ignore from google.api_core import operation_async # type: ignore from google.cloud.datastore_admin_v1.services.datastore_admin import pagers from google.cloud.datastore_admin_v1.types import datastore_admin from google.cloud.datastore_admin_v1.types import index -from google.protobuf import empty_pb2 as empty # type: ignore - +from google.protobuf import empty_pb2 # type: ignore from .transports.base import DatastoreAdminTransport, DEFAULT_CLIENT_INFO from .transports.grpc import DatastoreAdminGrpcTransport from .transports.grpc_asyncio import DatastoreAdminGrpcAsyncIOTransport @@ -59,7 +58,7 @@ class DatastoreAdminClientMeta(type): _transport_registry["grpc_asyncio"] = DatastoreAdminGrpcAsyncIOTransport def get_transport_class(cls, label: str = None,) -> Type[DatastoreAdminTransport]: - """Return an appropriate transport class. + """Returns an appropriate transport class. Args: label: The name of the desired transport. If none is @@ -137,7 +136,8 @@ class DatastoreAdminClient(metaclass=DatastoreAdminClientMeta): @staticmethod def _get_default_mtls_endpoint(api_endpoint): - """Convert api endpoint to mTLS endpoint. + """Converts api endpoint to mTLS endpoint. + Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. Args: @@ -169,10 +169,27 @@ def _get_default_mtls_endpoint(api_endpoint): DEFAULT_ENDPOINT ) + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + DatastoreAdminClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_info(info) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + @classmethod def from_service_account_file(cls, filename: str, *args, **kwargs): """Creates an instance of this client using the provided credentials - file. + file. Args: filename (str): The path to the service account private key json @@ -181,7 +198,7 @@ def from_service_account_file(cls, filename: str, *args, **kwargs): kwargs: Additional arguments to pass to the constructor. Returns: - {@api.name}: The constructed client. + DatastoreAdminClient: The constructed client. """ credentials = service_account.Credentials.from_service_account_file(filename) kwargs["credentials"] = credentials @@ -191,16 +208,17 @@ def from_service_account_file(cls, filename: str, *args, **kwargs): @property def transport(self) -> DatastoreAdminTransport: - """Return the transport used by the client instance. + """Returns the transport used by the client instance. Returns: - DatastoreAdminTransport: The transport used by the client instance. + DatastoreAdminTransport: The transport used by the client + instance. """ return self._transport @staticmethod def common_billing_account_path(billing_account: str,) -> str: - """Return a fully-qualified billing_account string.""" + """Returns a fully-qualified billing_account string.""" return "billingAccounts/{billing_account}".format( billing_account=billing_account, ) @@ -213,7 +231,7 @@ def parse_common_billing_account_path(path: str) -> Dict[str, str]: @staticmethod def common_folder_path(folder: str,) -> str: - """Return a fully-qualified folder string.""" + """Returns a fully-qualified folder string.""" return "folders/{folder}".format(folder=folder,) @staticmethod @@ -224,7 +242,7 @@ def parse_common_folder_path(path: str) -> Dict[str, str]: @staticmethod def common_organization_path(organization: str,) -> str: - """Return a fully-qualified organization string.""" + """Returns a fully-qualified organization string.""" return "organizations/{organization}".format(organization=organization,) @staticmethod @@ -235,7 +253,7 @@ def parse_common_organization_path(path: str) -> Dict[str, str]: @staticmethod def common_project_path(project: str,) -> str: - """Return a fully-qualified project string.""" + """Returns a fully-qualified project string.""" return "projects/{project}".format(project=project,) @staticmethod @@ -246,7 +264,7 @@ def parse_common_project_path(path: str) -> Dict[str, str]: @staticmethod def common_location_path(project: str, location: str,) -> str: - """Return a fully-qualified location string.""" + """Returns a fully-qualified location string.""" return "projects/{project}/locations/{location}".format( project=project, location=location, ) @@ -260,12 +278,12 @@ def parse_common_location_path(path: str) -> Dict[str, str]: def __init__( self, *, - credentials: Optional[credentials.Credentials] = None, + credentials: Optional[ga_credentials.Credentials] = None, transport: Union[str, DatastoreAdminTransport, None] = None, client_options: Optional[client_options_lib.ClientOptions] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, ) -> None: - """Instantiate the datastore admin client. + """Instantiates the datastore admin client. Args: credentials (Optional[google.auth.credentials.Credentials]): The @@ -273,10 +291,10 @@ def __init__( credentials identify the application to the service; if none are specified, the client will attempt to ascertain the credentials from the environment. - transport (Union[str, ~.DatastoreAdminTransport]): The + transport (Union[str, DatastoreAdminTransport]): The transport to use. If set to None, a transport is chosen automatically. - client_options (client_options_lib.ClientOptions): Custom options for the + client_options (google.api_core.client_options.ClientOptions): Custom options for the client. It won't take effect if a ``transport`` instance is provided. (1) The ``api_endpoint`` property can be used to override the default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT @@ -312,21 +330,18 @@ def __init__( util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false")) ) - ssl_credentials = None + client_cert_source_func = None is_mtls = False if use_client_cert: if client_options.client_cert_source: - import grpc # type: ignore - - cert, key = client_options.client_cert_source() - ssl_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) is_mtls = True + client_cert_source_func = client_options.client_cert_source else: - creds = SslCredentials() - is_mtls = creds.is_mtls - ssl_credentials = creds.ssl_credentials if is_mtls else None + is_mtls = mtls.has_default_client_cert_source() + if is_mtls: + client_cert_source_func = mtls.default_client_cert_source() + else: + client_cert_source_func = None # Figure out which api endpoint to use. if client_options.api_endpoint is not None: @@ -338,12 +353,14 @@ def __init__( elif use_mtls_env == "always": api_endpoint = self.DEFAULT_MTLS_ENDPOINT elif use_mtls_env == "auto": - api_endpoint = ( - self.DEFAULT_MTLS_ENDPOINT if is_mtls else self.DEFAULT_ENDPOINT - ) + if is_mtls: + api_endpoint = self.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = self.DEFAULT_ENDPOINT else: raise MutualTLSChannelError( - "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted values: never, auto, always" + "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted " + "values: never, auto, always" ) # Save or instantiate the transport. @@ -358,8 +375,8 @@ def __init__( ) if client_options.scopes: raise ValueError( - "When providing a transport instance, " - "provide its scopes directly." + "When providing a transport instance, provide its scopes " + "directly." ) self._transport = transport else: @@ -369,20 +386,21 @@ def __init__( credentials_file=client_options.credentials_file, host=api_endpoint, scopes=client_options.scopes, - ssl_channel_credentials=ssl_credentials, + client_cert_source_for_mtls=client_cert_source_func, quota_project_id=client_options.quota_project_id, client_info=client_info, + always_use_jwt_access=True, ) def export_entities( self, - request: datastore_admin.ExportEntitiesRequest = None, + request: Union[datastore_admin.ExportEntitiesRequest, dict] = None, *, project_id: str = None, labels: Sequence[datastore_admin.ExportEntitiesRequest.LabelsEntry] = None, entity_filter: datastore_admin.EntityFilter = None, output_url_prefix: str = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> operation.Operation: @@ -398,27 +416,29 @@ def export_entities( Google Cloud Storage. Args: - request (:class:`~.datastore_admin.ExportEntitiesRequest`): + request (Union[google.cloud.datastore_admin_v1.types.ExportEntitiesRequest, dict]): The request object. The request for [google.datastore.admin.v1.DatastoreAdmin.ExportEntities][google.datastore.admin.v1.DatastoreAdmin.ExportEntities]. - project_id (:class:`str`): + project_id (str): Required. Project ID against which to make the request. + This corresponds to the ``project_id`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - labels (:class:`Sequence[~.datastore_admin.ExportEntitiesRequest.LabelsEntry]`): + labels (Sequence[google.cloud.datastore_admin_v1.types.ExportEntitiesRequest.LabelsEntry]): Client-assigned labels. This corresponds to the ``labels`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - entity_filter (:class:`~.datastore_admin.EntityFilter`): + entity_filter (google.cloud.datastore_admin_v1.types.EntityFilter): Description of what data from the project is included in the export. + This corresponds to the ``entity_filter`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - output_url_prefix (:class:`str`): + output_url_prefix (str): Required. Location for the export metadata and data files. @@ -443,10 +463,10 @@ def export_entities( By nesting the data files deeper, the same Cloud Storage bucket can be used in multiple ExportEntities operations without conflict. + This corresponds to the ``output_url_prefix`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -454,13 +474,11 @@ def export_entities( sent along with the request as metadata. Returns: - ~.operation.Operation: + google.api_core.operation.Operation: An object representing a long-running operation. - The result type for the operation will be - :class:``~.datastore_admin.ExportEntitiesResponse``: The - response for - [google.datastore.admin.v1.DatastoreAdmin.ExportEntities][google.datastore.admin.v1.DatastoreAdmin.ExportEntities]. + The result type for the operation will be :class:`google.cloud.datastore_admin_v1.types.ExportEntitiesResponse` The response for + [google.datastore.admin.v1.DatastoreAdmin.ExportEntities][google.datastore.admin.v1.DatastoreAdmin.ExportEntities]. """ # Create or coerce a protobuf request object. @@ -481,20 +499,17 @@ def export_entities( # there are no flattened fields. if not isinstance(request, datastore_admin.ExportEntitiesRequest): request = datastore_admin.ExportEntitiesRequest(request) - # If we have keyword arguments corresponding to fields on the # request, apply these. - if project_id is not None: request.project_id = project_id + if labels is not None: + request.labels = labels if entity_filter is not None: request.entity_filter = entity_filter if output_url_prefix is not None: request.output_url_prefix = output_url_prefix - if labels: - request.labels.update(labels) - # Wrap the RPC method; this adds retry and timeout information, # and friendly error handling. rpc = self._transport._wrapped_methods[self._transport.export_entities] @@ -515,13 +530,13 @@ def export_entities( def import_entities( self, - request: datastore_admin.ImportEntitiesRequest = None, + request: Union[datastore_admin.ImportEntitiesRequest, dict] = None, *, project_id: str = None, labels: Sequence[datastore_admin.ImportEntitiesRequest.LabelsEntry] = None, input_url: str = None, entity_filter: datastore_admin.EntityFilter = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> operation.Operation: @@ -534,21 +549,22 @@ def import_entities( imported to Cloud Datastore. Args: - request (:class:`~.datastore_admin.ImportEntitiesRequest`): + request (Union[google.cloud.datastore_admin_v1.types.ImportEntitiesRequest, dict]): The request object. The request for [google.datastore.admin.v1.DatastoreAdmin.ImportEntities][google.datastore.admin.v1.DatastoreAdmin.ImportEntities]. - project_id (:class:`str`): + project_id (str): Required. Project ID against which to make the request. + This corresponds to the ``project_id`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - labels (:class:`Sequence[~.datastore_admin.ImportEntitiesRequest.LabelsEntry]`): + labels (Sequence[google.cloud.datastore_admin_v1.types.ImportEntitiesRequest.LabelsEntry]): Client-assigned labels. This corresponds to the ``labels`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - input_url (:class:`str`): + input_url (str): Required. The full resource URL of the external storage location. Currently, only Google Cloud Storage is supported. So input_url should be of the form: @@ -564,20 +580,21 @@ def import_entities( For more information, see [google.datastore.admin.v1.ExportEntitiesResponse.output_url][google.datastore.admin.v1.ExportEntitiesResponse.output_url]. + This corresponds to the ``input_url`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - entity_filter (:class:`~.datastore_admin.EntityFilter`): + entity_filter (google.cloud.datastore_admin_v1.types.EntityFilter): Optionally specify which kinds/namespaces are to be imported. If provided, the list must be a subset of the EntityFilter used in creating the export, otherwise a FAILED_PRECONDITION error will be returned. If no filter is specified then all entities from the export are imported. + This corresponds to the ``entity_filter`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -585,24 +602,22 @@ def import_entities( sent along with the request as metadata. Returns: - ~.operation.Operation: + google.api_core.operation.Operation: An object representing a long-running operation. - The result type for the operation will be - :class:``~.empty.Empty``: A generic empty message that - you can re-use to avoid defining duplicated empty - messages in your APIs. A typical example is to use it as - the request or the response type of an API method. For - instance: + The result type for the operation will be :class:`google.protobuf.empty_pb2.Empty` A generic empty message that you can re-use to avoid defining duplicated + empty messages in your APIs. A typical example is to + use it as the request or the response type of an API + method. For instance: - :: + service Foo { + rpc Bar(google.protobuf.Empty) returns + (google.protobuf.Empty); - service Foo { - rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); - } + } - The JSON representation for ``Empty`` is empty JSON - object ``{}``. + The JSON representation for Empty is empty JSON + object {}. """ # Create or coerce a protobuf request object. @@ -621,20 +636,17 @@ def import_entities( # there are no flattened fields. if not isinstance(request, datastore_admin.ImportEntitiesRequest): request = datastore_admin.ImportEntitiesRequest(request) - # If we have keyword arguments corresponding to fields on the # request, apply these. - if project_id is not None: request.project_id = project_id + if labels is not None: + request.labels = labels if input_url is not None: request.input_url = input_url if entity_filter is not None: request.entity_filter = entity_filter - if labels: - request.labels.update(labels) - # Wrap the RPC method; this adds retry and timeout information, # and friendly error handling. rpc = self._transport._wrapped_methods[self._transport.import_entities] @@ -646,28 +658,164 @@ def import_entities( response = operation.from_gapic( response, self._transport.operations_client, - empty.Empty, + empty_pb2.Empty, metadata_type=datastore_admin.ImportEntitiesMetadata, ) # Done; return the response. return response + def create_index( + self, + request: Union[datastore_admin.CreateIndexRequest, dict] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> operation.Operation: + r"""Creates the specified index. A newly created index's initial + state is ``CREATING``. On completion of the returned + [google.longrunning.Operation][google.longrunning.Operation], + the state will be ``READY``. If the index already exists, the + call will return an ``ALREADY_EXISTS`` status. + + During index creation, the process could result in an error, in + which case the index will move to the ``ERROR`` state. The + process can be recovered by fixing the data that caused the + error, removing the index with + [delete][google.datastore.admin.v1.DatastoreAdmin.DeleteIndex], + then re-creating the index with [create] + [google.datastore.admin.v1.DatastoreAdmin.CreateIndex]. + + Indexes with a single property cannot be created. + + Args: + request (Union[google.cloud.datastore_admin_v1.types.CreateIndexRequest, dict]): + The request object. The request for + [google.datastore.admin.v1.DatastoreAdmin.CreateIndex][google.datastore.admin.v1.DatastoreAdmin.CreateIndex]. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.api_core.operation.Operation: + An object representing a long-running operation. + + The result type for the operation will be + :class:`google.cloud.datastore_admin_v1.types.Index` + Datastore composite index definition. + + """ + # Create or coerce a protobuf request object. + # Minor optimization to avoid making a copy if the user passes + # in a datastore_admin.CreateIndexRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, datastore_admin.CreateIndexRequest): + request = datastore_admin.CreateIndexRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.create_index] + + # Send the request. + response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Wrap the response in an operation future. + response = operation.from_gapic( + response, + self._transport.operations_client, + index.Index, + metadata_type=datastore_admin.IndexOperationMetadata, + ) + + # Done; return the response. + return response + + def delete_index( + self, + request: Union[datastore_admin.DeleteIndexRequest, dict] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: float = None, + metadata: Sequence[Tuple[str, str]] = (), + ) -> operation.Operation: + r"""Deletes an existing index. An index can only be deleted if it is + in a ``READY`` or ``ERROR`` state. On successful execution of + the request, the index will be in a ``DELETING`` + [state][google.datastore.admin.v1.Index.State]. And on + completion of the returned + [google.longrunning.Operation][google.longrunning.Operation], + the index will be removed. + + During index deletion, the process could result in an error, in + which case the index will move to the ``ERROR`` state. The + process can be recovered by fixing the data that caused the + error, followed by calling + [delete][google.datastore.admin.v1.DatastoreAdmin.DeleteIndex] + again. + + Args: + request (Union[google.cloud.datastore_admin_v1.types.DeleteIndexRequest, dict]): + The request object. The request for + [google.datastore.admin.v1.DatastoreAdmin.DeleteIndex][google.datastore.admin.v1.DatastoreAdmin.DeleteIndex]. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.api_core.operation.Operation: + An object representing a long-running operation. + + The result type for the operation will be + :class:`google.cloud.datastore_admin_v1.types.Index` + Datastore composite index definition. + + """ + # Create or coerce a protobuf request object. + # Minor optimization to avoid making a copy if the user passes + # in a datastore_admin.DeleteIndexRequest. + # There's no risk of modifying the input as we've already verified + # there are no flattened fields. + if not isinstance(request, datastore_admin.DeleteIndexRequest): + request = datastore_admin.DeleteIndexRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.delete_index] + + # Send the request. + response = rpc(request, retry=retry, timeout=timeout, metadata=metadata,) + + # Wrap the response in an operation future. + response = operation.from_gapic( + response, + self._transport.operations_client, + index.Index, + metadata_type=datastore_admin.IndexOperationMetadata, + ) + + # Done; return the response. + return response + def get_index( self, - request: datastore_admin.GetIndexRequest = None, + request: Union[datastore_admin.GetIndexRequest, dict] = None, *, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> index.Index: r"""Gets an index. Args: - request (:class:`~.datastore_admin.GetIndexRequest`): + request (Union[google.cloud.datastore_admin_v1.types.GetIndexRequest, dict]): The request object. The request for [google.datastore.admin.v1.DatastoreAdmin.GetIndex][google.datastore.admin.v1.DatastoreAdmin.GetIndex]. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -675,11 +823,10 @@ def get_index( sent along with the request as metadata. Returns: - ~.index.Index: - A minimal index definition. + google.cloud.datastore_admin_v1.types.Index: + Datastore composite index definition. """ # Create or coerce a protobuf request object. - # Minor optimization to avoid making a copy if the user passes # in a datastore_admin.GetIndexRequest. # There's no risk of modifying the input as we've already verified @@ -699,9 +846,9 @@ def get_index( def list_indexes( self, - request: datastore_admin.ListIndexesRequest = None, + request: Union[datastore_admin.ListIndexesRequest, dict] = None, *, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> pagers.ListIndexesPager: @@ -711,10 +858,9 @@ def list_indexes( results. Args: - request (:class:`~.datastore_admin.ListIndexesRequest`): + request (Union[google.cloud.datastore_admin_v1.types.ListIndexesRequest, dict]): The request object. The request for [google.datastore.admin.v1.DatastoreAdmin.ListIndexes][google.datastore.admin.v1.DatastoreAdmin.ListIndexes]. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -722,16 +868,15 @@ def list_indexes( sent along with the request as metadata. Returns: - ~.pagers.ListIndexesPager: + google.cloud.datastore_admin_v1.services.datastore_admin.pagers.ListIndexesPager: The response for - [google.datastore.admin.v1.DatastoreAdmin.ListIndexes][google.datastore.admin.v1.DatastoreAdmin.ListIndexes]. + [google.datastore.admin.v1.DatastoreAdmin.ListIndexes][google.datastore.admin.v1.DatastoreAdmin.ListIndexes]. Iterating over this object will yield results and resolve additional pages automatically. """ # Create or coerce a protobuf request object. - # Minor optimization to avoid making a copy if the user passes # in a datastore_admin.ListIndexesRequest. # There's no risk of modifying the input as we've already verified @@ -755,6 +900,19 @@ def list_indexes( # Done; return the response. return response + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + """Releases underlying transport's resources. + + .. warning:: + ONLY use as a context manager if the transport is NOT shared + with other clients! Exiting the with block will CLOSE the transport + and may cause errors in other clients! + """ + self.transport.close() + try: DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( diff --git a/google/cloud/datastore_admin_v1/services/datastore_admin/pagers.py b/google/cloud/datastore_admin_v1/services/datastore_admin/pagers.py index 7c176fce..a2f14858 100644 --- a/google/cloud/datastore_admin_v1/services/datastore_admin/pagers.py +++ b/google/cloud/datastore_admin_v1/services/datastore_admin/pagers.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,8 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # - -from typing import Any, AsyncIterable, Awaitable, Callable, Iterable, Sequence, Tuple +from typing import ( + Any, + AsyncIterator, + Awaitable, + Callable, + Sequence, + Tuple, + Optional, + Iterator, +) from google.cloud.datastore_admin_v1.types import datastore_admin from google.cloud.datastore_admin_v1.types import index @@ -25,7 +32,7 @@ class ListIndexesPager: """A pager for iterating through ``list_indexes`` requests. This class thinly wraps an initial - :class:`~.datastore_admin.ListIndexesResponse` object, and + :class:`google.cloud.datastore_admin_v1.types.ListIndexesResponse` object, and provides an ``__iter__`` method to iterate through its ``indexes`` field. @@ -34,7 +41,7 @@ class ListIndexesPager: through the ``indexes`` field on the corresponding responses. - All the usual :class:`~.datastore_admin.ListIndexesResponse` + All the usual :class:`google.cloud.datastore_admin_v1.types.ListIndexesResponse` attributes are available on the pager. If multiple requests are made, only the most recent response is retained, and thus used for attribute lookup. """ @@ -52,9 +59,9 @@ def __init__( Args: method (Callable): The method that was originally called, and which instantiated this pager. - request (:class:`~.datastore_admin.ListIndexesRequest`): + request (google.cloud.datastore_admin_v1.types.ListIndexesRequest): The initial request object. - response (:class:`~.datastore_admin.ListIndexesResponse`): + response (google.cloud.datastore_admin_v1.types.ListIndexesResponse): The initial response object. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. @@ -68,14 +75,14 @@ def __getattr__(self, name: str) -> Any: return getattr(self._response, name) @property - def pages(self) -> Iterable[datastore_admin.ListIndexesResponse]: + def pages(self) -> Iterator[datastore_admin.ListIndexesResponse]: yield self._response while self._response.next_page_token: self._request.page_token = self._response.next_page_token self._response = self._method(self._request, metadata=self._metadata) yield self._response - def __iter__(self) -> Iterable[index.Index]: + def __iter__(self) -> Iterator[index.Index]: for page in self.pages: yield from page.indexes @@ -87,7 +94,7 @@ class ListIndexesAsyncPager: """A pager for iterating through ``list_indexes`` requests. This class thinly wraps an initial - :class:`~.datastore_admin.ListIndexesResponse` object, and + :class:`google.cloud.datastore_admin_v1.types.ListIndexesResponse` object, and provides an ``__aiter__`` method to iterate through its ``indexes`` field. @@ -96,7 +103,7 @@ class ListIndexesAsyncPager: through the ``indexes`` field on the corresponding responses. - All the usual :class:`~.datastore_admin.ListIndexesResponse` + All the usual :class:`google.cloud.datastore_admin_v1.types.ListIndexesResponse` attributes are available on the pager. If multiple requests are made, only the most recent response is retained, and thus used for attribute lookup. """ @@ -109,14 +116,14 @@ def __init__( *, metadata: Sequence[Tuple[str, str]] = () ): - """Instantiate the pager. + """Instantiates the pager. Args: method (Callable): The method that was originally called, and which instantiated this pager. - request (:class:`~.datastore_admin.ListIndexesRequest`): + request (google.cloud.datastore_admin_v1.types.ListIndexesRequest): The initial request object. - response (:class:`~.datastore_admin.ListIndexesResponse`): + response (google.cloud.datastore_admin_v1.types.ListIndexesResponse): The initial response object. metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the request as metadata. @@ -130,14 +137,14 @@ def __getattr__(self, name: str) -> Any: return getattr(self._response, name) @property - async def pages(self) -> AsyncIterable[datastore_admin.ListIndexesResponse]: + async def pages(self) -> AsyncIterator[datastore_admin.ListIndexesResponse]: yield self._response while self._response.next_page_token: self._request.page_token = self._response.next_page_token self._response = await self._method(self._request, metadata=self._metadata) yield self._response - def __aiter__(self) -> AsyncIterable[index.Index]: + def __aiter__(self) -> AsyncIterator[index.Index]: async def async_generator(): async for page in self.pages: for response in page.indexes: diff --git a/google/cloud/datastore_admin_v1/services/datastore_admin/transports/__init__.py b/google/cloud/datastore_admin_v1/services/datastore_admin/transports/__init__.py index 41b72bc3..376bbfa1 100644 --- a/google/cloud/datastore_admin_v1/services/datastore_admin/transports/__init__.py +++ b/google/cloud/datastore_admin_v1/services/datastore_admin/transports/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # - from collections import OrderedDict from typing import Dict, Type @@ -28,7 +26,6 @@ _transport_registry["grpc"] = DatastoreAdminGrpcTransport _transport_registry["grpc_asyncio"] = DatastoreAdminGrpcAsyncIOTransport - __all__ = ( "DatastoreAdminTransport", "DatastoreAdminGrpcTransport", diff --git a/google/cloud/datastore_admin_v1/services/datastore_admin/transports/base.py b/google/cloud/datastore_admin_v1/services/datastore_admin/transports/base.py index d2a8b621..8fc75028 100644 --- a/google/cloud/datastore_admin_v1/services/datastore_admin/transports/base.py +++ b/google/cloud/datastore_admin_v1/services/datastore_admin/transports/base.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,22 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import abc -import typing +from typing import Awaitable, Callable, Dict, Optional, Sequence, Union import pkg_resources -from google import auth # type: ignore -from google.api_core import exceptions # type: ignore +import google.auth # type: ignore +import google.api_core # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore from google.api_core import gapic_v1 # type: ignore from google.api_core import retry as retries # type: ignore from google.api_core import operations_v1 # type: ignore -from google.auth import credentials # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore from google.cloud.datastore_admin_v1.types import datastore_admin from google.cloud.datastore_admin_v1.types import index -from google.longrunning import operations_pb2 as operations # type: ignore - +from google.longrunning import operations_pb2 # type: ignore try: DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( @@ -49,21 +48,25 @@ class DatastoreAdminTransport(abc.ABC): "https://www.googleapis.com/auth/datastore", ) + DEFAULT_HOST: str = "datastore.googleapis.com" + def __init__( self, *, - host: str = "datastore.googleapis.com", - credentials: credentials.Credentials = None, - credentials_file: typing.Optional[str] = None, - scopes: typing.Optional[typing.Sequence[str]] = AUTH_SCOPES, - quota_project_id: typing.Optional[str] = None, + host: str = DEFAULT_HOST, + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, **kwargs, ) -> None: """Instantiate the transport. Args: - host (Optional[str]): The hostname to connect to. + host (Optional[str]): + The hostname to connect to. credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none @@ -72,43 +75,55 @@ def __init__( credentials_file (Optional[str]): A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is mutually exclusive with credentials. - scope (Optional[Sequence[str]]): A list of scopes. + scopes (Optional[Sequence[str]]): A list of scopes. quota_project_id (Optional[str]): An optional project to use for billing and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. """ # Save the hostname. Default to port 443 (HTTPS) if none is specified. if ":" not in host: host += ":443" self._host = host + scopes_kwargs = {"scopes": scopes, "default_scopes": self.AUTH_SCOPES} + + # Save the scopes. + self._scopes = scopes + # If no credentials are provided, then determine the appropriate # defaults. if credentials and credentials_file: - raise exceptions.DuplicateCredentialArgs( + raise core_exceptions.DuplicateCredentialArgs( "'credentials_file' and 'credentials' are mutually exclusive" ) if credentials_file is not None: - credentials, _ = auth.load_credentials_from_file( - credentials_file, scopes=scopes, quota_project_id=quota_project_id + credentials, _ = google.auth.load_credentials_from_file( + credentials_file, **scopes_kwargs, quota_project_id=quota_project_id ) elif credentials is None: - credentials, _ = auth.default( - scopes=scopes, quota_project_id=quota_project_id + credentials, _ = google.auth.default( + **scopes_kwargs, quota_project_id=quota_project_id ) + # If the credentials are service account credentials, then always try to use self signed JWT. + if ( + always_use_jwt_access + and isinstance(credentials, service_account.Credentials) + and hasattr(service_account.Credentials, "with_always_use_jwt_access") + ): + credentials = credentials.with_always_use_jwt_access(True) + # Save the credentials. self._credentials = credentials - # Lifted into its own function so it can be stubbed out during tests. - self._prep_wrapped_messages(client_info) - def _prep_wrapped_messages(self, client_info): # Precompute the wrapped methods. self._wrapped_methods = { @@ -118,6 +133,12 @@ def _prep_wrapped_messages(self, client_info): self.import_entities: gapic_v1.method.wrap_method( self.import_entities, default_timeout=60.0, client_info=client_info, ), + self.create_index: gapic_v1.method.wrap_method( + self.create_index, default_timeout=60.0, client_info=client_info, + ), + self.delete_index: gapic_v1.method.wrap_method( + self.delete_index, default_timeout=60.0, client_info=client_info, + ), self.get_index: gapic_v1.method.wrap_method( self.get_index, default_retry=retries.Retry( @@ -125,8 +146,10 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( - exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, ), + deadline=60.0, ), default_timeout=60.0, client_info=client_info, @@ -138,54 +161,82 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( - exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, ), + deadline=60.0, ), default_timeout=60.0, client_info=client_info, ), } + def close(self): + """Closes resources associated with the transport. + + .. warning:: + Only call this method if the transport is NOT shared + with other clients - this may cause errors in other clients! + """ + raise NotImplementedError() + @property - def operations_client(self) -> operations_v1.OperationsClient: + def operations_client(self): """Return the client designed to process long-running operations.""" raise NotImplementedError() @property def export_entities( self, - ) -> typing.Callable[ + ) -> Callable[ [datastore_admin.ExportEntitiesRequest], - typing.Union[operations.Operation, typing.Awaitable[operations.Operation]], + Union[operations_pb2.Operation, Awaitable[operations_pb2.Operation]], ]: raise NotImplementedError() @property def import_entities( self, - ) -> typing.Callable[ + ) -> Callable[ [datastore_admin.ImportEntitiesRequest], - typing.Union[operations.Operation, typing.Awaitable[operations.Operation]], + Union[operations_pb2.Operation, Awaitable[operations_pb2.Operation]], + ]: + raise NotImplementedError() + + @property + def create_index( + self, + ) -> Callable[ + [datastore_admin.CreateIndexRequest], + Union[operations_pb2.Operation, Awaitable[operations_pb2.Operation]], + ]: + raise NotImplementedError() + + @property + def delete_index( + self, + ) -> Callable[ + [datastore_admin.DeleteIndexRequest], + Union[operations_pb2.Operation, Awaitable[operations_pb2.Operation]], ]: raise NotImplementedError() @property def get_index( self, - ) -> typing.Callable[ - [datastore_admin.GetIndexRequest], - typing.Union[index.Index, typing.Awaitable[index.Index]], + ) -> Callable[ + [datastore_admin.GetIndexRequest], Union[index.Index, Awaitable[index.Index]] ]: raise NotImplementedError() @property def list_indexes( self, - ) -> typing.Callable[ + ) -> Callable[ [datastore_admin.ListIndexesRequest], - typing.Union[ + Union[ datastore_admin.ListIndexesResponse, - typing.Awaitable[datastore_admin.ListIndexesResponse], + Awaitable[datastore_admin.ListIndexesResponse], ], ]: raise NotImplementedError() diff --git a/google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc.py b/google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc.py index 498a6a53..07db8479 100644 --- a/google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc.py +++ b/google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,23 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import warnings -from typing import Callable, Dict, Optional, Sequence, Tuple +from typing import Callable, Dict, Optional, Sequence, Tuple, Union from google.api_core import grpc_helpers # type: ignore from google.api_core import operations_v1 # type: ignore from google.api_core import gapic_v1 # type: ignore -from google import auth # type: ignore -from google.auth import credentials # type: ignore +import google.auth # type: ignore +from google.auth import credentials as ga_credentials # type: ignore from google.auth.transport.grpc import SslCredentials # type: ignore import grpc # type: ignore from google.cloud.datastore_admin_v1.types import datastore_admin from google.cloud.datastore_admin_v1.types import index -from google.longrunning import operations_pb2 as operations # type: ignore - +from google.longrunning import operations_pb2 # type: ignore from .base import DatastoreAdminTransport, DEFAULT_CLIENT_INFO @@ -110,20 +107,23 @@ def __init__( self, *, host: str = "datastore.googleapis.com", - credentials: credentials.Credentials = None, + credentials: ga_credentials.Credentials = None, credentials_file: str = None, scopes: Sequence[str] = None, channel: grpc.Channel = None, api_mtls_endpoint: str = None, client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, ssl_channel_credentials: grpc.ChannelCredentials = None, + client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, ) -> None: """Instantiate the transport. Args: - host (Optional[str]): The hostname to connect to. + host (Optional[str]): + The hostname to connect to. credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none @@ -140,13 +140,17 @@ def __init__( api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. If provided, it overrides the ``host`` argument and tries to create a mutual TLS channel with client SSL credentials from - ``client_cert_source`` or applicatin default SSL credentials. + ``client_cert_source`` or application default SSL credentials. client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): Deprecated. A callback to provide client SSL certificate bytes and private key bytes, both in PEM format. It is ignored if ``api_mtls_endpoint`` is None. ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials - for grpc channel. It is ignored if ``channel`` is provided. + for the grpc channel. It is ignored if ``channel`` is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure a mutual TLS channel. It is + ignored if ``channel`` or ``ssl_channel_credentials`` is provided. quota_project_id (Optional[str]): An optional project to use for billing and quota. client_info (google.api_core.gapic_v1.client_info.ClientInfo): @@ -154,6 +158,8 @@ def __init__( API requests. If ``None``, then default info will be used. Generally, you only need to set this if you're developing your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. Raises: google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport @@ -161,88 +167,77 @@ def __init__( google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` and ``credentials_file`` are passed. """ + self._grpc_channel = None self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + self._operations_client: Optional[operations_v1.OperationsClient] = None + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) if channel: - # Sanity check: Ensure that channel and credentials are not both - # provided. + # Ignore credentials if a channel was passed. credentials = False - # If a channel was explicitly provided, set it. self._grpc_channel = channel self._ssl_channel_credentials = None - elif api_mtls_endpoint: - warnings.warn( - "api_mtls_endpoint and client_cert_source are deprecated", - DeprecationWarning, - ) - - host = ( - api_mtls_endpoint - if ":" in api_mtls_endpoint - else api_mtls_endpoint + ":443" - ) - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) - - # Create SSL credentials with client_cert_source or application - # default SSL credentials. - if client_cert_source: - cert, key = client_cert_source() - ssl_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - else: - ssl_credentials = SslCredentials().ssl_credentials - - # create a new channel. The provided one is ignored. - self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - ssl_credentials=ssl_credentials, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - ) - self._ssl_channel_credentials = ssl_credentials else: - host = host if ":" in host else host + ":443" - - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) - - # create a new channel. The provided one is ignored. - self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - ssl_credentials=ssl_channel_credentials, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - ) + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials - self._stubs = {} # type: Dict[str, Callable] + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) - # Run the base constructor. + # The base transport sets the host, credentials and scopes super().__init__( host=host, credentials=credentials, credentials_file=credentials_file, - scopes=scopes or self.AUTH_SCOPES, + scopes=scopes, quota_project_id=quota_project_id, client_info=client_info, + always_use_jwt_access=always_use_jwt_access, ) + if not self._grpc_channel: + self._grpc_channel = type(self).create_channel( + self._host, + credentials=self._credentials, + credentials_file=credentials_file, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) + @classmethod def create_channel( cls, host: str = "datastore.googleapis.com", - credentials: credentials.Credentials = None, + credentials: ga_credentials.Credentials = None, credentials_file: str = None, scopes: Optional[Sequence[str]] = None, quota_project_id: Optional[str] = None, @@ -250,7 +245,7 @@ def create_channel( ) -> grpc.Channel: """Create and return a gRPC channel object. Args: - address (Optionsl[str]): The host for the channel to use. + host (Optional[str]): The host for the channel to use. credentials (Optional[~.Credentials]): The authorization credentials to attach to requests. These credentials identify this application to the service. If @@ -273,13 +268,15 @@ def create_channel( google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` and ``credentials_file`` are passed. """ - scopes = scopes or cls.AUTH_SCOPES + return grpc_helpers.create_channel( host, credentials=credentials, credentials_file=credentials_file, - scopes=scopes, quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, **kwargs, ) @@ -297,18 +294,16 @@ def operations_client(self) -> operations_v1.OperationsClient: client. """ # Sanity check: Only create a new client if we do not already have one. - if "operations_client" not in self.__dict__: - self.__dict__["operations_client"] = operations_v1.OperationsClient( - self.grpc_channel - ) + if self._operations_client is None: + self._operations_client = operations_v1.OperationsClient(self.grpc_channel) # Return the client from cache. - return self.__dict__["operations_client"] + return self._operations_client @property def export_entities( self, - ) -> Callable[[datastore_admin.ExportEntitiesRequest], operations.Operation]: + ) -> Callable[[datastore_admin.ExportEntitiesRequest], operations_pb2.Operation]: r"""Return a callable for the export entities method over gRPC. Exports a copy of all or a subset of entities from @@ -336,14 +331,14 @@ def export_entities( self._stubs["export_entities"] = self.grpc_channel.unary_unary( "/google.datastore.admin.v1.DatastoreAdmin/ExportEntities", request_serializer=datastore_admin.ExportEntitiesRequest.serialize, - response_deserializer=operations.Operation.FromString, + response_deserializer=operations_pb2.Operation.FromString, ) return self._stubs["export_entities"] @property def import_entities( self, - ) -> Callable[[datastore_admin.ImportEntitiesRequest], operations.Operation]: + ) -> Callable[[datastore_admin.ImportEntitiesRequest], operations_pb2.Operation]: r"""Return a callable for the import entities method over gRPC. Imports entities into Google Cloud Datastore. @@ -368,10 +363,89 @@ def import_entities( self._stubs["import_entities"] = self.grpc_channel.unary_unary( "/google.datastore.admin.v1.DatastoreAdmin/ImportEntities", request_serializer=datastore_admin.ImportEntitiesRequest.serialize, - response_deserializer=operations.Operation.FromString, + response_deserializer=operations_pb2.Operation.FromString, ) return self._stubs["import_entities"] + @property + def create_index( + self, + ) -> Callable[[datastore_admin.CreateIndexRequest], operations_pb2.Operation]: + r"""Return a callable for the create index method over gRPC. + + Creates the specified index. A newly created index's initial + state is ``CREATING``. On completion of the returned + [google.longrunning.Operation][google.longrunning.Operation], + the state will be ``READY``. If the index already exists, the + call will return an ``ALREADY_EXISTS`` status. + + During index creation, the process could result in an error, in + which case the index will move to the ``ERROR`` state. The + process can be recovered by fixing the data that caused the + error, removing the index with + [delete][google.datastore.admin.v1.DatastoreAdmin.DeleteIndex], + then re-creating the index with [create] + [google.datastore.admin.v1.DatastoreAdmin.CreateIndex]. + + Indexes with a single property cannot be created. + + Returns: + Callable[[~.CreateIndexRequest], + ~.Operation]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "create_index" not in self._stubs: + self._stubs["create_index"] = self.grpc_channel.unary_unary( + "/google.datastore.admin.v1.DatastoreAdmin/CreateIndex", + request_serializer=datastore_admin.CreateIndexRequest.serialize, + response_deserializer=operations_pb2.Operation.FromString, + ) + return self._stubs["create_index"] + + @property + def delete_index( + self, + ) -> Callable[[datastore_admin.DeleteIndexRequest], operations_pb2.Operation]: + r"""Return a callable for the delete index method over gRPC. + + Deletes an existing index. An index can only be deleted if it is + in a ``READY`` or ``ERROR`` state. On successful execution of + the request, the index will be in a ``DELETING`` + [state][google.datastore.admin.v1.Index.State]. And on + completion of the returned + [google.longrunning.Operation][google.longrunning.Operation], + the index will be removed. + + During index deletion, the process could result in an error, in + which case the index will move to the ``ERROR`` state. The + process can be recovered by fixing the data that caused the + error, followed by calling + [delete][google.datastore.admin.v1.DatastoreAdmin.DeleteIndex] + again. + + Returns: + Callable[[~.DeleteIndexRequest], + ~.Operation]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "delete_index" not in self._stubs: + self._stubs["delete_index"] = self.grpc_channel.unary_unary( + "/google.datastore.admin.v1.DatastoreAdmin/DeleteIndex", + request_serializer=datastore_admin.DeleteIndexRequest.serialize, + response_deserializer=operations_pb2.Operation.FromString, + ) + return self._stubs["delete_index"] + @property def get_index(self) -> Callable[[datastore_admin.GetIndexRequest], index.Index]: r"""Return a callable for the get index method over gRPC. @@ -427,5 +501,8 @@ def list_indexes( ) return self._stubs["list_indexes"] + def close(self): + self.grpc_channel.close() + __all__ = ("DatastoreAdminGrpcTransport",) diff --git a/google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc_asyncio.py b/google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc_asyncio.py index f731d4c0..8a1f1a54 100644 --- a/google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc_asyncio.py +++ b/google/cloud/datastore_admin_v1/services/datastore_admin/transports/grpc_asyncio.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,15 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import warnings -from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple +from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union from google.api_core import gapic_v1 # type: ignore from google.api_core import grpc_helpers_async # type: ignore from google.api_core import operations_v1 # type: ignore -from google import auth # type: ignore -from google.auth import credentials # type: ignore +from google.auth import credentials as ga_credentials # type: ignore from google.auth.transport.grpc import SslCredentials # type: ignore import grpc # type: ignore @@ -30,8 +27,7 @@ from google.cloud.datastore_admin_v1.types import datastore_admin from google.cloud.datastore_admin_v1.types import index -from google.longrunning import operations_pb2 as operations # type: ignore - +from google.longrunning import operations_pb2 # type: ignore from .base import DatastoreAdminTransport, DEFAULT_CLIENT_INFO from .grpc import DatastoreAdminGrpcTransport @@ -113,7 +109,7 @@ class DatastoreAdminGrpcAsyncIOTransport(DatastoreAdminTransport): def create_channel( cls, host: str = "datastore.googleapis.com", - credentials: credentials.Credentials = None, + credentials: ga_credentials.Credentials = None, credentials_file: Optional[str] = None, scopes: Optional[Sequence[str]] = None, quota_project_id: Optional[str] = None, @@ -121,7 +117,7 @@ def create_channel( ) -> aio.Channel: """Create and return a gRPC AsyncIO channel object. Args: - address (Optional[str]): The host for the channel to use. + host (Optional[str]): The host for the channel to use. credentials (Optional[~.Credentials]): The authorization credentials to attach to requests. These credentials identify this application to the service. If @@ -140,13 +136,15 @@ def create_channel( Returns: aio.Channel: A gRPC AsyncIO channel object. """ - scopes = scopes or cls.AUTH_SCOPES + return grpc_helpers_async.create_channel( host, credentials=credentials, credentials_file=credentials_file, - scopes=scopes, quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, **kwargs, ) @@ -154,20 +152,23 @@ def __init__( self, *, host: str = "datastore.googleapis.com", - credentials: credentials.Credentials = None, + credentials: ga_credentials.Credentials = None, credentials_file: Optional[str] = None, scopes: Optional[Sequence[str]] = None, channel: aio.Channel = None, api_mtls_endpoint: str = None, client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, ssl_channel_credentials: grpc.ChannelCredentials = None, + client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, quota_project_id=None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, ) -> None: """Instantiate the transport. Args: - host (Optional[str]): The hostname to connect to. + host (Optional[str]): + The hostname to connect to. credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none @@ -185,20 +186,26 @@ def __init__( api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. If provided, it overrides the ``host`` argument and tries to create a mutual TLS channel with client SSL credentials from - ``client_cert_source`` or applicatin default SSL credentials. + ``client_cert_source`` or application default SSL credentials. client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): Deprecated. A callback to provide client SSL certificate bytes and private key bytes, both in PEM format. It is ignored if ``api_mtls_endpoint`` is None. ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials - for grpc channel. It is ignored if ``channel`` is provided. + for the grpc channel. It is ignored if ``channel`` is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure a mutual TLS channel. It is + ignored if ``channel`` or ``ssl_channel_credentials`` is provided. quota_project_id (Optional[str]): An optional project to use for billing and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. Raises: google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport @@ -206,82 +213,70 @@ def __init__( google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` and ``credentials_file`` are passed. """ + self._grpc_channel = None self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + self._operations_client: Optional[operations_v1.OperationsAsyncClient] = None + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) if channel: - # Sanity check: Ensure that channel and credentials are not both - # provided. + # Ignore credentials if a channel was passed. credentials = False - # If a channel was explicitly provided, set it. self._grpc_channel = channel self._ssl_channel_credentials = None - elif api_mtls_endpoint: - warnings.warn( - "api_mtls_endpoint and client_cert_source are deprecated", - DeprecationWarning, - ) - - host = ( - api_mtls_endpoint - if ":" in api_mtls_endpoint - else api_mtls_endpoint + ":443" - ) - - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) - - # Create SSL credentials with client_cert_source or application - # default SSL credentials. - if client_cert_source: - cert, key = client_cert_source() - ssl_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - else: - ssl_credentials = SslCredentials().ssl_credentials - - # create a new channel. The provided one is ignored. - self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - ssl_credentials=ssl_credentials, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - ) - self._ssl_channel_credentials = ssl_credentials else: - host = host if ":" in host else host + ":443" - - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials - # create a new channel. The provided one is ignored. - self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - ssl_credentials=ssl_channel_credentials, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - ) + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) - # Run the base constructor. + # The base transport sets the host, credentials and scopes super().__init__( host=host, credentials=credentials, credentials_file=credentials_file, - scopes=scopes or self.AUTH_SCOPES, + scopes=scopes, quota_project_id=quota_project_id, client_info=client_info, + always_use_jwt_access=always_use_jwt_access, ) - self._stubs = {} + if not self._grpc_channel: + self._grpc_channel = type(self).create_channel( + self._host, + credentials=self._credentials, + credentials_file=credentials_file, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) @property def grpc_channel(self) -> aio.Channel: @@ -301,19 +296,19 @@ def operations_client(self) -> operations_v1.OperationsAsyncClient: client. """ # Sanity check: Only create a new client if we do not already have one. - if "operations_client" not in self.__dict__: - self.__dict__["operations_client"] = operations_v1.OperationsAsyncClient( + if self._operations_client is None: + self._operations_client = operations_v1.OperationsAsyncClient( self.grpc_channel ) # Return the client from cache. - return self.__dict__["operations_client"] + return self._operations_client @property def export_entities( self, ) -> Callable[ - [datastore_admin.ExportEntitiesRequest], Awaitable[operations.Operation] + [datastore_admin.ExportEntitiesRequest], Awaitable[operations_pb2.Operation] ]: r"""Return a callable for the export entities method over gRPC. @@ -342,7 +337,7 @@ def export_entities( self._stubs["export_entities"] = self.grpc_channel.unary_unary( "/google.datastore.admin.v1.DatastoreAdmin/ExportEntities", request_serializer=datastore_admin.ExportEntitiesRequest.serialize, - response_deserializer=operations.Operation.FromString, + response_deserializer=operations_pb2.Operation.FromString, ) return self._stubs["export_entities"] @@ -350,7 +345,7 @@ def export_entities( def import_entities( self, ) -> Callable[ - [datastore_admin.ImportEntitiesRequest], Awaitable[operations.Operation] + [datastore_admin.ImportEntitiesRequest], Awaitable[operations_pb2.Operation] ]: r"""Return a callable for the import entities method over gRPC. @@ -376,10 +371,93 @@ def import_entities( self._stubs["import_entities"] = self.grpc_channel.unary_unary( "/google.datastore.admin.v1.DatastoreAdmin/ImportEntities", request_serializer=datastore_admin.ImportEntitiesRequest.serialize, - response_deserializer=operations.Operation.FromString, + response_deserializer=operations_pb2.Operation.FromString, ) return self._stubs["import_entities"] + @property + def create_index( + self, + ) -> Callable[ + [datastore_admin.CreateIndexRequest], Awaitable[operations_pb2.Operation] + ]: + r"""Return a callable for the create index method over gRPC. + + Creates the specified index. A newly created index's initial + state is ``CREATING``. On completion of the returned + [google.longrunning.Operation][google.longrunning.Operation], + the state will be ``READY``. If the index already exists, the + call will return an ``ALREADY_EXISTS`` status. + + During index creation, the process could result in an error, in + which case the index will move to the ``ERROR`` state. The + process can be recovered by fixing the data that caused the + error, removing the index with + [delete][google.datastore.admin.v1.DatastoreAdmin.DeleteIndex], + then re-creating the index with [create] + [google.datastore.admin.v1.DatastoreAdmin.CreateIndex]. + + Indexes with a single property cannot be created. + + Returns: + Callable[[~.CreateIndexRequest], + Awaitable[~.Operation]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "create_index" not in self._stubs: + self._stubs["create_index"] = self.grpc_channel.unary_unary( + "/google.datastore.admin.v1.DatastoreAdmin/CreateIndex", + request_serializer=datastore_admin.CreateIndexRequest.serialize, + response_deserializer=operations_pb2.Operation.FromString, + ) + return self._stubs["create_index"] + + @property + def delete_index( + self, + ) -> Callable[ + [datastore_admin.DeleteIndexRequest], Awaitable[operations_pb2.Operation] + ]: + r"""Return a callable for the delete index method over gRPC. + + Deletes an existing index. An index can only be deleted if it is + in a ``READY`` or ``ERROR`` state. On successful execution of + the request, the index will be in a ``DELETING`` + [state][google.datastore.admin.v1.Index.State]. And on + completion of the returned + [google.longrunning.Operation][google.longrunning.Operation], + the index will be removed. + + During index deletion, the process could result in an error, in + which case the index will move to the ``ERROR`` state. The + process can be recovered by fixing the data that caused the + error, followed by calling + [delete][google.datastore.admin.v1.DatastoreAdmin.DeleteIndex] + again. + + Returns: + Callable[[~.DeleteIndexRequest], + Awaitable[~.Operation]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if "delete_index" not in self._stubs: + self._stubs["delete_index"] = self.grpc_channel.unary_unary( + "/google.datastore.admin.v1.DatastoreAdmin/DeleteIndex", + request_serializer=datastore_admin.DeleteIndexRequest.serialize, + response_deserializer=operations_pb2.Operation.FromString, + ) + return self._stubs["delete_index"] + @property def get_index( self, @@ -438,5 +516,8 @@ def list_indexes( ) return self._stubs["list_indexes"] + def close(self): + return self.grpc_channel.close() + __all__ = ("DatastoreAdminGrpcAsyncIOTransport",) diff --git a/google/cloud/datastore_admin_v1/types/__init__.py b/google/cloud/datastore_admin_v1/types/__init__.py index b3bf63d8..ac4ff905 100644 --- a/google/cloud/datastore_admin_v1/types/__init__.py +++ b/google/cloud/datastore_admin_v1/types/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,36 +13,40 @@ # See the License for the specific language governing permissions and # limitations under the License. # - -from .index import Index from .datastore_admin import ( CommonMetadata, - Progress, + CreateIndexRequest, + DeleteIndexRequest, + EntityFilter, + ExportEntitiesMetadata, ExportEntitiesRequest, - ImportEntitiesRequest, ExportEntitiesResponse, - ExportEntitiesMetadata, - ImportEntitiesMetadata, - EntityFilter, GetIndexRequest, + ImportEntitiesMetadata, + ImportEntitiesRequest, + IndexOperationMetadata, ListIndexesRequest, ListIndexesResponse, - IndexOperationMetadata, + Progress, + OperationType, ) - +from .index import Index __all__ = ( - "Index", "CommonMetadata", - "Progress", + "CreateIndexRequest", + "DeleteIndexRequest", + "EntityFilter", + "ExportEntitiesMetadata", "ExportEntitiesRequest", - "ImportEntitiesRequest", "ExportEntitiesResponse", - "ExportEntitiesMetadata", - "ImportEntitiesMetadata", - "EntityFilter", "GetIndexRequest", + "ImportEntitiesMetadata", + "ImportEntitiesRequest", + "IndexOperationMetadata", "ListIndexesRequest", "ListIndexesResponse", - "IndexOperationMetadata", + "Progress", + "OperationType", + "Index", ) diff --git a/google/cloud/datastore_admin_v1/types/datastore_admin.py b/google/cloud/datastore_admin_v1/types/datastore_admin.py index 1fd3c8d5..0f4546fd 100644 --- a/google/cloud/datastore_admin_v1/types/datastore_admin.py +++ b/google/cloud/datastore_admin_v1/types/datastore_admin.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,12 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import proto # type: ignore - -from google.cloud.datastore_admin_v1.types import index -from google.protobuf import timestamp_pb2 as timestamp # type: ignore +from google.cloud.datastore_admin_v1.types import index as gda_index +from google.protobuf import timestamp_pb2 # type: ignore __protobuf__ = proto.module( @@ -34,6 +31,8 @@ "ExportEntitiesMetadata", "ImportEntitiesMetadata", "EntityFilter", + "CreateIndexRequest", + "DeleteIndexRequest", "GetIndexRequest", "ListIndexesRequest", "ListIndexesResponse", @@ -55,19 +54,19 @@ class CommonMetadata(proto.Message): r"""Metadata common to all Datastore Admin operations. Attributes: - start_time (~.timestamp.Timestamp): + start_time (google.protobuf.timestamp_pb2.Timestamp): The time that work began on the operation. - end_time (~.timestamp.Timestamp): + end_time (google.protobuf.timestamp_pb2.Timestamp): The time the operation ended, either successfully or otherwise. - operation_type (~.datastore_admin.OperationType): + operation_type (google.cloud.datastore_admin_v1.types.OperationType): The type of the operation. Can be used as a filter in ListOperationsRequest. - labels (Sequence[~.datastore_admin.CommonMetadata.LabelsEntry]): + labels (Sequence[google.cloud.datastore_admin_v1.types.CommonMetadata.LabelsEntry]): The client-assigned labels which were provided when the operation was created. May also include additional labels. - state (~.datastore_admin.CommonMetadata.State): + state (google.cloud.datastore_admin_v1.types.CommonMetadata.State): The current state of the Operation. """ @@ -82,14 +81,10 @@ class State(proto.Enum): FAILED = 6 CANCELLED = 7 - start_time = proto.Field(proto.MESSAGE, number=1, message=timestamp.Timestamp,) - - end_time = proto.Field(proto.MESSAGE, number=2, message=timestamp.Timestamp,) - + start_time = proto.Field(proto.MESSAGE, number=1, message=timestamp_pb2.Timestamp,) + end_time = proto.Field(proto.MESSAGE, number=2, message=timestamp_pb2.Timestamp,) operation_type = proto.Field(proto.ENUM, number=3, enum="OperationType",) - - labels = proto.MapField(proto.STRING, proto.STRING, number=4) - + labels = proto.MapField(proto.STRING, proto.STRING, number=4,) state = proto.Field(proto.ENUM, number=5, enum=State,) @@ -106,9 +101,8 @@ class Progress(proto.Message): unavailable. """ - work_completed = proto.Field(proto.INT64, number=1) - - work_estimated = proto.Field(proto.INT64, number=2) + work_completed = proto.Field(proto.INT64, number=1,) + work_estimated = proto.Field(proto.INT64, number=2,) class ExportEntitiesRequest(proto.Message): @@ -119,9 +113,9 @@ class ExportEntitiesRequest(proto.Message): project_id (str): Required. Project ID against which to make the request. - labels (Sequence[~.datastore_admin.ExportEntitiesRequest.LabelsEntry]): + labels (Sequence[google.cloud.datastore_admin_v1.types.ExportEntitiesRequest.LabelsEntry]): Client-assigned labels. - entity_filter (~.datastore_admin.EntityFilter): + entity_filter (google.cloud.datastore_admin_v1.types.EntityFilter): Description of what data from the project is included in the export. output_url_prefix (str): @@ -149,13 +143,10 @@ class ExportEntitiesRequest(proto.Message): without conflict. """ - project_id = proto.Field(proto.STRING, number=1) - - labels = proto.MapField(proto.STRING, proto.STRING, number=2) - + project_id = proto.Field(proto.STRING, number=1,) + labels = proto.MapField(proto.STRING, proto.STRING, number=2,) entity_filter = proto.Field(proto.MESSAGE, number=3, message="EntityFilter",) - - output_url_prefix = proto.Field(proto.STRING, number=4) + output_url_prefix = proto.Field(proto.STRING, number=4,) class ImportEntitiesRequest(proto.Message): @@ -166,7 +157,7 @@ class ImportEntitiesRequest(proto.Message): project_id (str): Required. Project ID against which to make the request. - labels (Sequence[~.datastore_admin.ImportEntitiesRequest.LabelsEntry]): + labels (Sequence[google.cloud.datastore_admin_v1.types.ImportEntitiesRequest.LabelsEntry]): Client-assigned labels. input_url (str): Required. The full resource URL of the external storage @@ -184,7 +175,7 @@ class ImportEntitiesRequest(proto.Message): For more information, see [google.datastore.admin.v1.ExportEntitiesResponse.output_url][google.datastore.admin.v1.ExportEntitiesResponse.output_url]. - entity_filter (~.datastore_admin.EntityFilter): + entity_filter (google.cloud.datastore_admin_v1.types.EntityFilter): Optionally specify which kinds/namespaces are to be imported. If provided, the list must be a subset of the EntityFilter used in creating the export, otherwise a @@ -192,12 +183,9 @@ class ImportEntitiesRequest(proto.Message): specified then all entities from the export are imported. """ - project_id = proto.Field(proto.STRING, number=1) - - labels = proto.MapField(proto.STRING, proto.STRING, number=2) - - input_url = proto.Field(proto.STRING, number=3) - + project_id = proto.Field(proto.STRING, number=1,) + labels = proto.MapField(proto.STRING, proto.STRING, number=2,) + input_url = proto.Field(proto.STRING, number=3,) entity_filter = proto.Field(proto.MESSAGE, number=4, message="EntityFilter",) @@ -214,22 +202,22 @@ class ExportEntitiesResponse(proto.Message): Only present if the operation completed successfully. """ - output_url = proto.Field(proto.STRING, number=1) + output_url = proto.Field(proto.STRING, number=1,) class ExportEntitiesMetadata(proto.Message): r"""Metadata for ExportEntities operations. Attributes: - common (~.datastore_admin.CommonMetadata): + common (google.cloud.datastore_admin_v1.types.CommonMetadata): Metadata common to all Datastore Admin operations. - progress_entities (~.datastore_admin.Progress): + progress_entities (google.cloud.datastore_admin_v1.types.Progress): An estimate of the number of entities processed. - progress_bytes (~.datastore_admin.Progress): + progress_bytes (google.cloud.datastore_admin_v1.types.Progress): An estimate of the number of bytes processed. - entity_filter (~.datastore_admin.EntityFilter): + entity_filter (google.cloud.datastore_admin_v1.types.EntityFilter): Description of which entities are being exported. output_url_prefix (str): @@ -241,29 +229,25 @@ class ExportEntitiesMetadata(proto.Message): """ common = proto.Field(proto.MESSAGE, number=1, message="CommonMetadata",) - progress_entities = proto.Field(proto.MESSAGE, number=2, message="Progress",) - progress_bytes = proto.Field(proto.MESSAGE, number=3, message="Progress",) - entity_filter = proto.Field(proto.MESSAGE, number=4, message="EntityFilter",) - - output_url_prefix = proto.Field(proto.STRING, number=5) + output_url_prefix = proto.Field(proto.STRING, number=5,) class ImportEntitiesMetadata(proto.Message): r"""Metadata for ImportEntities operations. Attributes: - common (~.datastore_admin.CommonMetadata): + common (google.cloud.datastore_admin_v1.types.CommonMetadata): Metadata common to all Datastore Admin operations. - progress_entities (~.datastore_admin.Progress): + progress_entities (google.cloud.datastore_admin_v1.types.Progress): An estimate of the number of entities processed. - progress_bytes (~.datastore_admin.Progress): + progress_bytes (google.cloud.datastore_admin_v1.types.Progress): An estimate of the number of bytes processed. - entity_filter (~.datastore_admin.EntityFilter): + entity_filter (google.cloud.datastore_admin_v1.types.EntityFilter): Description of which entities are being imported. input_url (str): @@ -274,14 +258,10 @@ class ImportEntitiesMetadata(proto.Message): """ common = proto.Field(proto.MESSAGE, number=1, message="CommonMetadata",) - progress_entities = proto.Field(proto.MESSAGE, number=2, message="Progress",) - progress_bytes = proto.Field(proto.MESSAGE, number=3, message="Progress",) - entity_filter = proto.Field(proto.MESSAGE, number=4, message="EntityFilter",) - - input_url = proto.Field(proto.STRING, number=5) + input_url = proto.Field(proto.STRING, number=5,) class EntityFilter(proto.Message): @@ -316,9 +296,41 @@ class EntityFilter(proto.Message): Each namespace in this list must be unique. """ - kinds = proto.RepeatedField(proto.STRING, number=1) + kinds = proto.RepeatedField(proto.STRING, number=1,) + namespace_ids = proto.RepeatedField(proto.STRING, number=2,) - namespace_ids = proto.RepeatedField(proto.STRING, number=2) + +class CreateIndexRequest(proto.Message): + r"""The request for + [google.datastore.admin.v1.DatastoreAdmin.CreateIndex][google.datastore.admin.v1.DatastoreAdmin.CreateIndex]. + + Attributes: + project_id (str): + Project ID against which to make the request. + index (google.cloud.datastore_admin_v1.types.Index): + The index to create. The name and state + fields are output only and will be ignored. + Single property indexes cannot be created or + deleted. + """ + + project_id = proto.Field(proto.STRING, number=1,) + index = proto.Field(proto.MESSAGE, number=3, message=gda_index.Index,) + + +class DeleteIndexRequest(proto.Message): + r"""The request for + [google.datastore.admin.v1.DatastoreAdmin.DeleteIndex][google.datastore.admin.v1.DatastoreAdmin.DeleteIndex]. + + Attributes: + project_id (str): + Project ID against which to make the request. + index_id (str): + The resource ID of the index to delete. + """ + + project_id = proto.Field(proto.STRING, number=1,) + index_id = proto.Field(proto.STRING, number=3,) class GetIndexRequest(proto.Message): @@ -332,9 +344,8 @@ class GetIndexRequest(proto.Message): The resource ID of the index to get. """ - project_id = proto.Field(proto.STRING, number=1) - - index_id = proto.Field(proto.STRING, number=3) + project_id = proto.Field(proto.STRING, number=1,) + index_id = proto.Field(proto.STRING, number=3,) class ListIndexesRequest(proto.Message): @@ -354,13 +365,10 @@ class ListIndexesRequest(proto.Message): request, if any. """ - project_id = proto.Field(proto.STRING, number=1) - - filter = proto.Field(proto.STRING, number=3) - - page_size = proto.Field(proto.INT32, number=4) - - page_token = proto.Field(proto.STRING, number=5) + project_id = proto.Field(proto.STRING, number=1,) + filter = proto.Field(proto.STRING, number=3,) + page_size = proto.Field(proto.INT32, number=4,) + page_token = proto.Field(proto.STRING, number=5,) class ListIndexesResponse(proto.Message): @@ -368,7 +376,7 @@ class ListIndexesResponse(proto.Message): [google.datastore.admin.v1.DatastoreAdmin.ListIndexes][google.datastore.admin.v1.DatastoreAdmin.ListIndexes]. Attributes: - indexes (Sequence[~.index.Index]): + indexes (Sequence[google.cloud.datastore_admin_v1.types.Index]): The indexes. next_page_token (str): The standard List next-page token. @@ -378,19 +386,18 @@ class ListIndexesResponse(proto.Message): def raw_page(self): return self - indexes = proto.RepeatedField(proto.MESSAGE, number=1, message=index.Index,) - - next_page_token = proto.Field(proto.STRING, number=2) + indexes = proto.RepeatedField(proto.MESSAGE, number=1, message=gda_index.Index,) + next_page_token = proto.Field(proto.STRING, number=2,) class IndexOperationMetadata(proto.Message): r"""Metadata for Index operations. Attributes: - common (~.datastore_admin.CommonMetadata): + common (google.cloud.datastore_admin_v1.types.CommonMetadata): Metadata common to all Datastore Admin operations. - progress_entities (~.datastore_admin.Progress): + progress_entities (google.cloud.datastore_admin_v1.types.Progress): An estimate of the number of entities processed. index_id (str): @@ -399,10 +406,8 @@ class IndexOperationMetadata(proto.Message): """ common = proto.Field(proto.MESSAGE, number=1, message="CommonMetadata",) - progress_entities = proto.Field(proto.MESSAGE, number=2, message="Progress",) - - index_id = proto.Field(proto.STRING, number=3) + index_id = proto.Field(proto.STRING, number=3,) __all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/datastore_admin_v1/types/index.py b/google/cloud/datastore_admin_v1/types/index.py index e11a27a5..b372cccf 100644 --- a/google/cloud/datastore_admin_v1/types/index.py +++ b/google/cloud/datastore_admin_v1/types/index.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import proto # type: ignore @@ -22,7 +20,7 @@ class Index(proto.Message): - r"""A minimal index definition. + r"""Datastore composite index definition. Attributes: project_id (str): @@ -32,13 +30,13 @@ class Index(proto.Message): kind (str): Required. The entity kind to which this index applies. - ancestor (~.index.Index.AncestorMode): + ancestor (google.cloud.datastore_admin_v1.types.Index.AncestorMode): Required. The index's ancestor mode. Must not be ANCESTOR_MODE_UNSPECIFIED. - properties (Sequence[~.index.Index.IndexedProperty]): + properties (Sequence[google.cloud.datastore_admin_v1.types.Index.IndexedProperty]): Required. An ordered sequence of property names and their index attributes. - state (~.index.Index.State): + state (google.cloud.datastore_admin_v1.types.Index.State): Output only. The state of the index. """ @@ -70,25 +68,19 @@ class IndexedProperty(proto.Message): Attributes: name (str): Required. The property name to index. - direction (~.index.Index.Direction): + direction (google.cloud.datastore_admin_v1.types.Index.Direction): Required. The indexed property's direction. Must not be DIRECTION_UNSPECIFIED. """ - name = proto.Field(proto.STRING, number=1) - + name = proto.Field(proto.STRING, number=1,) direction = proto.Field(proto.ENUM, number=2, enum="Index.Direction",) - project_id = proto.Field(proto.STRING, number=1) - - index_id = proto.Field(proto.STRING, number=3) - - kind = proto.Field(proto.STRING, number=4) - + project_id = proto.Field(proto.STRING, number=1,) + index_id = proto.Field(proto.STRING, number=3,) + kind = proto.Field(proto.STRING, number=4,) ancestor = proto.Field(proto.ENUM, number=5, enum=AncestorMode,) - properties = proto.RepeatedField(proto.MESSAGE, number=6, message=IndexedProperty,) - state = proto.Field(proto.ENUM, number=7, enum=State,) diff --git a/google/cloud/datastore_v1/__init__.py b/google/cloud/datastore_v1/__init__.py index a4b5de76..247eec15 100644 --- a/google/cloud/datastore_v1/__init__.py +++ b/google/cloud/datastore_v1/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,6 +15,8 @@ # from .services.datastore import DatastoreClient +from .services.datastore import DatastoreAsyncClient + from .types.datastore import AllocateIdsRequest from .types.datastore import AllocateIdsResponse from .types.datastore import BeginTransactionRequest @@ -52,8 +53,8 @@ from .types.query import Query from .types.query import QueryResultBatch - __all__ = ( + "DatastoreAsyncClient", "AllocateIdsRequest", "AllocateIdsResponse", "ArrayValue", @@ -62,6 +63,7 @@ "CommitRequest", "CommitResponse", "CompositeFilter", + "DatastoreClient", "Entity", "EntityResult", "Filter", @@ -89,5 +91,4 @@ "RunQueryResponse", "TransactionOptions", "Value", - "DatastoreClient", ) diff --git a/google/cloud/datastore_v1/gapic_metadata.json b/google/cloud/datastore_v1/gapic_metadata.json new file mode 100644 index 00000000..5da47e53 --- /dev/null +++ b/google/cloud/datastore_v1/gapic_metadata.json @@ -0,0 +1,93 @@ + { + "comment": "This file maps proto services/RPCs to the corresponding library clients/methods", + "language": "python", + "libraryPackage": "google.cloud.datastore_v1", + "protoPackage": "google.datastore.v1", + "schema": "1.0", + "services": { + "Datastore": { + "clients": { + "grpc": { + "libraryClient": "DatastoreClient", + "rpcs": { + "AllocateIds": { + "methods": [ + "allocate_ids" + ] + }, + "BeginTransaction": { + "methods": [ + "begin_transaction" + ] + }, + "Commit": { + "methods": [ + "commit" + ] + }, + "Lookup": { + "methods": [ + "lookup" + ] + }, + "ReserveIds": { + "methods": [ + "reserve_ids" + ] + }, + "Rollback": { + "methods": [ + "rollback" + ] + }, + "RunQuery": { + "methods": [ + "run_query" + ] + } + } + }, + "grpc-async": { + "libraryClient": "DatastoreAsyncClient", + "rpcs": { + "AllocateIds": { + "methods": [ + "allocate_ids" + ] + }, + "BeginTransaction": { + "methods": [ + "begin_transaction" + ] + }, + "Commit": { + "methods": [ + "commit" + ] + }, + "Lookup": { + "methods": [ + "lookup" + ] + }, + "ReserveIds": { + "methods": [ + "reserve_ids" + ] + }, + "Rollback": { + "methods": [ + "rollback" + ] + }, + "RunQuery": { + "methods": [ + "run_query" + ] + } + } + } + } + } + } +} diff --git a/google/cloud/datastore_v1/services/__init__.py b/google/cloud/datastore_v1/services/__init__.py index 42ffdf2b..4de65971 100644 --- a/google/cloud/datastore_v1/services/__init__.py +++ b/google/cloud/datastore_v1/services/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/google/cloud/datastore_v1/services/datastore/__init__.py b/google/cloud/datastore_v1/services/datastore/__init__.py index a8a82886..611f280b 100644 --- a/google/cloud/datastore_v1/services/datastore/__init__.py +++ b/google/cloud/datastore_v1/services/datastore/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # - from .client import DatastoreClient from .async_client import DatastoreAsyncClient diff --git a/google/cloud/datastore_v1/services/datastore/async_client.py b/google/cloud/datastore_v1/services/datastore/async_client.py index 01a2cbee..ca6beef2 100644 --- a/google/cloud/datastore_v1/services/datastore/async_client.py +++ b/google/cloud/datastore_v1/services/datastore/async_client.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,24 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. # - from collections import OrderedDict import functools import re from typing import Dict, Sequence, Tuple, Type, Union import pkg_resources -import google.api_core.client_options as ClientOptions # type: ignore -from google.api_core import exceptions # type: ignore +from google.api_core.client_options import ClientOptions # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore from google.api_core import gapic_v1 # type: ignore from google.api_core import retry as retries # type: ignore -from google.auth import credentials # type: ignore +from google.auth import credentials as ga_credentials # type: ignore from google.oauth2 import service_account # type: ignore +OptionalRetry = Union[retries.Retry, object] + from google.cloud.datastore_v1.types import datastore from google.cloud.datastore_v1.types import entity from google.cloud.datastore_v1.types import query - from .transports.base import DatastoreTransport, DEFAULT_CLIENT_INFO from .transports.grpc_asyncio import DatastoreGrpcAsyncIOTransport from .client import DatastoreClient @@ -58,29 +57,55 @@ class DatastoreAsyncClient: parse_common_billing_account_path = staticmethod( DatastoreClient.parse_common_billing_account_path ) - common_folder_path = staticmethod(DatastoreClient.common_folder_path) parse_common_folder_path = staticmethod(DatastoreClient.parse_common_folder_path) - common_organization_path = staticmethod(DatastoreClient.common_organization_path) parse_common_organization_path = staticmethod( DatastoreClient.parse_common_organization_path ) - common_project_path = staticmethod(DatastoreClient.common_project_path) parse_common_project_path = staticmethod(DatastoreClient.parse_common_project_path) - common_location_path = staticmethod(DatastoreClient.common_location_path) parse_common_location_path = staticmethod( DatastoreClient.parse_common_location_path ) - from_service_account_file = DatastoreClient.from_service_account_file + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + DatastoreAsyncClient: The constructed client. + """ + return DatastoreClient.from_service_account_info.__func__(DatastoreAsyncClient, info, *args, **kwargs) # type: ignore + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + DatastoreAsyncClient: The constructed client. + """ + return DatastoreClient.from_service_account_file.__func__(DatastoreAsyncClient, filename, *args, **kwargs) # type: ignore + from_service_account_json = from_service_account_file @property def transport(self) -> DatastoreTransport: - """Return the transport used by the client instance. + """Returns the transport used by the client instance. Returns: DatastoreTransport: The transport used by the client instance. @@ -94,12 +119,12 @@ def transport(self) -> DatastoreTransport: def __init__( self, *, - credentials: credentials.Credentials = None, + credentials: ga_credentials.Credentials = None, transport: Union[str, DatastoreTransport] = "grpc_asyncio", client_options: ClientOptions = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, ) -> None: - """Instantiate the datastore client. + """Instantiates the datastore client. Args: credentials (Optional[google.auth.credentials.Credentials]): The @@ -131,7 +156,6 @@ def __init__( google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport creation failed for any reason. """ - self._client = DatastoreClient( credentials=credentials, transport=transport, @@ -141,39 +165,40 @@ def __init__( async def lookup( self, - request: datastore.LookupRequest = None, + request: Union[datastore.LookupRequest, dict] = None, *, project_id: str = None, read_options: datastore.ReadOptions = None, keys: Sequence[entity.Key] = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> datastore.LookupResponse: r"""Looks up entities by key. Args: - request (:class:`~.datastore.LookupRequest`): + request (Union[google.cloud.datastore_v1.types.LookupRequest, dict]): The request object. The request for [Datastore.Lookup][google.datastore.v1.Datastore.Lookup]. project_id (:class:`str`): Required. The ID of the project against which to make the request. + This corresponds to the ``project_id`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - read_options (:class:`~.datastore.ReadOptions`): + read_options (:class:`google.cloud.datastore_v1.types.ReadOptions`): The options for this lookup request. This corresponds to the ``read_options`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - keys (:class:`Sequence[~.entity.Key]`): + keys (:class:`Sequence[google.cloud.datastore_v1.types.Key]`): Required. Keys of entities to look up. + This corresponds to the ``keys`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -181,7 +206,7 @@ async def lookup( sent along with the request as metadata. Returns: - ~.datastore.LookupResponse: + google.cloud.datastore_v1.types.LookupResponse: The response for [Datastore.Lookup][google.datastore.v1.Datastore.Lookup]. @@ -200,12 +225,10 @@ async def lookup( # If we have keyword arguments corresponding to fields on the # request, apply these. - if project_id is not None: request.project_id = project_id if read_options is not None: request.read_options = read_options - if keys: request.keys.extend(keys) @@ -218,8 +241,10 @@ async def lookup( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( - exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, ), + deadline=60.0, ), default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, @@ -233,19 +258,18 @@ async def lookup( async def run_query( self, - request: datastore.RunQueryRequest = None, + request: Union[datastore.RunQueryRequest, dict] = None, *, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> datastore.RunQueryResponse: r"""Queries for entities. Args: - request (:class:`~.datastore.RunQueryRequest`): + request (Union[google.cloud.datastore_v1.types.RunQueryRequest, dict]): The request object. The request for [Datastore.RunQuery][google.datastore.v1.Datastore.RunQuery]. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -253,13 +277,12 @@ async def run_query( sent along with the request as metadata. Returns: - ~.datastore.RunQueryResponse: + google.cloud.datastore_v1.types.RunQueryResponse: The response for [Datastore.RunQuery][google.datastore.v1.Datastore.RunQuery]. """ # Create or coerce a protobuf request object. - request = datastore.RunQueryRequest(request) # Wrap the RPC method; this adds retry and timeout information, @@ -271,8 +294,10 @@ async def run_query( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( - exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, ), + deadline=60.0, ), default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, @@ -286,26 +311,26 @@ async def run_query( async def begin_transaction( self, - request: datastore.BeginTransactionRequest = None, + request: Union[datastore.BeginTransactionRequest, dict] = None, *, project_id: str = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> datastore.BeginTransactionResponse: r"""Begins a new transaction. Args: - request (:class:`~.datastore.BeginTransactionRequest`): + request (Union[google.cloud.datastore_v1.types.BeginTransactionRequest, dict]): The request object. The request for [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction]. project_id (:class:`str`): Required. The ID of the project against which to make the request. + This corresponds to the ``project_id`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -313,7 +338,7 @@ async def begin_transaction( sent along with the request as metadata. Returns: - ~.datastore.BeginTransactionResponse: + google.cloud.datastore_v1.types.BeginTransactionResponse: The response for [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction]. @@ -332,7 +357,6 @@ async def begin_transaction( # If we have keyword arguments corresponding to fields on the # request, apply these. - if project_id is not None: request.project_id = project_id @@ -352,13 +376,13 @@ async def begin_transaction( async def commit( self, - request: datastore.CommitRequest = None, + request: Union[datastore.CommitRequest, dict] = None, *, project_id: str = None, mode: datastore.CommitRequest.Mode = None, transaction: bytes = None, mutations: Sequence[datastore.Mutation] = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> datastore.CommitResponse: @@ -366,18 +390,20 @@ async def commit( or modifying some entities. Args: - request (:class:`~.datastore.CommitRequest`): + request (Union[google.cloud.datastore_v1.types.CommitRequest, dict]): The request object. The request for [Datastore.Commit][google.datastore.v1.Datastore.Commit]. project_id (:class:`str`): Required. The ID of the project against which to make the request. + This corresponds to the ``project_id`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - mode (:class:`~.datastore.CommitRequest.Mode`): + mode (:class:`google.cloud.datastore_v1.types.CommitRequest.Mode`): The type of commit to perform. Defaults to ``TRANSACTIONAL``. + This corresponds to the ``mode`` field on the ``request`` instance; if ``request`` is provided, this should not be set. @@ -386,10 +412,11 @@ async def commit( commit. A transaction identifier is returned by a call to [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction]. + This corresponds to the ``transaction`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - mutations (:class:`Sequence[~.datastore.Mutation]`): + mutations (:class:`Sequence[google.cloud.datastore_v1.types.Mutation]`): The mutations to perform. When mode is ``TRANSACTIONAL``, mutations affecting a @@ -404,10 +431,10 @@ async def commit( When mode is ``NON_TRANSACTIONAL``, no two mutations may affect a single entity. + This corresponds to the ``mutations`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -415,7 +442,7 @@ async def commit( sent along with the request as metadata. Returns: - ~.datastore.CommitResponse: + google.cloud.datastore_v1.types.CommitResponse: The response for [Datastore.Commit][google.datastore.v1.Datastore.Commit]. @@ -434,14 +461,12 @@ async def commit( # If we have keyword arguments corresponding to fields on the # request, apply these. - if project_id is not None: request.project_id = project_id if mode is not None: request.mode = mode if transaction is not None: request.transaction = transaction - if mutations: request.mutations.extend(mutations) @@ -461,23 +486,24 @@ async def commit( async def rollback( self, - request: datastore.RollbackRequest = None, + request: Union[datastore.RollbackRequest, dict] = None, *, project_id: str = None, transaction: bytes = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> datastore.RollbackResponse: r"""Rolls back a transaction. Args: - request (:class:`~.datastore.RollbackRequest`): + request (Union[google.cloud.datastore_v1.types.RollbackRequest, dict]): The request object. The request for [Datastore.Rollback][google.datastore.v1.Datastore.Rollback]. project_id (:class:`str`): Required. The ID of the project against which to make the request. + This corresponds to the ``project_id`` field on the ``request`` instance; if ``request`` is provided, this should not be set. @@ -485,10 +511,10 @@ async def rollback( Required. The transaction identifier, returned by a call to [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction]. + This corresponds to the ``transaction`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -496,10 +522,9 @@ async def rollback( sent along with the request as metadata. Returns: - ~.datastore.RollbackResponse: - The response for - [Datastore.Rollback][google.datastore.v1.Datastore.Rollback]. - (an empty message). + google.cloud.datastore_v1.types.RollbackResponse: + The response for [Datastore.Rollback][google.datastore.v1.Datastore.Rollback]. + (an empty message). """ # Create or coerce a protobuf request object. @@ -516,7 +541,6 @@ async def rollback( # If we have keyword arguments corresponding to fields on the # request, apply these. - if project_id is not None: request.project_id = project_id if transaction is not None: @@ -538,11 +562,11 @@ async def rollback( async def allocate_ids( self, - request: datastore.AllocateIdsRequest = None, + request: Union[datastore.AllocateIdsRequest, dict] = None, *, project_id: str = None, keys: Sequence[entity.Key] = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> datastore.AllocateIdsResponse: @@ -550,24 +574,25 @@ async def allocate_ids( referencing an entity before it is inserted. Args: - request (:class:`~.datastore.AllocateIdsRequest`): + request (Union[google.cloud.datastore_v1.types.AllocateIdsRequest, dict]): The request object. The request for [Datastore.AllocateIds][google.datastore.v1.Datastore.AllocateIds]. project_id (:class:`str`): Required. The ID of the project against which to make the request. + This corresponds to the ``project_id`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - keys (:class:`Sequence[~.entity.Key]`): + keys (:class:`Sequence[google.cloud.datastore_v1.types.Key]`): Required. A list of keys with incomplete key paths for which to allocate IDs. No key may be reserved/read-only. + This corresponds to the ``keys`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -575,7 +600,7 @@ async def allocate_ids( sent along with the request as metadata. Returns: - ~.datastore.AllocateIdsResponse: + google.cloud.datastore_v1.types.AllocateIdsResponse: The response for [Datastore.AllocateIds][google.datastore.v1.Datastore.AllocateIds]. @@ -594,10 +619,8 @@ async def allocate_ids( # If we have keyword arguments corresponding to fields on the # request, apply these. - if project_id is not None: request.project_id = project_id - if keys: request.keys.extend(keys) @@ -617,11 +640,11 @@ async def allocate_ids( async def reserve_ids( self, - request: datastore.ReserveIdsRequest = None, + request: Union[datastore.ReserveIdsRequest, dict] = None, *, project_id: str = None, keys: Sequence[entity.Key] = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> datastore.ReserveIdsResponse: @@ -629,23 +652,24 @@ async def reserve_ids( llocated by Cloud Datastore. Args: - request (:class:`~.datastore.ReserveIdsRequest`): + request (Union[google.cloud.datastore_v1.types.ReserveIdsRequest, dict]): The request object. The request for [Datastore.ReserveIds][google.datastore.v1.Datastore.ReserveIds]. project_id (:class:`str`): Required. The ID of the project against which to make the request. + This corresponds to the ``project_id`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - keys (:class:`Sequence[~.entity.Key]`): + keys (:class:`Sequence[google.cloud.datastore_v1.types.Key]`): Required. A list of keys with complete key paths whose numeric IDs should not be auto-allocated. + This corresponds to the ``keys`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -653,7 +677,7 @@ async def reserve_ids( sent along with the request as metadata. Returns: - ~.datastore.ReserveIdsResponse: + google.cloud.datastore_v1.types.ReserveIdsResponse: The response for [Datastore.ReserveIds][google.datastore.v1.Datastore.ReserveIds]. @@ -672,10 +696,8 @@ async def reserve_ids( # If we have keyword arguments corresponding to fields on the # request, apply these. - if project_id is not None: request.project_id = project_id - if keys: request.keys.extend(keys) @@ -688,8 +710,10 @@ async def reserve_ids( maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( - exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, ), + deadline=60.0, ), default_timeout=60.0, client_info=DEFAULT_CLIENT_INFO, @@ -701,6 +725,12 @@ async def reserve_ids( # Done; return the response. return response + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + await self.transport.close() + try: DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( diff --git a/google/cloud/datastore_v1/services/datastore/client.py b/google/cloud/datastore_v1/services/datastore/client.py index e1379158..4c53cc1f 100644 --- a/google/cloud/datastore_v1/services/datastore/client.py +++ b/google/cloud/datastore_v1/services/datastore/client.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,28 +13,28 @@ # See the License for the specific language governing permissions and # limitations under the License. # - from collections import OrderedDict from distutils import util import os import re -from typing import Callable, Dict, Optional, Sequence, Tuple, Type, Union +from typing import Dict, Optional, Sequence, Tuple, Type, Union import pkg_resources from google.api_core import client_options as client_options_lib # type: ignore -from google.api_core import exceptions # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore from google.api_core import gapic_v1 # type: ignore from google.api_core import retry as retries # type: ignore -from google.auth import credentials # type: ignore +from google.auth import credentials as ga_credentials # type: ignore from google.auth.transport import mtls # type: ignore from google.auth.transport.grpc import SslCredentials # type: ignore from google.auth.exceptions import MutualTLSChannelError # type: ignore from google.oauth2 import service_account # type: ignore +OptionalRetry = Union[retries.Retry, object] + from google.cloud.datastore_v1.types import datastore from google.cloud.datastore_v1.types import entity from google.cloud.datastore_v1.types import query - from .transports.base import DatastoreTransport, DEFAULT_CLIENT_INFO from .transports.grpc import DatastoreGrpcTransport from .transports.grpc_asyncio import DatastoreGrpcAsyncIOTransport @@ -54,7 +53,7 @@ class DatastoreClientMeta(type): _transport_registry["grpc_asyncio"] = DatastoreGrpcAsyncIOTransport def get_transport_class(cls, label: str = None,) -> Type[DatastoreTransport]: - """Return an appropriate transport class. + """Returns an appropriate transport class. Args: label: The name of the desired transport. If none is @@ -84,7 +83,8 @@ class DatastoreClient(metaclass=DatastoreClientMeta): @staticmethod def _get_default_mtls_endpoint(api_endpoint): - """Convert api endpoint to mTLS endpoint. + """Converts api endpoint to mTLS endpoint. + Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. Args: @@ -116,10 +116,27 @@ def _get_default_mtls_endpoint(api_endpoint): DEFAULT_ENDPOINT ) + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + DatastoreClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_info(info) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + @classmethod def from_service_account_file(cls, filename: str, *args, **kwargs): """Creates an instance of this client using the provided credentials - file. + file. Args: filename (str): The path to the service account private key json @@ -128,7 +145,7 @@ def from_service_account_file(cls, filename: str, *args, **kwargs): kwargs: Additional arguments to pass to the constructor. Returns: - {@api.name}: The constructed client. + DatastoreClient: The constructed client. """ credentials = service_account.Credentials.from_service_account_file(filename) kwargs["credentials"] = credentials @@ -138,16 +155,17 @@ def from_service_account_file(cls, filename: str, *args, **kwargs): @property def transport(self) -> DatastoreTransport: - """Return the transport used by the client instance. + """Returns the transport used by the client instance. Returns: - DatastoreTransport: The transport used by the client instance. + DatastoreTransport: The transport used by the client + instance. """ return self._transport @staticmethod def common_billing_account_path(billing_account: str,) -> str: - """Return a fully-qualified billing_account string.""" + """Returns a fully-qualified billing_account string.""" return "billingAccounts/{billing_account}".format( billing_account=billing_account, ) @@ -160,7 +178,7 @@ def parse_common_billing_account_path(path: str) -> Dict[str, str]: @staticmethod def common_folder_path(folder: str,) -> str: - """Return a fully-qualified folder string.""" + """Returns a fully-qualified folder string.""" return "folders/{folder}".format(folder=folder,) @staticmethod @@ -171,7 +189,7 @@ def parse_common_folder_path(path: str) -> Dict[str, str]: @staticmethod def common_organization_path(organization: str,) -> str: - """Return a fully-qualified organization string.""" + """Returns a fully-qualified organization string.""" return "organizations/{organization}".format(organization=organization,) @staticmethod @@ -182,7 +200,7 @@ def parse_common_organization_path(path: str) -> Dict[str, str]: @staticmethod def common_project_path(project: str,) -> str: - """Return a fully-qualified project string.""" + """Returns a fully-qualified project string.""" return "projects/{project}".format(project=project,) @staticmethod @@ -193,7 +211,7 @@ def parse_common_project_path(path: str) -> Dict[str, str]: @staticmethod def common_location_path(project: str, location: str,) -> str: - """Return a fully-qualified location string.""" + """Returns a fully-qualified location string.""" return "projects/{project}/locations/{location}".format( project=project, location=location, ) @@ -207,12 +225,12 @@ def parse_common_location_path(path: str) -> Dict[str, str]: def __init__( self, *, - credentials: Optional[credentials.Credentials] = None, + credentials: Optional[ga_credentials.Credentials] = None, transport: Union[str, DatastoreTransport, None] = None, client_options: Optional[client_options_lib.ClientOptions] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, ) -> None: - """Instantiate the datastore client. + """Instantiates the datastore client. Args: credentials (Optional[google.auth.credentials.Credentials]): The @@ -220,10 +238,10 @@ def __init__( credentials identify the application to the service; if none are specified, the client will attempt to ascertain the credentials from the environment. - transport (Union[str, ~.DatastoreTransport]): The + transport (Union[str, DatastoreTransport]): The transport to use. If set to None, a transport is chosen automatically. - client_options (client_options_lib.ClientOptions): Custom options for the + client_options (google.api_core.client_options.ClientOptions): Custom options for the client. It won't take effect if a ``transport`` instance is provided. (1) The ``api_endpoint`` property can be used to override the default endpoint provided by the client. GOOGLE_API_USE_MTLS_ENDPOINT @@ -259,21 +277,18 @@ def __init__( util.strtobool(os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false")) ) - ssl_credentials = None + client_cert_source_func = None is_mtls = False if use_client_cert: if client_options.client_cert_source: - import grpc # type: ignore - - cert, key = client_options.client_cert_source() - ssl_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) is_mtls = True + client_cert_source_func = client_options.client_cert_source else: - creds = SslCredentials() - is_mtls = creds.is_mtls - ssl_credentials = creds.ssl_credentials if is_mtls else None + is_mtls = mtls.has_default_client_cert_source() + if is_mtls: + client_cert_source_func = mtls.default_client_cert_source() + else: + client_cert_source_func = None # Figure out which api endpoint to use. if client_options.api_endpoint is not None: @@ -285,12 +300,14 @@ def __init__( elif use_mtls_env == "always": api_endpoint = self.DEFAULT_MTLS_ENDPOINT elif use_mtls_env == "auto": - api_endpoint = ( - self.DEFAULT_MTLS_ENDPOINT if is_mtls else self.DEFAULT_ENDPOINT - ) + if is_mtls: + api_endpoint = self.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = self.DEFAULT_ENDPOINT else: raise MutualTLSChannelError( - "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted values: never, auto, always" + "Unsupported GOOGLE_API_USE_MTLS_ENDPOINT value. Accepted " + "values: never, auto, always" ) # Save or instantiate the transport. @@ -305,8 +322,8 @@ def __init__( ) if client_options.scopes: raise ValueError( - "When providing a transport instance, " - "provide its scopes directly." + "When providing a transport instance, provide its scopes " + "directly." ) self._transport = transport else: @@ -316,46 +333,48 @@ def __init__( credentials_file=client_options.credentials_file, host=api_endpoint, scopes=client_options.scopes, - ssl_channel_credentials=ssl_credentials, + client_cert_source_for_mtls=client_cert_source_func, quota_project_id=client_options.quota_project_id, client_info=client_info, + always_use_jwt_access=True, ) def lookup( self, - request: datastore.LookupRequest = None, + request: Union[datastore.LookupRequest, dict] = None, *, project_id: str = None, read_options: datastore.ReadOptions = None, keys: Sequence[entity.Key] = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> datastore.LookupResponse: r"""Looks up entities by key. Args: - request (:class:`~.datastore.LookupRequest`): + request (Union[google.cloud.datastore_v1.types.LookupRequest, dict]): The request object. The request for [Datastore.Lookup][google.datastore.v1.Datastore.Lookup]. - project_id (:class:`str`): + project_id (str): Required. The ID of the project against which to make the request. + This corresponds to the ``project_id`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - read_options (:class:`~.datastore.ReadOptions`): + read_options (google.cloud.datastore_v1.types.ReadOptions): The options for this lookup request. This corresponds to the ``read_options`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - keys (:class:`Sequence[~.entity.Key]`): + keys (Sequence[google.cloud.datastore_v1.types.Key]): Required. Keys of entities to look up. + This corresponds to the ``keys`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -363,7 +382,7 @@ def lookup( sent along with the request as metadata. Returns: - ~.datastore.LookupResponse: + google.cloud.datastore_v1.types.LookupResponse: The response for [Datastore.Lookup][google.datastore.v1.Datastore.Lookup]. @@ -384,17 +403,14 @@ def lookup( # there are no flattened fields. if not isinstance(request, datastore.LookupRequest): request = datastore.LookupRequest(request) - # If we have keyword arguments corresponding to fields on the # request, apply these. - if project_id is not None: request.project_id = project_id if read_options is not None: request.read_options = read_options - - if keys: - request.keys.extend(keys) + if keys is not None: + request.keys = keys # Wrap the RPC method; this adds retry and timeout information, # and friendly error handling. @@ -408,19 +424,18 @@ def lookup( def run_query( self, - request: datastore.RunQueryRequest = None, + request: Union[datastore.RunQueryRequest, dict] = None, *, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> datastore.RunQueryResponse: r"""Queries for entities. Args: - request (:class:`~.datastore.RunQueryRequest`): + request (Union[google.cloud.datastore_v1.types.RunQueryRequest, dict]): The request object. The request for [Datastore.RunQuery][google.datastore.v1.Datastore.RunQuery]. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -428,13 +443,12 @@ def run_query( sent along with the request as metadata. Returns: - ~.datastore.RunQueryResponse: + google.cloud.datastore_v1.types.RunQueryResponse: The response for [Datastore.RunQuery][google.datastore.v1.Datastore.RunQuery]. """ # Create or coerce a protobuf request object. - # Minor optimization to avoid making a copy if the user passes # in a datastore.RunQueryRequest. # There's no risk of modifying the input as we've already verified @@ -454,26 +468,26 @@ def run_query( def begin_transaction( self, - request: datastore.BeginTransactionRequest = None, + request: Union[datastore.BeginTransactionRequest, dict] = None, *, project_id: str = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> datastore.BeginTransactionResponse: r"""Begins a new transaction. Args: - request (:class:`~.datastore.BeginTransactionRequest`): + request (Union[google.cloud.datastore_v1.types.BeginTransactionRequest, dict]): The request object. The request for [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction]. - project_id (:class:`str`): + project_id (str): Required. The ID of the project against which to make the request. + This corresponds to the ``project_id`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -481,7 +495,7 @@ def begin_transaction( sent along with the request as metadata. Returns: - ~.datastore.BeginTransactionResponse: + google.cloud.datastore_v1.types.BeginTransactionResponse: The response for [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction]. @@ -502,10 +516,8 @@ def begin_transaction( # there are no flattened fields. if not isinstance(request, datastore.BeginTransactionRequest): request = datastore.BeginTransactionRequest(request) - # If we have keyword arguments corresponding to fields on the # request, apply these. - if project_id is not None: request.project_id = project_id @@ -521,13 +533,13 @@ def begin_transaction( def commit( self, - request: datastore.CommitRequest = None, + request: Union[datastore.CommitRequest, dict] = None, *, project_id: str = None, mode: datastore.CommitRequest.Mode = None, transaction: bytes = None, mutations: Sequence[datastore.Mutation] = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> datastore.CommitResponse: @@ -535,30 +547,33 @@ def commit( or modifying some entities. Args: - request (:class:`~.datastore.CommitRequest`): + request (Union[google.cloud.datastore_v1.types.CommitRequest, dict]): The request object. The request for [Datastore.Commit][google.datastore.v1.Datastore.Commit]. - project_id (:class:`str`): + project_id (str): Required. The ID of the project against which to make the request. + This corresponds to the ``project_id`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - mode (:class:`~.datastore.CommitRequest.Mode`): + mode (google.cloud.datastore_v1.types.CommitRequest.Mode): The type of commit to perform. Defaults to ``TRANSACTIONAL``. + This corresponds to the ``mode`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - transaction (:class:`bytes`): + transaction (bytes): The identifier of the transaction associated with the commit. A transaction identifier is returned by a call to [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction]. + This corresponds to the ``transaction`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - mutations (:class:`Sequence[~.datastore.Mutation]`): + mutations (Sequence[google.cloud.datastore_v1.types.Mutation]): The mutations to perform. When mode is ``TRANSACTIONAL``, mutations affecting a @@ -573,10 +588,10 @@ def commit( When mode is ``NON_TRANSACTIONAL``, no two mutations may affect a single entity. + This corresponds to the ``mutations`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -584,7 +599,7 @@ def commit( sent along with the request as metadata. Returns: - ~.datastore.CommitResponse: + google.cloud.datastore_v1.types.CommitResponse: The response for [Datastore.Commit][google.datastore.v1.Datastore.Commit]. @@ -605,19 +620,16 @@ def commit( # there are no flattened fields. if not isinstance(request, datastore.CommitRequest): request = datastore.CommitRequest(request) - # If we have keyword arguments corresponding to fields on the # request, apply these. - if project_id is not None: request.project_id = project_id if mode is not None: request.mode = mode if transaction is not None: request.transaction = transaction - - if mutations: - request.mutations.extend(mutations) + if mutations is not None: + request.mutations = mutations # Wrap the RPC method; this adds retry and timeout information, # and friendly error handling. @@ -631,34 +643,35 @@ def commit( def rollback( self, - request: datastore.RollbackRequest = None, + request: Union[datastore.RollbackRequest, dict] = None, *, project_id: str = None, transaction: bytes = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> datastore.RollbackResponse: r"""Rolls back a transaction. Args: - request (:class:`~.datastore.RollbackRequest`): + request (Union[google.cloud.datastore_v1.types.RollbackRequest, dict]): The request object. The request for [Datastore.Rollback][google.datastore.v1.Datastore.Rollback]. - project_id (:class:`str`): + project_id (str): Required. The ID of the project against which to make the request. + This corresponds to the ``project_id`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - transaction (:class:`bytes`): + transaction (bytes): Required. The transaction identifier, returned by a call to [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction]. + This corresponds to the ``transaction`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -666,10 +679,9 @@ def rollback( sent along with the request as metadata. Returns: - ~.datastore.RollbackResponse: - The response for - [Datastore.Rollback][google.datastore.v1.Datastore.Rollback]. - (an empty message). + google.cloud.datastore_v1.types.RollbackResponse: + The response for [Datastore.Rollback][google.datastore.v1.Datastore.Rollback]. + (an empty message). """ # Create or coerce a protobuf request object. @@ -688,10 +700,8 @@ def rollback( # there are no flattened fields. if not isinstance(request, datastore.RollbackRequest): request = datastore.RollbackRequest(request) - # If we have keyword arguments corresponding to fields on the # request, apply these. - if project_id is not None: request.project_id = project_id if transaction is not None: @@ -709,11 +719,11 @@ def rollback( def allocate_ids( self, - request: datastore.AllocateIdsRequest = None, + request: Union[datastore.AllocateIdsRequest, dict] = None, *, project_id: str = None, keys: Sequence[entity.Key] = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> datastore.AllocateIdsResponse: @@ -721,24 +731,25 @@ def allocate_ids( referencing an entity before it is inserted. Args: - request (:class:`~.datastore.AllocateIdsRequest`): + request (Union[google.cloud.datastore_v1.types.AllocateIdsRequest, dict]): The request object. The request for [Datastore.AllocateIds][google.datastore.v1.Datastore.AllocateIds]. - project_id (:class:`str`): + project_id (str): Required. The ID of the project against which to make the request. + This corresponds to the ``project_id`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - keys (:class:`Sequence[~.entity.Key]`): + keys (Sequence[google.cloud.datastore_v1.types.Key]): Required. A list of keys with incomplete key paths for which to allocate IDs. No key may be reserved/read-only. + This corresponds to the ``keys`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -746,7 +757,7 @@ def allocate_ids( sent along with the request as metadata. Returns: - ~.datastore.AllocateIdsResponse: + google.cloud.datastore_v1.types.AllocateIdsResponse: The response for [Datastore.AllocateIds][google.datastore.v1.Datastore.AllocateIds]. @@ -767,15 +778,12 @@ def allocate_ids( # there are no flattened fields. if not isinstance(request, datastore.AllocateIdsRequest): request = datastore.AllocateIdsRequest(request) - # If we have keyword arguments corresponding to fields on the # request, apply these. - if project_id is not None: request.project_id = project_id - - if keys: - request.keys.extend(keys) + if keys is not None: + request.keys = keys # Wrap the RPC method; this adds retry and timeout information, # and friendly error handling. @@ -789,11 +797,11 @@ def allocate_ids( def reserve_ids( self, - request: datastore.ReserveIdsRequest = None, + request: Union[datastore.ReserveIdsRequest, dict] = None, *, project_id: str = None, keys: Sequence[entity.Key] = None, - retry: retries.Retry = gapic_v1.method.DEFAULT, + retry: OptionalRetry = gapic_v1.method.DEFAULT, timeout: float = None, metadata: Sequence[Tuple[str, str]] = (), ) -> datastore.ReserveIdsResponse: @@ -801,23 +809,24 @@ def reserve_ids( llocated by Cloud Datastore. Args: - request (:class:`~.datastore.ReserveIdsRequest`): + request (Union[google.cloud.datastore_v1.types.ReserveIdsRequest, dict]): The request object. The request for [Datastore.ReserveIds][google.datastore.v1.Datastore.ReserveIds]. - project_id (:class:`str`): + project_id (str): Required. The ID of the project against which to make the request. + This corresponds to the ``project_id`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - keys (:class:`Sequence[~.entity.Key]`): + keys (Sequence[google.cloud.datastore_v1.types.Key]): Required. A list of keys with complete key paths whose numeric IDs should not be auto-allocated. + This corresponds to the ``keys`` field on the ``request`` instance; if ``request`` is provided, this should not be set. - retry (google.api_core.retry.Retry): Designation of what errors, if any, should be retried. timeout (float): The timeout for this request. @@ -825,7 +834,7 @@ def reserve_ids( sent along with the request as metadata. Returns: - ~.datastore.ReserveIdsResponse: + google.cloud.datastore_v1.types.ReserveIdsResponse: The response for [Datastore.ReserveIds][google.datastore.v1.Datastore.ReserveIds]. @@ -846,15 +855,12 @@ def reserve_ids( # there are no flattened fields. if not isinstance(request, datastore.ReserveIdsRequest): request = datastore.ReserveIdsRequest(request) - # If we have keyword arguments corresponding to fields on the # request, apply these. - if project_id is not None: request.project_id = project_id - - if keys: - request.keys.extend(keys) + if keys is not None: + request.keys = keys # Wrap the RPC method; this adds retry and timeout information, # and friendly error handling. @@ -866,6 +872,19 @@ def reserve_ids( # Done; return the response. return response + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + """Releases underlying transport's resources. + + .. warning:: + ONLY use as a context manager if the transport is NOT shared + with other clients! Exiting the with block will CLOSE the transport + and may cause errors in other clients! + """ + self.transport.close() + try: DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( diff --git a/google/cloud/datastore_v1/services/datastore/transports/__init__.py b/google/cloud/datastore_v1/services/datastore/transports/__init__.py index 2d0659d9..41074a07 100644 --- a/google/cloud/datastore_v1/services/datastore/transports/__init__.py +++ b/google/cloud/datastore_v1/services/datastore/transports/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # - from collections import OrderedDict from typing import Dict, Type @@ -28,7 +26,6 @@ _transport_registry["grpc"] = DatastoreGrpcTransport _transport_registry["grpc_asyncio"] = DatastoreGrpcAsyncIOTransport - __all__ = ( "DatastoreTransport", "DatastoreGrpcTransport", diff --git a/google/cloud/datastore_v1/services/datastore/transports/base.py b/google/cloud/datastore_v1/services/datastore/transports/base.py index ad00b33f..7959b72e 100644 --- a/google/cloud/datastore_v1/services/datastore/transports/base.py +++ b/google/cloud/datastore_v1/services/datastore/transports/base.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,20 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import abc -import typing +from typing import Awaitable, Callable, Dict, Optional, Sequence, Union import pkg_resources -from google import auth # type: ignore -from google.api_core import exceptions # type: ignore +import google.auth # type: ignore +import google.api_core # type: ignore +from google.api_core import exceptions as core_exceptions # type: ignore from google.api_core import gapic_v1 # type: ignore from google.api_core import retry as retries # type: ignore -from google.auth import credentials # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore from google.cloud.datastore_v1.types import datastore - try: DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo( gapic_version=pkg_resources.get_distribution("google-cloud-datastore",).version, @@ -44,21 +43,25 @@ class DatastoreTransport(abc.ABC): "https://www.googleapis.com/auth/datastore", ) + DEFAULT_HOST: str = "datastore.googleapis.com" + def __init__( self, *, - host: str = "datastore.googleapis.com", - credentials: credentials.Credentials = None, - credentials_file: typing.Optional[str] = None, - scopes: typing.Optional[typing.Sequence[str]] = AUTH_SCOPES, - quota_project_id: typing.Optional[str] = None, + host: str = DEFAULT_HOST, + credentials: ga_credentials.Credentials = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, **kwargs, ) -> None: """Instantiate the transport. Args: - host (Optional[str]): The hostname to connect to. + host (Optional[str]): + The hostname to connect to. credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none @@ -67,43 +70,55 @@ def __init__( credentials_file (Optional[str]): A file with credentials that can be loaded with :func:`google.auth.load_credentials_from_file`. This argument is mutually exclusive with credentials. - scope (Optional[Sequence[str]]): A list of scopes. + scopes (Optional[Sequence[str]]): A list of scopes. quota_project_id (Optional[str]): An optional project to use for billing and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. """ # Save the hostname. Default to port 443 (HTTPS) if none is specified. if ":" not in host: host += ":443" self._host = host + scopes_kwargs = {"scopes": scopes, "default_scopes": self.AUTH_SCOPES} + + # Save the scopes. + self._scopes = scopes + # If no credentials are provided, then determine the appropriate # defaults. if credentials and credentials_file: - raise exceptions.DuplicateCredentialArgs( + raise core_exceptions.DuplicateCredentialArgs( "'credentials_file' and 'credentials' are mutually exclusive" ) if credentials_file is not None: - credentials, _ = auth.load_credentials_from_file( - credentials_file, scopes=scopes, quota_project_id=quota_project_id + credentials, _ = google.auth.load_credentials_from_file( + credentials_file, **scopes_kwargs, quota_project_id=quota_project_id ) elif credentials is None: - credentials, _ = auth.default( - scopes=scopes, quota_project_id=quota_project_id + credentials, _ = google.auth.default( + **scopes_kwargs, quota_project_id=quota_project_id ) + # If the credentials are service account credentials, then always try to use self signed JWT. + if ( + always_use_jwt_access + and isinstance(credentials, service_account.Credentials) + and hasattr(service_account.Credentials, "with_always_use_jwt_access") + ): + credentials = credentials.with_always_use_jwt_access(True) + # Save the credentials. self._credentials = credentials - # Lifted into its own function so it can be stubbed out during tests. - self._prep_wrapped_messages(client_info) - def _prep_wrapped_messages(self, client_info): # Precompute the wrapped methods. self._wrapped_methods = { @@ -114,8 +129,10 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( - exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, ), + deadline=60.0, ), default_timeout=60.0, client_info=client_info, @@ -127,8 +144,10 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( - exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, ), + deadline=60.0, ), default_timeout=60.0, client_info=client_info, @@ -152,44 +171,51 @@ def _prep_wrapped_messages(self, client_info): maximum=60.0, multiplier=1.3, predicate=retries.if_exception_type( - exceptions.DeadlineExceeded, exceptions.ServiceUnavailable, + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, ), + deadline=60.0, ), default_timeout=60.0, client_info=client_info, ), } + def close(self): + """Closes resources associated with the transport. + + .. warning:: + Only call this method if the transport is NOT shared + with other clients - this may cause errors in other clients! + """ + raise NotImplementedError() + @property def lookup( self, - ) -> typing.Callable[ + ) -> Callable[ [datastore.LookupRequest], - typing.Union[ - datastore.LookupResponse, typing.Awaitable[datastore.LookupResponse] - ], + Union[datastore.LookupResponse, Awaitable[datastore.LookupResponse]], ]: raise NotImplementedError() @property def run_query( self, - ) -> typing.Callable[ + ) -> Callable[ [datastore.RunQueryRequest], - typing.Union[ - datastore.RunQueryResponse, typing.Awaitable[datastore.RunQueryResponse] - ], + Union[datastore.RunQueryResponse, Awaitable[datastore.RunQueryResponse]], ]: raise NotImplementedError() @property def begin_transaction( self, - ) -> typing.Callable[ + ) -> Callable[ [datastore.BeginTransactionRequest], - typing.Union[ + Union[ datastore.BeginTransactionResponse, - typing.Awaitable[datastore.BeginTransactionResponse], + Awaitable[datastore.BeginTransactionResponse], ], ]: raise NotImplementedError() @@ -197,45 +223,36 @@ def begin_transaction( @property def commit( self, - ) -> typing.Callable[ + ) -> Callable[ [datastore.CommitRequest], - typing.Union[ - datastore.CommitResponse, typing.Awaitable[datastore.CommitResponse] - ], + Union[datastore.CommitResponse, Awaitable[datastore.CommitResponse]], ]: raise NotImplementedError() @property def rollback( self, - ) -> typing.Callable[ + ) -> Callable[ [datastore.RollbackRequest], - typing.Union[ - datastore.RollbackResponse, typing.Awaitable[datastore.RollbackResponse] - ], + Union[datastore.RollbackResponse, Awaitable[datastore.RollbackResponse]], ]: raise NotImplementedError() @property def allocate_ids( self, - ) -> typing.Callable[ + ) -> Callable[ [datastore.AllocateIdsRequest], - typing.Union[ - datastore.AllocateIdsResponse, - typing.Awaitable[datastore.AllocateIdsResponse], - ], + Union[datastore.AllocateIdsResponse, Awaitable[datastore.AllocateIdsResponse]], ]: raise NotImplementedError() @property def reserve_ids( self, - ) -> typing.Callable[ + ) -> Callable[ [datastore.ReserveIdsRequest], - typing.Union[ - datastore.ReserveIdsResponse, typing.Awaitable[datastore.ReserveIdsResponse] - ], + Union[datastore.ReserveIdsResponse, Awaitable[datastore.ReserveIdsResponse]], ]: raise NotImplementedError() diff --git a/google/cloud/datastore_v1/services/datastore/transports/grpc.py b/google/cloud/datastore_v1/services/datastore/transports/grpc.py index 7d170570..afcc6a15 100644 --- a/google/cloud/datastore_v1/services/datastore/transports/grpc.py +++ b/google/cloud/datastore_v1/services/datastore/transports/grpc.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,20 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import warnings -from typing import Callable, Dict, Optional, Sequence, Tuple +from typing import Callable, Dict, Optional, Sequence, Tuple, Union from google.api_core import grpc_helpers # type: ignore from google.api_core import gapic_v1 # type: ignore -from google import auth # type: ignore -from google.auth import credentials # type: ignore +import google.auth # type: ignore +from google.auth import credentials as ga_credentials # type: ignore from google.auth.transport.grpc import SslCredentials # type: ignore import grpc # type: ignore from google.cloud.datastore_v1.types import datastore - from .base import DatastoreTransport, DEFAULT_CLIENT_INFO @@ -56,20 +53,23 @@ def __init__( self, *, host: str = "datastore.googleapis.com", - credentials: credentials.Credentials = None, + credentials: ga_credentials.Credentials = None, credentials_file: str = None, scopes: Sequence[str] = None, channel: grpc.Channel = None, api_mtls_endpoint: str = None, client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, ssl_channel_credentials: grpc.ChannelCredentials = None, + client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, quota_project_id: Optional[str] = None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, ) -> None: """Instantiate the transport. Args: - host (Optional[str]): The hostname to connect to. + host (Optional[str]): + The hostname to connect to. credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none @@ -86,13 +86,17 @@ def __init__( api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. If provided, it overrides the ``host`` argument and tries to create a mutual TLS channel with client SSL credentials from - ``client_cert_source`` or applicatin default SSL credentials. + ``client_cert_source`` or application default SSL credentials. client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): Deprecated. A callback to provide client SSL certificate bytes and private key bytes, both in PEM format. It is ignored if ``api_mtls_endpoint`` is None. ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials - for grpc channel. It is ignored if ``channel`` is provided. + for the grpc channel. It is ignored if ``channel`` is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure a mutual TLS channel. It is + ignored if ``channel`` or ``ssl_channel_credentials`` is provided. quota_project_id (Optional[str]): An optional project to use for billing and quota. client_info (google.api_core.gapic_v1.client_info.ClientInfo): @@ -100,6 +104,8 @@ def __init__( API requests. If ``None``, then default info will be used. Generally, you only need to set this if you're developing your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. Raises: google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport @@ -107,88 +113,76 @@ def __init__( google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` and ``credentials_file`` are passed. """ + self._grpc_channel = None self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) if channel: - # Sanity check: Ensure that channel and credentials are not both - # provided. + # Ignore credentials if a channel was passed. credentials = False - # If a channel was explicitly provided, set it. self._grpc_channel = channel self._ssl_channel_credentials = None - elif api_mtls_endpoint: - warnings.warn( - "api_mtls_endpoint and client_cert_source are deprecated", - DeprecationWarning, - ) - - host = ( - api_mtls_endpoint - if ":" in api_mtls_endpoint - else api_mtls_endpoint + ":443" - ) - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) - - # Create SSL credentials with client_cert_source or application - # default SSL credentials. - if client_cert_source: - cert, key = client_cert_source() - ssl_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - else: - ssl_credentials = SslCredentials().ssl_credentials - - # create a new channel. The provided one is ignored. - self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - ssl_credentials=ssl_credentials, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - ) - self._ssl_channel_credentials = ssl_credentials else: - host = host if ":" in host else host + ":443" - - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) - - # create a new channel. The provided one is ignored. - self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - ssl_credentials=ssl_channel_credentials, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - ) + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials - self._stubs = {} # type: Dict[str, Callable] + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) - # Run the base constructor. + # The base transport sets the host, credentials and scopes super().__init__( host=host, credentials=credentials, credentials_file=credentials_file, - scopes=scopes or self.AUTH_SCOPES, + scopes=scopes, quota_project_id=quota_project_id, client_info=client_info, + always_use_jwt_access=always_use_jwt_access, ) + if not self._grpc_channel: + self._grpc_channel = type(self).create_channel( + self._host, + credentials=self._credentials, + credentials_file=credentials_file, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) + @classmethod def create_channel( cls, host: str = "datastore.googleapis.com", - credentials: credentials.Credentials = None, + credentials: ga_credentials.Credentials = None, credentials_file: str = None, scopes: Optional[Sequence[str]] = None, quota_project_id: Optional[str] = None, @@ -196,7 +190,7 @@ def create_channel( ) -> grpc.Channel: """Create and return a gRPC channel object. Args: - address (Optionsl[str]): The host for the channel to use. + host (Optional[str]): The host for the channel to use. credentials (Optional[~.Credentials]): The authorization credentials to attach to requests. These credentials identify this application to the service. If @@ -219,13 +213,15 @@ def create_channel( google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` and ``credentials_file`` are passed. """ - scopes = scopes or cls.AUTH_SCOPES + return grpc_helpers.create_channel( host, credentials=credentials, credentials_file=credentials_file, - scopes=scopes, quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, **kwargs, ) @@ -418,5 +414,8 @@ def reserve_ids( ) return self._stubs["reserve_ids"] + def close(self): + self.grpc_channel.close() + __all__ = ("DatastoreGrpcTransport",) diff --git a/google/cloud/datastore_v1/services/datastore/transports/grpc_asyncio.py b/google/cloud/datastore_v1/services/datastore/transports/grpc_asyncio.py index 8ba5f66d..20c51f7c 100644 --- a/google/cloud/datastore_v1/services/datastore/transports/grpc_asyncio.py +++ b/google/cloud/datastore_v1/services/datastore/transports/grpc_asyncio.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,21 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import warnings -from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple +from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union from google.api_core import gapic_v1 # type: ignore from google.api_core import grpc_helpers_async # type: ignore -from google import auth # type: ignore -from google.auth import credentials # type: ignore +from google.auth import credentials as ga_credentials # type: ignore from google.auth.transport.grpc import SslCredentials # type: ignore import grpc # type: ignore from grpc.experimental import aio # type: ignore from google.cloud.datastore_v1.types import datastore - from .base import DatastoreTransport, DEFAULT_CLIENT_INFO from .grpc import DatastoreGrpcTransport @@ -59,7 +55,7 @@ class DatastoreGrpcAsyncIOTransport(DatastoreTransport): def create_channel( cls, host: str = "datastore.googleapis.com", - credentials: credentials.Credentials = None, + credentials: ga_credentials.Credentials = None, credentials_file: Optional[str] = None, scopes: Optional[Sequence[str]] = None, quota_project_id: Optional[str] = None, @@ -67,7 +63,7 @@ def create_channel( ) -> aio.Channel: """Create and return a gRPC AsyncIO channel object. Args: - address (Optional[str]): The host for the channel to use. + host (Optional[str]): The host for the channel to use. credentials (Optional[~.Credentials]): The authorization credentials to attach to requests. These credentials identify this application to the service. If @@ -86,13 +82,15 @@ def create_channel( Returns: aio.Channel: A gRPC AsyncIO channel object. """ - scopes = scopes or cls.AUTH_SCOPES + return grpc_helpers_async.create_channel( host, credentials=credentials, credentials_file=credentials_file, - scopes=scopes, quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, **kwargs, ) @@ -100,20 +98,23 @@ def __init__( self, *, host: str = "datastore.googleapis.com", - credentials: credentials.Credentials = None, + credentials: ga_credentials.Credentials = None, credentials_file: Optional[str] = None, scopes: Optional[Sequence[str]] = None, channel: aio.Channel = None, api_mtls_endpoint: str = None, client_cert_source: Callable[[], Tuple[bytes, bytes]] = None, ssl_channel_credentials: grpc.ChannelCredentials = None, + client_cert_source_for_mtls: Callable[[], Tuple[bytes, bytes]] = None, quota_project_id=None, client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, ) -> None: """Instantiate the transport. Args: - host (Optional[str]): The hostname to connect to. + host (Optional[str]): + The hostname to connect to. credentials (Optional[google.auth.credentials.Credentials]): The authorization credentials to attach to requests. These credentials identify the application to the service; if none @@ -131,20 +132,26 @@ def __init__( api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. If provided, it overrides the ``host`` argument and tries to create a mutual TLS channel with client SSL credentials from - ``client_cert_source`` or applicatin default SSL credentials. + ``client_cert_source`` or application default SSL credentials. client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): Deprecated. A callback to provide client SSL certificate bytes and private key bytes, both in PEM format. It is ignored if ``api_mtls_endpoint`` is None. ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials - for grpc channel. It is ignored if ``channel`` is provided. + for the grpc channel. It is ignored if ``channel`` is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure a mutual TLS channel. It is + ignored if ``channel`` or ``ssl_channel_credentials`` is provided. quota_project_id (Optional[str]): An optional project to use for billing and quota. - client_info (google.api_core.gapic_v1.client_info.ClientInfo): - The client info used to send a user-agent string along with - API requests. If ``None``, then default info will be used. - Generally, you only need to set this if you're developing + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. Raises: google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport @@ -152,82 +159,69 @@ def __init__( google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` and ``credentials_file`` are passed. """ + self._grpc_channel = None self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) if channel: - # Sanity check: Ensure that channel and credentials are not both - # provided. + # Ignore credentials if a channel was passed. credentials = False - # If a channel was explicitly provided, set it. self._grpc_channel = channel self._ssl_channel_credentials = None - elif api_mtls_endpoint: - warnings.warn( - "api_mtls_endpoint and client_cert_source are deprecated", - DeprecationWarning, - ) - - host = ( - api_mtls_endpoint - if ":" in api_mtls_endpoint - else api_mtls_endpoint + ":443" - ) - - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) - - # Create SSL credentials with client_cert_source or application - # default SSL credentials. - if client_cert_source: - cert, key = client_cert_source() - ssl_credentials = grpc.ssl_channel_credentials( - certificate_chain=cert, private_key=key - ) - else: - ssl_credentials = SslCredentials().ssl_credentials - - # create a new channel. The provided one is ignored. - self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - ssl_credentials=ssl_credentials, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - ) - self._ssl_channel_credentials = ssl_credentials else: - host = host if ":" in host else host + ":443" + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials - if credentials is None: - credentials, _ = auth.default( - scopes=self.AUTH_SCOPES, quota_project_id=quota_project_id - ) - - # create a new channel. The provided one is ignored. - self._grpc_channel = type(self).create_channel( - host, - credentials=credentials, - credentials_file=credentials_file, - ssl_credentials=ssl_channel_credentials, - scopes=scopes or self.AUTH_SCOPES, - quota_project_id=quota_project_id, - ) + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) - # Run the base constructor. + # The base transport sets the host, credentials and scopes super().__init__( host=host, credentials=credentials, credentials_file=credentials_file, - scopes=scopes or self.AUTH_SCOPES, + scopes=scopes, quota_project_id=quota_project_id, client_info=client_info, + always_use_jwt_access=always_use_jwt_access, ) - self._stubs = {} + if not self._grpc_channel: + self._grpc_channel = type(self).create_channel( + self._host, + credentials=self._credentials, + credentials_file=credentials_file, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) @property def grpc_channel(self) -> aio.Channel: @@ -431,5 +425,8 @@ def reserve_ids( ) return self._stubs["reserve_ids"] + def close(self): + return self.grpc_channel.close() + __all__ = ("DatastoreGrpcAsyncIOTransport",) diff --git a/google/cloud/datastore_v1/types/__init__.py b/google/cloud/datastore_v1/types/__init__.py index 2148caa0..7553ac77 100644 --- a/google/cloud/datastore_v1/types/__init__.py +++ b/google/cloud/datastore_v1/types/__init__.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,84 +13,82 @@ # See the License for the specific language governing permissions and # limitations under the License. # - +from .datastore import ( + AllocateIdsRequest, + AllocateIdsResponse, + BeginTransactionRequest, + BeginTransactionResponse, + CommitRequest, + CommitResponse, + LookupRequest, + LookupResponse, + Mutation, + MutationResult, + ReadOptions, + ReserveIdsRequest, + ReserveIdsResponse, + RollbackRequest, + RollbackResponse, + RunQueryRequest, + RunQueryResponse, + TransactionOptions, +) from .entity import ( - PartitionId, - Key, ArrayValue, - Value, Entity, + Key, + PartitionId, + Value, ) from .query import ( + CompositeFilter, EntityResult, - Query, - KindExpression, - PropertyReference, - Projection, - PropertyOrder, Filter, - CompositeFilter, - PropertyFilter, GqlQuery, GqlQueryParameter, + KindExpression, + Projection, + PropertyFilter, + PropertyOrder, + PropertyReference, + Query, QueryResultBatch, ) -from .datastore import ( - LookupRequest, - LookupResponse, - RunQueryRequest, - RunQueryResponse, - BeginTransactionRequest, - BeginTransactionResponse, - RollbackRequest, - RollbackResponse, - CommitRequest, - CommitResponse, - AllocateIdsRequest, - AllocateIdsResponse, - ReserveIdsRequest, - ReserveIdsResponse, - Mutation, - MutationResult, - ReadOptions, - TransactionOptions, -) - __all__ = ( - "PartitionId", - "Key", - "ArrayValue", - "Value", - "Entity", - "EntityResult", - "Query", - "KindExpression", - "PropertyReference", - "Projection", - "PropertyOrder", - "Filter", - "CompositeFilter", - "PropertyFilter", - "GqlQuery", - "GqlQueryParameter", - "QueryResultBatch", - "LookupRequest", - "LookupResponse", - "RunQueryRequest", - "RunQueryResponse", + "AllocateIdsRequest", + "AllocateIdsResponse", "BeginTransactionRequest", "BeginTransactionResponse", - "RollbackRequest", - "RollbackResponse", "CommitRequest", "CommitResponse", - "AllocateIdsRequest", - "AllocateIdsResponse", - "ReserveIdsRequest", - "ReserveIdsResponse", + "LookupRequest", + "LookupResponse", "Mutation", "MutationResult", "ReadOptions", + "ReserveIdsRequest", + "ReserveIdsResponse", + "RollbackRequest", + "RollbackResponse", + "RunQueryRequest", + "RunQueryResponse", "TransactionOptions", + "ArrayValue", + "Entity", + "Key", + "PartitionId", + "Value", + "CompositeFilter", + "EntityResult", + "Filter", + "GqlQuery", + "GqlQueryParameter", + "KindExpression", + "Projection", + "PropertyFilter", + "PropertyOrder", + "PropertyReference", + "Query", + "QueryResultBatch", ) diff --git a/google/cloud/datastore_v1/types/datastore.py b/google/cloud/datastore_v1/types/datastore.py index e1124457..a36a7293 100644 --- a/google/cloud/datastore_v1/types/datastore.py +++ b/google/cloud/datastore_v1/types/datastore.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,10 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import proto # type: ignore - from google.cloud.datastore_v1.types import entity from google.cloud.datastore_v1.types import query as gd_query @@ -55,16 +52,14 @@ class LookupRequest(proto.Message): project_id (str): Required. The ID of the project against which to make the request. - read_options (~.datastore.ReadOptions): + read_options (google.cloud.datastore_v1.types.ReadOptions): The options for this lookup request. - keys (Sequence[~.entity.Key]): + keys (Sequence[google.cloud.datastore_v1.types.Key]): Required. Keys of entities to look up. """ - project_id = proto.Field(proto.STRING, number=8) - + project_id = proto.Field(proto.STRING, number=8,) read_options = proto.Field(proto.MESSAGE, number=1, message="ReadOptions",) - keys = proto.RepeatedField(proto.MESSAGE, number=3, message=entity.Key,) @@ -73,15 +68,15 @@ class LookupResponse(proto.Message): [Datastore.Lookup][google.datastore.v1.Datastore.Lookup]. Attributes: - found (Sequence[~.gd_query.EntityResult]): + found (Sequence[google.cloud.datastore_v1.types.EntityResult]): Entities found as ``ResultType.FULL`` entities. The order of results in this field is undefined and has no relation to the order of the keys in the input. - missing (Sequence[~.gd_query.EntityResult]): + missing (Sequence[google.cloud.datastore_v1.types.EntityResult]): Entities not found as ``ResultType.KEY_ONLY`` entities. The order of results in this field is undefined and has no relation to the order of the keys in the input. - deferred (Sequence[~.entity.Key]): + deferred (Sequence[google.cloud.datastore_v1.types.Key]): A list of keys that were not looked up due to resource constraints. The order of results in this field is undefined and has no relation to @@ -89,11 +84,9 @@ class LookupResponse(proto.Message): """ found = proto.RepeatedField(proto.MESSAGE, number=1, message=gd_query.EntityResult,) - missing = proto.RepeatedField( proto.MESSAGE, number=2, message=gd_query.EntityResult, ) - deferred = proto.RepeatedField(proto.MESSAGE, number=3, message=entity.Key,) @@ -101,34 +94,39 @@ class RunQueryRequest(proto.Message): r"""The request for [Datastore.RunQuery][google.datastore.v1.Datastore.RunQuery]. + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + Attributes: project_id (str): Required. The ID of the project against which to make the request. - partition_id (~.entity.PartitionId): + partition_id (google.cloud.datastore_v1.types.PartitionId): Entities are partitioned into subsets, identified by a partition ID. Queries are scoped to a single partition. This partition ID is normalized with the standard default context partition ID. - read_options (~.datastore.ReadOptions): + read_options (google.cloud.datastore_v1.types.ReadOptions): The options for this query. - query (~.gd_query.Query): + query (google.cloud.datastore_v1.types.Query): The query to run. - gql_query (~.gd_query.GqlQuery): + This field is a member of `oneof`_ ``query_type``. + gql_query (google.cloud.datastore_v1.types.GqlQuery): The GQL query to run. + This field is a member of `oneof`_ ``query_type``. """ - project_id = proto.Field(proto.STRING, number=8) - + project_id = proto.Field(proto.STRING, number=8,) partition_id = proto.Field(proto.MESSAGE, number=2, message=entity.PartitionId,) - read_options = proto.Field(proto.MESSAGE, number=1, message="ReadOptions",) - query = proto.Field( proto.MESSAGE, number=3, oneof="query_type", message=gd_query.Query, ) - gql_query = proto.Field( proto.MESSAGE, number=7, oneof="query_type", message=gd_query.GqlQuery, ) @@ -139,15 +137,14 @@ class RunQueryResponse(proto.Message): [Datastore.RunQuery][google.datastore.v1.Datastore.RunQuery]. Attributes: - batch (~.gd_query.QueryResultBatch): + batch (google.cloud.datastore_v1.types.QueryResultBatch): A batch of query results (always present). - query (~.gd_query.Query): + query (google.cloud.datastore_v1.types.Query): The parsed form of the ``GqlQuery`` from the request, if it was set. """ batch = proto.Field(proto.MESSAGE, number=1, message=gd_query.QueryResultBatch,) - query = proto.Field(proto.MESSAGE, number=2, message=gd_query.Query,) @@ -159,12 +156,11 @@ class BeginTransactionRequest(proto.Message): project_id (str): Required. The ID of the project against which to make the request. - transaction_options (~.datastore.TransactionOptions): + transaction_options (google.cloud.datastore_v1.types.TransactionOptions): Options for a new transaction. """ - project_id = proto.Field(proto.STRING, number=8) - + project_id = proto.Field(proto.STRING, number=8,) transaction_options = proto.Field( proto.MESSAGE, number=10, message="TransactionOptions", ) @@ -179,7 +175,7 @@ class BeginTransactionResponse(proto.Message): The transaction identifier (always present). """ - transaction = proto.Field(proto.BYTES, number=1) + transaction = proto.Field(proto.BYTES, number=1,) class RollbackRequest(proto.Message): @@ -195,15 +191,15 @@ class RollbackRequest(proto.Message): [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction]. """ - project_id = proto.Field(proto.STRING, number=8) - - transaction = proto.Field(proto.BYTES, number=1) + project_id = proto.Field(proto.STRING, number=8,) + transaction = proto.Field(proto.BYTES, number=1,) class RollbackResponse(proto.Message): r"""The response for [Datastore.Rollback][google.datastore.v1.Datastore.Rollback]. (an empty message). + """ @@ -211,18 +207,22 @@ class CommitRequest(proto.Message): r"""The request for [Datastore.Commit][google.datastore.v1.Datastore.Commit]. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + Attributes: project_id (str): Required. The ID of the project against which to make the request. - mode (~.datastore.CommitRequest.Mode): + mode (google.cloud.datastore_v1.types.CommitRequest.Mode): The type of commit to perform. Defaults to ``TRANSACTIONAL``. transaction (bytes): The identifier of the transaction associated with the commit. A transaction identifier is returned by a call to [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction]. - mutations (Sequence[~.datastore.Mutation]): + This field is a member of `oneof`_ ``transaction_selector``. + mutations (Sequence[google.cloud.datastore_v1.types.Mutation]): The mutations to perform. When mode is ``TRANSACTIONAL``, mutations affecting a single @@ -245,12 +245,9 @@ class Mode(proto.Enum): TRANSACTIONAL = 1 NON_TRANSACTIONAL = 2 - project_id = proto.Field(proto.STRING, number=8) - + project_id = proto.Field(proto.STRING, number=8,) mode = proto.Field(proto.ENUM, number=5, enum=Mode,) - - transaction = proto.Field(proto.BYTES, number=1, oneof="transaction_selector") - + transaction = proto.Field(proto.BYTES, number=1, oneof="transaction_selector",) mutations = proto.RepeatedField(proto.MESSAGE, number=6, message="Mutation",) @@ -259,7 +256,7 @@ class CommitResponse(proto.Message): [Datastore.Commit][google.datastore.v1.Datastore.Commit]. Attributes: - mutation_results (Sequence[~.datastore.MutationResult]): + mutation_results (Sequence[google.cloud.datastore_v1.types.MutationResult]): The result of performing the mutations. The i-th mutation result corresponds to the i-th mutation in the request. @@ -271,8 +268,7 @@ class CommitResponse(proto.Message): mutation_results = proto.RepeatedField( proto.MESSAGE, number=3, message="MutationResult", ) - - index_updates = proto.Field(proto.INT32, number=4) + index_updates = proto.Field(proto.INT32, number=4,) class AllocateIdsRequest(proto.Message): @@ -283,14 +279,13 @@ class AllocateIdsRequest(proto.Message): project_id (str): Required. The ID of the project against which to make the request. - keys (Sequence[~.entity.Key]): + keys (Sequence[google.cloud.datastore_v1.types.Key]): Required. A list of keys with incomplete key paths for which to allocate IDs. No key may be reserved/read-only. """ - project_id = proto.Field(proto.STRING, number=8) - + project_id = proto.Field(proto.STRING, number=8,) keys = proto.RepeatedField(proto.MESSAGE, number=1, message=entity.Key,) @@ -299,7 +294,7 @@ class AllocateIdsResponse(proto.Message): [Datastore.AllocateIds][google.datastore.v1.Datastore.AllocateIds]. Attributes: - keys (Sequence[~.entity.Key]): + keys (Sequence[google.cloud.datastore_v1.types.Key]): The keys specified in the request (in the same order), each with its key path completed with a newly allocated ID. @@ -319,70 +314,77 @@ class ReserveIdsRequest(proto.Message): database_id (str): If not empty, the ID of the database against which to make the request. - keys (Sequence[~.entity.Key]): + keys (Sequence[google.cloud.datastore_v1.types.Key]): Required. A list of keys with complete key paths whose numeric IDs should not be auto- allocated. """ - project_id = proto.Field(proto.STRING, number=8) - - database_id = proto.Field(proto.STRING, number=9) - + project_id = proto.Field(proto.STRING, number=8,) + database_id = proto.Field(proto.STRING, number=9,) keys = proto.RepeatedField(proto.MESSAGE, number=1, message=entity.Key,) class ReserveIdsResponse(proto.Message): r"""The response for [Datastore.ReserveIds][google.datastore.v1.Datastore.ReserveIds]. + """ class Mutation(proto.Message): r"""A mutation to apply to an entity. + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + Attributes: - insert (~.entity.Entity): + insert (google.cloud.datastore_v1.types.Entity): The entity to insert. The entity must not already exist. The entity key's final path element may be incomplete. - update (~.entity.Entity): + This field is a member of `oneof`_ ``operation``. + update (google.cloud.datastore_v1.types.Entity): The entity to update. The entity must already exist. Must have a complete key path. - upsert (~.entity.Entity): + This field is a member of `oneof`_ ``operation``. + upsert (google.cloud.datastore_v1.types.Entity): The entity to upsert. The entity may or may not already exist. The entity key's final path element may be incomplete. - delete (~.entity.Key): + This field is a member of `oneof`_ ``operation``. + delete (google.cloud.datastore_v1.types.Key): The key of the entity to delete. The entity may or may not already exist. Must have a complete key path and must not be reserved/read- only. + This field is a member of `oneof`_ ``operation``. base_version (int): The version of the entity that this mutation is being applied to. If this does not match the current version on the server, the mutation conflicts. + This field is a member of `oneof`_ ``conflict_detection_strategy``. """ insert = proto.Field( proto.MESSAGE, number=4, oneof="operation", message=entity.Entity, ) - update = proto.Field( proto.MESSAGE, number=5, oneof="operation", message=entity.Entity, ) - upsert = proto.Field( proto.MESSAGE, number=6, oneof="operation", message=entity.Entity, ) - delete = proto.Field( proto.MESSAGE, number=7, oneof="operation", message=entity.Key, ) - base_version = proto.Field( - proto.INT64, number=8, oneof="conflict_detection_strategy" + proto.INT64, number=8, oneof="conflict_detection_strategy", ) @@ -390,7 +392,7 @@ class MutationResult(proto.Message): r"""The result of applying a mutation. Attributes: - key (~.entity.Key): + key (google.cloud.datastore_v1.types.Key): The automatically allocated key. Set only when the mutation allocated a key. version (int): @@ -409,23 +411,30 @@ class MutationResult(proto.Message): """ key = proto.Field(proto.MESSAGE, number=3, message=entity.Key,) - - version = proto.Field(proto.INT64, number=4) - - conflict_detected = proto.Field(proto.BOOL, number=5) + version = proto.Field(proto.INT64, number=4,) + conflict_detected = proto.Field(proto.BOOL, number=5,) class ReadOptions(proto.Message): r"""The options shared by read requests. + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + Attributes: - read_consistency (~.datastore.ReadOptions.ReadConsistency): + read_consistency (google.cloud.datastore_v1.types.ReadOptions.ReadConsistency): The non-transactional read consistency to use. Cannot be set to ``STRONG`` for global queries. + This field is a member of `oneof`_ ``consistency_type``. transaction (bytes): The identifier of the transaction in which to read. A transaction identifier is returned by a call to [Datastore.BeginTransaction][google.datastore.v1.Datastore.BeginTransaction]. + This field is a member of `oneof`_ ``consistency_type``. """ class ReadConsistency(proto.Enum): @@ -437,8 +446,7 @@ class ReadConsistency(proto.Enum): read_consistency = proto.Field( proto.ENUM, number=1, oneof="consistency_type", enum=ReadConsistency, ) - - transaction = proto.Field(proto.BYTES, number=2, oneof="consistency_type") + transaction = proto.Field(proto.BYTES, number=2, oneof="consistency_type",) class TransactionOptions(proto.Message): @@ -450,12 +458,21 @@ class TransactionOptions(proto.Message): [ReadOptions.new_transaction][google.datastore.v1.ReadOptions.new_transaction] in read requests. + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + Attributes: - read_write (~.datastore.TransactionOptions.ReadWrite): + read_write (google.cloud.datastore_v1.types.TransactionOptions.ReadWrite): The transaction should allow both reads and writes. - read_only (~.datastore.TransactionOptions.ReadOnly): + This field is a member of `oneof`_ ``mode``. + read_only (google.cloud.datastore_v1.types.TransactionOptions.ReadOnly): The transaction should only allow reads. + This field is a member of `oneof`_ ``mode``. """ class ReadWrite(proto.Message): @@ -467,13 +484,13 @@ class ReadWrite(proto.Message): being retried. """ - previous_transaction = proto.Field(proto.BYTES, number=1) + previous_transaction = proto.Field(proto.BYTES, number=1,) class ReadOnly(proto.Message): - r"""Options specific to read-only transactions.""" + r"""Options specific to read-only transactions. + """ read_write = proto.Field(proto.MESSAGE, number=1, oneof="mode", message=ReadWrite,) - read_only = proto.Field(proto.MESSAGE, number=2, oneof="mode", message=ReadOnly,) diff --git a/google/cloud/datastore_v1/types/entity.py b/google/cloud/datastore_v1/types/entity.py index cc1be6e2..8ff844f7 100644 --- a/google/cloud/datastore_v1/types/entity.py +++ b/google/cloud/datastore_v1/types/entity.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,13 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import proto # type: ignore - -from google.protobuf import struct_pb2 as struct # type: ignore -from google.protobuf import timestamp_pb2 as timestamp # type: ignore -from google.type import latlng_pb2 as latlng # type: ignore +from google.protobuf import struct_pb2 # type: ignore +from google.protobuf import timestamp_pb2 # type: ignore +from google.type import latlng_pb2 # type: ignore __protobuf__ = proto.module( @@ -59,9 +56,8 @@ class PartitionId(proto.Message): which the entities belong. """ - project_id = proto.Field(proto.STRING, number=2) - - namespace_id = proto.Field(proto.STRING, number=4) + project_id = proto.Field(proto.STRING, number=2,) + namespace_id = proto.Field(proto.STRING, number=4,) class Key(proto.Message): @@ -72,12 +68,12 @@ class Key(proto.Message): contexts. Attributes: - partition_id (~.entity.PartitionId): + partition_id (google.cloud.datastore_v1.types.PartitionId): Entities are partitioned into subsets, currently identified by a project ID and namespace ID. Queries are scoped to a single partition. - path (Sequence[~.entity.Key.PathElement]): + path (Sequence[google.cloud.datastore_v1.types.Key.PathElement]): The entity path. An entity path consists of one or more elements composed of a kind and a string or numerical identifier, which identify entities. The first element @@ -104,6 +100,13 @@ class PathElement(proto.Message): If either name or ID is set, the element is complete. If neither is set, the element is incomplete. + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + Attributes: kind (str): The kind of the entity. A kind matching regex ``__.*__`` is @@ -114,20 +117,19 @@ class PathElement(proto.Message): Never equal to zero. Values less than zero are discouraged and may not be supported in the future. + This field is a member of `oneof`_ ``id_type``. name (str): The name of the entity. A name matching regex ``__.*__`` is reserved/read-only. A name must not be more than 1500 bytes when UTF-8 encoded. Cannot be ``""``. + This field is a member of `oneof`_ ``id_type``. """ - kind = proto.Field(proto.STRING, number=1) - - id = proto.Field(proto.INT64, number=2, oneof="id_type") - - name = proto.Field(proto.STRING, number=3, oneof="id_type") + kind = proto.Field(proto.STRING, number=1,) + id = proto.Field(proto.INT64, number=2, oneof="id_type",) + name = proto.Field(proto.STRING, number=3, oneof="id_type",) partition_id = proto.Field(proto.MESSAGE, number=1, message="PartitionId",) - path = proto.RepeatedField(proto.MESSAGE, number=2, message=PathElement,) @@ -135,7 +137,7 @@ class ArrayValue(proto.Message): r"""An array value. Attributes: - values (Sequence[~.entity.Value]): + values (Sequence[google.cloud.datastore_v1.types.Value]): Values in the array. The order of values in an array is preserved as long as all values have identical settings for 'exclude_from_indexes'. @@ -148,42 +150,60 @@ class Value(proto.Message): r"""A message that can hold any of the supported value types and associated metadata. + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + Attributes: - null_value (~.struct.NullValue): + null_value (google.protobuf.struct_pb2.NullValue): A null value. + This field is a member of `oneof`_ ``value_type``. boolean_value (bool): A boolean value. + This field is a member of `oneof`_ ``value_type``. integer_value (int): An integer value. + This field is a member of `oneof`_ ``value_type``. double_value (float): A double value. - timestamp_value (~.timestamp.Timestamp): + This field is a member of `oneof`_ ``value_type``. + timestamp_value (google.protobuf.timestamp_pb2.Timestamp): A timestamp value. When stored in the Datastore, precise only to microseconds; any additional precision is rounded down. - key_value (~.entity.Key): + This field is a member of `oneof`_ ``value_type``. + key_value (google.cloud.datastore_v1.types.Key): A key value. + This field is a member of `oneof`_ ``value_type``. string_value (str): A UTF-8 encoded string value. When ``exclude_from_indexes`` is false (it is indexed), may have at most 1500 bytes. Otherwise, may be set to at most 1,000,000 bytes. + This field is a member of `oneof`_ ``value_type``. blob_value (bytes): A blob value. May have at most 1,000,000 bytes. When ``exclude_from_indexes`` is false, may have at most 1500 bytes. In JSON requests, must be base64-encoded. - geo_point_value (~.latlng.LatLng): + This field is a member of `oneof`_ ``value_type``. + geo_point_value (google.type.latlng_pb2.LatLng): A geo point value representing a point on the surface of Earth. - entity_value (~.entity.Entity): + This field is a member of `oneof`_ ``value_type``. + entity_value (google.cloud.datastore_v1.types.Entity): An entity value. - May have no key. - May have a key with an incomplete key path. - May have a reserved/read-only key. - array_value (~.entity.ArrayValue): + This field is a member of `oneof`_ ``value_type``. + array_value (google.cloud.datastore_v1.types.ArrayValue): An array value. Cannot contain another array value. A ``Value`` instance that sets field ``array_value`` must not set fields ``meaning`` or ``exclude_from_indexes``. + This field is a member of `oneof`_ ``value_type``. meaning (int): The ``meaning`` field should only be populated for backwards compatibility. @@ -193,40 +213,28 @@ class Value(proto.Message): """ null_value = proto.Field( - proto.ENUM, number=11, oneof="value_type", enum=struct.NullValue, + proto.ENUM, number=11, oneof="value_type", enum=struct_pb2.NullValue, ) - - boolean_value = proto.Field(proto.BOOL, number=1, oneof="value_type") - - integer_value = proto.Field(proto.INT64, number=2, oneof="value_type") - - double_value = proto.Field(proto.DOUBLE, number=3, oneof="value_type") - + boolean_value = proto.Field(proto.BOOL, number=1, oneof="value_type",) + integer_value = proto.Field(proto.INT64, number=2, oneof="value_type",) + double_value = proto.Field(proto.DOUBLE, number=3, oneof="value_type",) timestamp_value = proto.Field( - proto.MESSAGE, number=10, oneof="value_type", message=timestamp.Timestamp, + proto.MESSAGE, number=10, oneof="value_type", message=timestamp_pb2.Timestamp, ) - key_value = proto.Field(proto.MESSAGE, number=5, oneof="value_type", message="Key",) - - string_value = proto.Field(proto.STRING, number=17, oneof="value_type") - - blob_value = proto.Field(proto.BYTES, number=18, oneof="value_type") - + string_value = proto.Field(proto.STRING, number=17, oneof="value_type",) + blob_value = proto.Field(proto.BYTES, number=18, oneof="value_type",) geo_point_value = proto.Field( - proto.MESSAGE, number=8, oneof="value_type", message=latlng.LatLng, + proto.MESSAGE, number=8, oneof="value_type", message=latlng_pb2.LatLng, ) - entity_value = proto.Field( proto.MESSAGE, number=6, oneof="value_type", message="Entity", ) - array_value = proto.Field( proto.MESSAGE, number=9, oneof="value_type", message="ArrayValue", ) - - meaning = proto.Field(proto.INT32, number=14) - - exclude_from_indexes = proto.Field(proto.BOOL, number=19) + meaning = proto.Field(proto.INT32, number=14,) + exclude_from_indexes = proto.Field(proto.BOOL, number=19,) class Entity(proto.Message): @@ -237,14 +245,14 @@ class Entity(proto.Message): message. Attributes: - key (~.entity.Key): + key (google.cloud.datastore_v1.types.Key): The entity's key. An entity must have a key, unless otherwise documented (for example, an entity in ``Value.entity_value`` may have no key). An entity's kind is its key path's last element's kind, or null if it has no key. - properties (Sequence[~.entity.Entity.PropertiesEntry]): + properties (Sequence[google.cloud.datastore_v1.types.Entity.PropertiesEntry]): The entity's properties. The map's keys are property names. A property name matching regex ``__.*__`` is reserved. A reserved property name is forbidden in certain documented @@ -253,7 +261,6 @@ class Entity(proto.Message): """ key = proto.Field(proto.MESSAGE, number=1, message="Key",) - properties = proto.MapField(proto.STRING, proto.MESSAGE, number=3, message="Value",) diff --git a/google/cloud/datastore_v1/types/query.py b/google/cloud/datastore_v1/types/query.py index 173626b0..1c69e89f 100644 --- a/google/cloud/datastore_v1/types/query.py +++ b/google/cloud/datastore_v1/types/query.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,12 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import proto # type: ignore - from google.cloud.datastore_v1.types import entity as gd_entity -from google.protobuf import wrappers_pb2 as wrappers # type: ignore +from google.protobuf import wrappers_pb2 # type: ignore __protobuf__ = proto.module( @@ -45,7 +42,7 @@ class EntityResult(proto.Message): r"""The result of fetching an entity from Datastore. Attributes: - entity (~.gd_entity.Entity): + entity (google.cloud.datastore_v1.types.Entity): The resulting entity. version (int): The version of the entity, a strictly positive number that @@ -79,29 +76,27 @@ class ResultType(proto.Enum): KEY_ONLY = 3 entity = proto.Field(proto.MESSAGE, number=1, message=gd_entity.Entity,) - - version = proto.Field(proto.INT64, number=4) - - cursor = proto.Field(proto.BYTES, number=3) + version = proto.Field(proto.INT64, number=4,) + cursor = proto.Field(proto.BYTES, number=3,) class Query(proto.Message): r"""A query for entities. Attributes: - projection (Sequence[~.query.Projection]): + projection (Sequence[google.cloud.datastore_v1.types.Projection]): The projection to return. Defaults to returning all properties. - kind (Sequence[~.query.KindExpression]): + kind (Sequence[google.cloud.datastore_v1.types.KindExpression]): The kinds to query (if empty, returns entities of all kinds). Currently at most 1 kind may be specified. - filter (~.query.Filter): + filter (google.cloud.datastore_v1.types.Filter): The filter to apply. - order (Sequence[~.query.PropertyOrder]): + order (Sequence[google.cloud.datastore_v1.types.PropertyOrder]): The order to apply to the query results (if empty, order is unspecified). - distinct_on (Sequence[~.query.PropertyReference]): + distinct_on (Sequence[google.cloud.datastore_v1.types.PropertyReference]): The properties to make distinct. The query results will contain the first result for each distinct combination of values for the given @@ -120,7 +115,7 @@ class Query(proto.Message): The number of results to skip. Applies before limit, but after all other constraints. Optional. Must be >= 0 if specified. - limit (~.wrappers.Int32Value): + limit (google.protobuf.wrappers_pb2.Int32Value): The maximum number of results to return. Applies after all other constraints. Optional. Unspecified is interpreted as no limit. @@ -128,24 +123,16 @@ class Query(proto.Message): """ projection = proto.RepeatedField(proto.MESSAGE, number=2, message="Projection",) - kind = proto.RepeatedField(proto.MESSAGE, number=3, message="KindExpression",) - filter = proto.Field(proto.MESSAGE, number=4, message="Filter",) - order = proto.RepeatedField(proto.MESSAGE, number=5, message="PropertyOrder",) - distinct_on = proto.RepeatedField( proto.MESSAGE, number=6, message="PropertyReference", ) - - start_cursor = proto.Field(proto.BYTES, number=7) - - end_cursor = proto.Field(proto.BYTES, number=8) - - offset = proto.Field(proto.INT32, number=10) - - limit = proto.Field(proto.MESSAGE, number=12, message=wrappers.Int32Value,) + start_cursor = proto.Field(proto.BYTES, number=7,) + end_cursor = proto.Field(proto.BYTES, number=8,) + offset = proto.Field(proto.INT32, number=10,) + limit = proto.Field(proto.MESSAGE, number=12, message=wrappers_pb2.Int32Value,) class KindExpression(proto.Message): @@ -156,7 +143,7 @@ class KindExpression(proto.Message): The name of the kind. """ - name = proto.Field(proto.STRING, number=1) + name = proto.Field(proto.STRING, number=1,) class PropertyReference(proto.Message): @@ -169,14 +156,14 @@ class PropertyReference(proto.Message): a property name path. """ - name = proto.Field(proto.STRING, number=2) + name = proto.Field(proto.STRING, number=2,) class Projection(proto.Message): r"""A representation of a property in a projection. Attributes: - property (~.query.PropertyReference): + property (google.cloud.datastore_v1.types.PropertyReference): The property to project. """ @@ -187,9 +174,9 @@ class PropertyOrder(proto.Message): r"""The desired order for a specific property. Attributes: - property (~.query.PropertyReference): + property (google.cloud.datastore_v1.types.PropertyReference): The property to order by. - direction (~.query.PropertyOrder.Direction): + direction (google.cloud.datastore_v1.types.PropertyOrder.Direction): The direction to order by. Defaults to ``ASCENDING``. """ @@ -200,24 +187,31 @@ class Direction(proto.Enum): DESCENDING = 2 property = proto.Field(proto.MESSAGE, number=1, message="PropertyReference",) - direction = proto.Field(proto.ENUM, number=2, enum=Direction,) class Filter(proto.Message): r"""A holder for any type of filter. + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + Attributes: - composite_filter (~.query.CompositeFilter): + composite_filter (google.cloud.datastore_v1.types.CompositeFilter): A composite filter. - property_filter (~.query.PropertyFilter): + This field is a member of `oneof`_ ``filter_type``. + property_filter (google.cloud.datastore_v1.types.PropertyFilter): A filter on a property. + This field is a member of `oneof`_ ``filter_type``. """ composite_filter = proto.Field( proto.MESSAGE, number=1, oneof="filter_type", message="CompositeFilter", ) - property_filter = proto.Field( proto.MESSAGE, number=2, oneof="filter_type", message="PropertyFilter", ) @@ -228,9 +222,9 @@ class CompositeFilter(proto.Message): operator. Attributes: - op (~.query.CompositeFilter.Operator): + op (google.cloud.datastore_v1.types.CompositeFilter.Operator): The operator for combining multiple filters. - filters (Sequence[~.query.Filter]): + filters (Sequence[google.cloud.datastore_v1.types.Filter]): The list of filters to combine. Must contain at least one filter. """ @@ -241,7 +235,6 @@ class Operator(proto.Enum): AND = 1 op = proto.Field(proto.ENUM, number=1, enum=Operator,) - filters = proto.RepeatedField(proto.MESSAGE, number=2, message="Filter",) @@ -249,11 +242,11 @@ class PropertyFilter(proto.Message): r"""A filter on a specific property. Attributes: - property (~.query.PropertyReference): + property (google.cloud.datastore_v1.types.PropertyReference): The property to filter by. - op (~.query.PropertyFilter.Operator): + op (google.cloud.datastore_v1.types.PropertyFilter.Operator): The operator to filter by. - value (~.gd_entity.Value): + value (google.cloud.datastore_v1.types.Value): The value to compare the property to. """ @@ -268,9 +261,7 @@ class Operator(proto.Enum): HAS_ANCESTOR = 11 property = proto.Field(proto.MESSAGE, number=1, message="PropertyReference",) - op = proto.Field(proto.ENUM, number=2, enum=Operator,) - value = proto.Field(proto.MESSAGE, number=3, message=gd_entity.Value,) @@ -287,14 +278,14 @@ class GqlQuery(proto.Message): and instead must bind all values. For example, ``SELECT * FROM Kind WHERE a = 'string literal'`` is not allowed, while ``SELECT * FROM Kind WHERE a = @value`` is. - named_bindings (Sequence[~.query.GqlQuery.NamedBindingsEntry]): + named_bindings (Sequence[google.cloud.datastore_v1.types.GqlQuery.NamedBindingsEntry]): For each non-reserved named binding site in the query string, there must be a named parameter with that name, but not necessarily the inverse. Key must match regex ``[A-Za-z_$][A-Za-z_$0-9]*``, must not match regex ``__.*__``, and must not be ``""``. - positional_bindings (Sequence[~.query.GqlQueryParameter]): + positional_bindings (Sequence[google.cloud.datastore_v1.types.GqlQueryParameter]): Numbered binding site @1 references the first numbered parameter, effectively using 1-based indexing, rather than the usual 0. @@ -304,14 +295,11 @@ class GqlQuery(proto.Message): true. """ - query_string = proto.Field(proto.STRING, number=1) - - allow_literals = proto.Field(proto.BOOL, number=2) - + query_string = proto.Field(proto.STRING, number=1,) + allow_literals = proto.Field(proto.BOOL, number=2,) named_bindings = proto.MapField( proto.STRING, proto.MESSAGE, number=5, message="GqlQueryParameter", ) - positional_bindings = proto.RepeatedField( proto.MESSAGE, number=4, message="GqlQueryParameter", ) @@ -320,19 +308,27 @@ class GqlQuery(proto.Message): class GqlQueryParameter(proto.Message): r"""A binding parameter for a GQL query. + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + Attributes: - value (~.gd_entity.Value): + value (google.cloud.datastore_v1.types.Value): A value parameter. + This field is a member of `oneof`_ ``parameter_type``. cursor (bytes): A query cursor. Query cursors are returned in query result batches. + This field is a member of `oneof`_ ``parameter_type``. """ value = proto.Field( proto.MESSAGE, number=2, oneof="parameter_type", message=gd_entity.Value, ) - - cursor = proto.Field(proto.BYTES, number=3, oneof="parameter_type") + cursor = proto.Field(proto.BYTES, number=3, oneof="parameter_type",) class QueryResultBatch(proto.Message): @@ -345,14 +341,14 @@ class QueryResultBatch(proto.Message): skipped_cursor (bytes): A cursor that points to the position after the last skipped result. Will be set when ``skipped_results`` != 0. - entity_result_type (~.query.EntityResult.ResultType): + entity_result_type (google.cloud.datastore_v1.types.EntityResult.ResultType): The result type for every entity in ``entity_results``. - entity_results (Sequence[~.query.EntityResult]): + entity_results (Sequence[google.cloud.datastore_v1.types.EntityResult]): The results for this batch. end_cursor (bytes): A cursor that points to the position after the last result in the batch. - more_results (~.query.QueryResultBatch.MoreResultsType): + more_results (google.cloud.datastore_v1.types.QueryResultBatch.MoreResultsType): The state of the query after the current batch. snapshot_version (int): @@ -377,23 +373,17 @@ class MoreResultsType(proto.Enum): MORE_RESULTS_AFTER_CURSOR = 4 NO_MORE_RESULTS = 3 - skipped_results = proto.Field(proto.INT32, number=6) - - skipped_cursor = proto.Field(proto.BYTES, number=3) - + skipped_results = proto.Field(proto.INT32, number=6,) + skipped_cursor = proto.Field(proto.BYTES, number=3,) entity_result_type = proto.Field( proto.ENUM, number=1, enum="EntityResult.ResultType", ) - entity_results = proto.RepeatedField( proto.MESSAGE, number=2, message="EntityResult", ) - - end_cursor = proto.Field(proto.BYTES, number=4) - + end_cursor = proto.Field(proto.BYTES, number=4,) more_results = proto.Field(proto.ENUM, number=5, enum=MoreResultsType,) - - snapshot_version = proto.Field(proto.INT64, number=7) + snapshot_version = proto.Field(proto.INT64, number=7,) __all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owlbot.py b/owlbot.py index 0ad059b7..e5b43cca 100644 --- a/owlbot.py +++ b/owlbot.py @@ -13,41 +13,80 @@ # limitations under the License. """This script is used to synthesize generated parts of this library.""" +from pathlib import Path +from typing import List, Optional + import synthtool as s from synthtool import gcp from synthtool.languages import python common = gcp.CommonTemplates() +# This is a customized version of the s.get_staging_dirs() function from synthtool to +# cater for copying 2 different folders from googleapis-gen +# which are datastore and datastore/admin +# Source https://github.com/googleapis/synthtool/blob/master/synthtool/transforms.py#L280 +def get_staging_dirs( + default_version: Optional[str] = None, sub_directory: Optional[str] = None +) -> List[Path]: + """Returns the list of directories, one per version, copied from + https://github.com/googleapis/googleapis-gen. Will return in lexical sorting + order with the exception of the default_version which will be last (if specified). + Args: + default_version (str): the default version of the API. The directory for this version + will be the last item in the returned list if specified. + sub_directory (str): if a `sub_directory` is provided, only the directories within the + specified `sub_directory` will be returned. + Returns: the empty list if no file were copied. + """ + + staging = Path("owl-bot-staging") + + if sub_directory: + staging /= sub_directory + + if staging.is_dir(): + # Collect the subdirectories of the staging directory. + versions = [v.name for v in staging.iterdir() if v.is_dir()] + # Reorder the versions so the default version always comes last. + versions = [v for v in versions if v != default_version] + versions.sort() + if default_version is not None: + versions += [default_version] + dirs = [staging / v for v in versions] + for dir in dirs: + s._tracked_paths.add(dir) + return dirs + else: + return [] + # This library ships clients for two different APIs, # Datastore and Datastore Admin datastore_default_version = "v1" datastore_admin_default_version = "v1" -for library in s.get_staging_dirs(datastore_default_version): - if library.parent.absolute() == "datastore": - s.move(library / f"google/cloud/datastore_{library.name}") - s.move(library / "tests/") - s.move(library / "scripts") - -for library in s.get_staging_dirs(datastore_admin_default_version): - if library.parent.absolute() == "datastore_admin": - s.replace( - library / "google/**/datastore_admin_client.py", - "google-cloud-datastore-admin", - "google-cloud-datstore", - ) - - # Remove spurious markup - s.replace( - "google/**/datastore_admin/client.py", - r"\s+---------------------------------(-)+", - "", - ) - - s.move(library / f"google/cloud/datastore_admin_{library.name}") - s.move(library / "tests") - s.move(library / "scripts") +for library in get_staging_dirs(datastore_default_version, "datastore"): + s.move(library / f"google/cloud/datastore_{library.name}") + s.move(library / "tests/") + s.move(library / "scripts") + +for library in get_staging_dirs(datastore_admin_default_version, "datastore_admin"): + s.replace( + library / "google/**/datastore_admin_client.py", + "google-cloud-datastore-admin", + "google-cloud-datstore", + ) + + # Remove spurious markup + s.replace( + library / "google/**/datastore_admin/client.py", + r"\s+---------------------------------(-)+", + "", + ) + + s.move(library / f"google/cloud/datastore_admin_{library.name}") + s.move(library / "tests") + s.move(library / "scripts") s.remove_staging_dirs() diff --git a/scripts/fixup_datastore_admin_v1_keywords.py b/scripts/fixup_datastore_admin_v1_keywords.py index fae3ea91..12e217de 100644 --- a/scripts/fixup_datastore_admin_v1_keywords.py +++ b/scripts/fixup_datastore_admin_v1_keywords.py @@ -1,6 +1,5 @@ #! /usr/bin/env python3 # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import argparse import os import libcst as cst @@ -41,11 +39,12 @@ def partition( class datastore_adminCallTransformer(cst.CSTTransformer): CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { - 'export_entities': ('project_id', 'output_url_prefix', 'labels', 'entity_filter', ), - 'get_index': ('project_id', 'index_id', ), - 'import_entities': ('project_id', 'input_url', 'labels', 'entity_filter', ), - 'list_indexes': ('project_id', 'filter', 'page_size', 'page_token', ), - + 'create_index': ('project_id', 'index', ), + 'delete_index': ('project_id', 'index_id', ), + 'export_entities': ('project_id', 'output_url_prefix', 'labels', 'entity_filter', ), + 'get_index': ('project_id', 'index_id', ), + 'import_entities': ('project_id', 'input_url', 'labels', 'entity_filter', ), + 'list_indexes': ('project_id', 'filter', 'page_size', 'page_token', ), } def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: @@ -64,7 +63,7 @@ def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: return updated kwargs, ctrl_kwargs = partition( - lambda a: not a.keyword.value in self.CTRL_PARAMS, + lambda a: a.keyword.value not in self.CTRL_PARAMS, kwargs ) @@ -76,7 +75,7 @@ def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: value=cst.Dict([ cst.DictElement( cst.SimpleString("'{}'".format(name)), - cst.Element(value=arg.value) +cst.Element(value=arg.value) ) # Note: the args + kwargs looks silly, but keep in mind that # the control parameters had to be stripped out, and that diff --git a/scripts/fixup_datastore_v1_keywords.py b/scripts/fixup_datastore_v1_keywords.py index 8b04f6fe..e0358795 100644 --- a/scripts/fixup_datastore_v1_keywords.py +++ b/scripts/fixup_datastore_v1_keywords.py @@ -1,6 +1,5 @@ #! /usr/bin/env python3 # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import argparse import os import libcst as cst @@ -41,14 +39,13 @@ def partition( class datastoreCallTransformer(cst.CSTTransformer): CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { - 'allocate_ids': ('project_id', 'keys', ), - 'begin_transaction': ('project_id', 'transaction_options', ), - 'commit': ('project_id', 'mode', 'transaction', 'mutations', ), - 'lookup': ('project_id', 'keys', 'read_options', ), - 'reserve_ids': ('project_id', 'keys', 'database_id', ), - 'rollback': ('project_id', 'transaction', ), - 'run_query': ('project_id', 'partition_id', 'read_options', 'query', 'gql_query', ), - + 'allocate_ids': ('project_id', 'keys', ), + 'begin_transaction': ('project_id', 'transaction_options', ), + 'commit': ('project_id', 'mode', 'transaction', 'mutations', ), + 'lookup': ('project_id', 'keys', 'read_options', ), + 'reserve_ids': ('project_id', 'keys', 'database_id', ), + 'rollback': ('project_id', 'transaction', ), + 'run_query': ('project_id', 'partition_id', 'read_options', 'query', 'gql_query', ), } def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: @@ -67,7 +64,7 @@ def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: return updated kwargs, ctrl_kwargs = partition( - lambda a: not a.keyword.value in self.CTRL_PARAMS, + lambda a: a.keyword.value not in self.CTRL_PARAMS, kwargs ) @@ -79,7 +76,7 @@ def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: value=cst.Dict([ cst.DictElement( cst.SimpleString("'{}'".format(name)), - cst.Element(value=arg.value) +cst.Element(value=arg.value) ) # Note: the args + kwargs looks silly, but keep in mind that # the control parameters had to be stripped out, and that diff --git a/setup.py b/setup.py index 6550cea3..286653d5 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ # NOTE: Maintainers, please do not require google-api-core>=2.x.x # Until this issue is closed # https://github.com/googleapis/google-cloud-python/issues/10566 - "google-api-core[grpc] >= 1.22.2, <3.0.0dev", + "google-api-core[grpc] >= 1.28.0, <3.0.0dev", # NOTE: Maintainers, please do not require google-api-core>=2.x.x # Until this issue is closed # https://github.com/googleapis/google-cloud-python/issues/10566 diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 01fc45a4..1800ac45 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -5,8 +5,7 @@ # # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 -google-api-core==1.22.2 +google-api-core==1.28.0 google-cloud-core==1.4.0 proto-plus==1.4.0 libcst==0.2.5 -google-auth==1.24.0 # TODO: remove when google-auth>=1.25.0 is required through google-api-core \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py index e69de29b..4de65971 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index df379f1e..4de65971 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -1,4 +1,5 @@ -# Copyright 2016 Google LLC +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,3 +12,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# diff --git a/tests/unit/gapic/__init__.py b/tests/unit/gapic/__init__.py new file mode 100644 index 00000000..4de65971 --- /dev/null +++ b/tests/unit/gapic/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/unit/gapic/datastore_admin_v1/__init__.py b/tests/unit/gapic/datastore_admin_v1/__init__.py index 8b137891..4de65971 100644 --- a/tests/unit/gapic/datastore_admin_v1/__init__.py +++ b/tests/unit/gapic/datastore_admin_v1/__init__.py @@ -1 +1,15 @@ - +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/unit/gapic/datastore_admin_v1/test_datastore_admin.py b/tests/unit/gapic/datastore_admin_v1/test_datastore_admin.py index 54320c97..a8f4a7b6 100644 --- a/tests/unit/gapic/datastore_admin_v1/test_datastore_admin.py +++ b/tests/unit/gapic/datastore_admin_v1/test_datastore_admin.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import os import mock @@ -24,16 +22,17 @@ import pytest from proto.marshal.rules.dates import DurationRule, TimestampRule -from google import auth + from google.api_core import client_options -from google.api_core import exceptions +from google.api_core import exceptions as core_exceptions from google.api_core import future from google.api_core import gapic_v1 from google.api_core import grpc_helpers from google.api_core import grpc_helpers_async from google.api_core import operation_async # type: ignore from google.api_core import operations_v1 -from google.auth import credentials +from google.api_core import path_template +from google.auth import credentials as ga_credentials from google.auth.exceptions import MutualTLSChannelError from google.cloud.datastore_admin_v1.services.datastore_admin import ( DatastoreAdminAsyncClient, @@ -47,6 +46,7 @@ from google.cloud.datastore_admin_v1.types import index from google.longrunning import operations_pb2 from google.oauth2 import service_account +import google.auth def client_cert_source_callback(): @@ -94,26 +94,73 @@ def test__get_default_mtls_endpoint(): @pytest.mark.parametrize( - "client_class", [DatastoreAdminClient, DatastoreAdminAsyncClient] + "client_class", [DatastoreAdminClient, DatastoreAdminAsyncClient,] +) +def test_datastore_admin_client_from_service_account_info(client_class): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object( + service_account.Credentials, "from_service_account_info" + ) as factory: + factory.return_value = creds + info = {"valid": True} + client = client_class.from_service_account_info(info) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == "datastore.googleapis.com:443" + + +@pytest.mark.parametrize( + "transport_class,transport_name", + [ + (transports.DatastoreAdminGrpcTransport, "grpc"), + (transports.DatastoreAdminGrpcAsyncIOTransport, "grpc_asyncio"), + ], +) +def test_datastore_admin_client_service_account_always_use_jwt( + transport_class, transport_name +): + with mock.patch.object( + service_account.Credentials, "with_always_use_jwt_access", create=True + ) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=True) + use_jwt.assert_called_once_with(True) + + with mock.patch.object( + service_account.Credentials, "with_always_use_jwt_access", create=True + ) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=False) + use_jwt.assert_not_called() + + +@pytest.mark.parametrize( + "client_class", [DatastoreAdminClient, DatastoreAdminAsyncClient,] ) def test_datastore_admin_client_from_service_account_file(client_class): - creds = credentials.AnonymousCredentials() + creds = ga_credentials.AnonymousCredentials() with mock.patch.object( service_account.Credentials, "from_service_account_file" ) as factory: factory.return_value = creds client = client_class.from_service_account_file("dummy/file/path.json") assert client.transport._credentials == creds + assert isinstance(client, client_class) client = client_class.from_service_account_json("dummy/file/path.json") assert client.transport._credentials == creds + assert isinstance(client, client_class) assert client.transport._host == "datastore.googleapis.com:443" def test_datastore_admin_client_get_transport_class(): transport = DatastoreAdminClient.get_transport_class() - assert transport == transports.DatastoreAdminGrpcTransport + available_transports = [ + transports.DatastoreAdminGrpcTransport, + ] + assert transport in available_transports transport = DatastoreAdminClient.get_transport_class("grpc") assert transport == transports.DatastoreAdminGrpcTransport @@ -145,7 +192,7 @@ def test_datastore_admin_client_client_options( ): # Check that if channel is provided we won't create a new one. with mock.patch.object(DatastoreAdminClient, "get_transport_class") as gtc: - transport = transport_class(credentials=credentials.AnonymousCredentials()) + transport = transport_class(credentials=ga_credentials.AnonymousCredentials()) client = client_class(transport=transport) gtc.assert_not_called() @@ -158,15 +205,16 @@ def test_datastore_admin_client_client_options( options = client_options.ClientOptions(api_endpoint="squid.clam.whelk") with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class(client_options=options) + client = client_class(transport=transport_name, client_options=options) patched.assert_called_once_with( credentials=None, credentials_file=None, host="squid.clam.whelk", scopes=None, - ssl_channel_credentials=None, + client_cert_source_for_mtls=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, ) # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is @@ -174,15 +222,16 @@ def test_datastore_admin_client_client_options( with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class() + client = client_class(transport=transport_name) patched.assert_called_once_with( credentials=None, credentials_file=None, host=client.DEFAULT_ENDPOINT, scopes=None, - ssl_channel_credentials=None, + client_cert_source_for_mtls=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, ) # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is @@ -190,15 +239,16 @@ def test_datastore_admin_client_client_options( with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class() + client = client_class(transport=transport_name) patched.assert_called_once_with( credentials=None, credentials_file=None, host=client.DEFAULT_MTLS_ENDPOINT, scopes=None, - ssl_channel_credentials=None, + client_cert_source_for_mtls=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, ) # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has @@ -218,15 +268,16 @@ def test_datastore_admin_client_client_options( options = client_options.ClientOptions(quota_project_id="octopus") with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class(client_options=options) + client = client_class(transport=transport_name, client_options=options) patched.assert_called_once_with( credentials=None, credentials_file=None, host=client.DEFAULT_ENDPOINT, scopes=None, - ssl_channel_credentials=None, + client_cert_source_for_mtls=None, quota_project_id="octopus", client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, ) @@ -275,29 +326,26 @@ def test_datastore_admin_client_mtls_env_auto( client_cert_source=client_cert_source_callback ) with mock.patch.object(transport_class, "__init__") as patched: - ssl_channel_creds = mock.Mock() - with mock.patch( - "grpc.ssl_channel_credentials", return_value=ssl_channel_creds - ): - patched.return_value = None - client = client_class(client_options=options) + patched.return_value = None + client = client_class(transport=transport_name, client_options=options) - if use_client_cert_env == "false": - expected_ssl_channel_creds = None - expected_host = client.DEFAULT_ENDPOINT - else: - expected_ssl_channel_creds = ssl_channel_creds - expected_host = client.DEFAULT_MTLS_ENDPOINT + if use_client_cert_env == "false": + expected_client_cert_source = None + expected_host = client.DEFAULT_ENDPOINT + else: + expected_client_cert_source = client_cert_source_callback + expected_host = client.DEFAULT_MTLS_ENDPOINT - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=expected_host, - scopes=None, - ssl_channel_credentials=expected_ssl_channel_creds, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - ) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) # Check the case ADC client cert is provided. Whether client cert is used depends on # GOOGLE_API_USE_CLIENT_CERTIFICATE value. @@ -306,66 +354,55 @@ def test_datastore_admin_client_mtls_env_auto( ): with mock.patch.object(transport_class, "__init__") as patched: with mock.patch( - "google.auth.transport.grpc.SslCredentials.__init__", return_value=None + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=True, ): with mock.patch( - "google.auth.transport.grpc.SslCredentials.is_mtls", - new_callable=mock.PropertyMock, - ) as is_mtls_mock: - with mock.patch( - "google.auth.transport.grpc.SslCredentials.ssl_credentials", - new_callable=mock.PropertyMock, - ) as ssl_credentials_mock: - if use_client_cert_env == "false": - is_mtls_mock.return_value = False - ssl_credentials_mock.return_value = None - expected_host = client.DEFAULT_ENDPOINT - expected_ssl_channel_creds = None - else: - is_mtls_mock.return_value = True - ssl_credentials_mock.return_value = mock.Mock() - expected_host = client.DEFAULT_MTLS_ENDPOINT - expected_ssl_channel_creds = ( - ssl_credentials_mock.return_value - ) - - patched.return_value = None - client = client_class() - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=expected_host, - scopes=None, - ssl_channel_credentials=expected_ssl_channel_creds, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - ) + "google.auth.transport.mtls.default_client_cert_source", + return_value=client_cert_source_callback, + ): + if use_client_cert_env == "false": + expected_host = client.DEFAULT_ENDPOINT + expected_client_cert_source = None + else: + expected_host = client.DEFAULT_MTLS_ENDPOINT + expected_client_cert_source = client_cert_source_callback - # Check the case client_cert_source and ADC client cert are not provided. - with mock.patch.dict( - os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env} - ): - with mock.patch.object(transport_class, "__init__") as patched: - with mock.patch( - "google.auth.transport.grpc.SslCredentials.__init__", return_value=None - ): - with mock.patch( - "google.auth.transport.grpc.SslCredentials.is_mtls", - new_callable=mock.PropertyMock, - ) as is_mtls_mock: - is_mtls_mock.return_value = False patched.return_value = None - client = client_class() + client = client_class(transport=transport_name) patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=expected_host, scopes=None, - ssl_channel_credentials=None, + client_cert_source_for_mtls=expected_client_cert_source, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, ) + # Check the case client_cert_source and ADC client cert are not provided. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env} + ): + with mock.patch.object(transport_class, "__init__") as patched: + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=False, + ): + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + @pytest.mark.parametrize( "client_class,transport_class,transport_name", @@ -385,15 +422,16 @@ def test_datastore_admin_client_client_options_scopes( options = client_options.ClientOptions(scopes=["1", "2"],) with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class(client_options=options) + client = client_class(transport=transport_name, client_options=options) patched.assert_called_once_with( credentials=None, credentials_file=None, host=client.DEFAULT_ENDPOINT, scopes=["1", "2"], - ssl_channel_credentials=None, + client_cert_source_for_mtls=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, ) @@ -415,15 +453,16 @@ def test_datastore_admin_client_client_options_credentials_file( options = client_options.ClientOptions(credentials_file="credentials.json") with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class(client_options=options) + client = client_class(transport=transport_name, client_options=options) patched.assert_called_once_with( credentials=None, credentials_file="credentials.json", host=client.DEFAULT_ENDPOINT, scopes=None, - ssl_channel_credentials=None, + client_cert_source_for_mtls=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, ) @@ -440,9 +479,10 @@ def test_datastore_admin_client_client_options_from_dict(): credentials_file=None, host="squid.clam.whelk", scopes=None, - ssl_channel_credentials=None, + client_cert_source_for_mtls=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, ) @@ -450,7 +490,7 @@ def test_export_entities( transport: str = "grpc", request_type=datastore_admin.ExportEntitiesRequest ): client = DatastoreAdminClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -461,13 +501,11 @@ def test_export_entities( with mock.patch.object(type(client.transport.export_entities), "__call__") as call: # Designate an appropriate return value for the call. call.return_value = operations_pb2.Operation(name="operations/spam") - response = client.export_entities(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0] == datastore_admin.ExportEntitiesRequest() # Establish that the response is the type that we expect. @@ -478,12 +516,27 @@ def test_export_entities_from_dict(): test_export_entities(request_type=dict) +def test_export_entities_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = DatastoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.export_entities), "__call__") as call: + client.export_entities() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == datastore_admin.ExportEntitiesRequest() + + @pytest.mark.asyncio async def test_export_entities_async( transport: str = "grpc_asyncio", request_type=datastore_admin.ExportEntitiesRequest ): client = DatastoreAdminAsyncClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -496,13 +549,11 @@ async def test_export_entities_async( call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( operations_pb2.Operation(name="operations/spam") ) - response = await client.export_entities(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0] == datastore_admin.ExportEntitiesRequest() # Establish that the response is the type that we expect. @@ -515,13 +566,12 @@ async def test_export_entities_async_from_dict(): def test_export_entities_flattened(): - client = DatastoreAdminClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAdminClient(credentials=ga_credentials.AnonymousCredentials(),) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.export_entities), "__call__") as call: # Designate an appropriate return value for the call. call.return_value = operations_pb2.Operation(name="operations/op") - # Call the method with a truthy value for each flattened field, # using the keyword arguments to the method. client.export_entities( @@ -535,20 +585,16 @@ def test_export_entities_flattened(): # request object values. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0].project_id == "project_id_value" - assert args[0].labels == {"key_value": "value_value"} - assert args[0].entity_filter == datastore_admin.EntityFilter( kinds=["kinds_value"] ) - assert args[0].output_url_prefix == "output_url_prefix_value" def test_export_entities_flattened_error(): - client = DatastoreAdminClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAdminClient(credentials=ga_credentials.AnonymousCredentials(),) # Attempting to call a method with both a request object and flattened # fields is an error. @@ -564,7 +610,9 @@ def test_export_entities_flattened_error(): @pytest.mark.asyncio async def test_export_entities_flattened_async(): - client = DatastoreAdminAsyncClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAdminAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.export_entities), "__call__") as call: @@ -587,21 +635,19 @@ async def test_export_entities_flattened_async(): # request object values. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0].project_id == "project_id_value" - assert args[0].labels == {"key_value": "value_value"} - assert args[0].entity_filter == datastore_admin.EntityFilter( kinds=["kinds_value"] ) - assert args[0].output_url_prefix == "output_url_prefix_value" @pytest.mark.asyncio async def test_export_entities_flattened_error_async(): - client = DatastoreAdminAsyncClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAdminAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) # Attempting to call a method with both a request object and flattened # fields is an error. @@ -619,7 +665,7 @@ def test_import_entities( transport: str = "grpc", request_type=datastore_admin.ImportEntitiesRequest ): client = DatastoreAdminClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -630,13 +676,11 @@ def test_import_entities( with mock.patch.object(type(client.transport.import_entities), "__call__") as call: # Designate an appropriate return value for the call. call.return_value = operations_pb2.Operation(name="operations/spam") - response = client.import_entities(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0] == datastore_admin.ImportEntitiesRequest() # Establish that the response is the type that we expect. @@ -647,12 +691,27 @@ def test_import_entities_from_dict(): test_import_entities(request_type=dict) +def test_import_entities_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = DatastoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.import_entities), "__call__") as call: + client.import_entities() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == datastore_admin.ImportEntitiesRequest() + + @pytest.mark.asyncio async def test_import_entities_async( transport: str = "grpc_asyncio", request_type=datastore_admin.ImportEntitiesRequest ): client = DatastoreAdminAsyncClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -665,13 +724,11 @@ async def test_import_entities_async( call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( operations_pb2.Operation(name="operations/spam") ) - response = await client.import_entities(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0] == datastore_admin.ImportEntitiesRequest() # Establish that the response is the type that we expect. @@ -684,13 +741,12 @@ async def test_import_entities_async_from_dict(): def test_import_entities_flattened(): - client = DatastoreAdminClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAdminClient(credentials=ga_credentials.AnonymousCredentials(),) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.import_entities), "__call__") as call: # Designate an appropriate return value for the call. call.return_value = operations_pb2.Operation(name="operations/op") - # Call the method with a truthy value for each flattened field, # using the keyword arguments to the method. client.import_entities( @@ -704,20 +760,16 @@ def test_import_entities_flattened(): # request object values. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0].project_id == "project_id_value" - assert args[0].labels == {"key_value": "value_value"} - assert args[0].input_url == "input_url_value" - assert args[0].entity_filter == datastore_admin.EntityFilter( kinds=["kinds_value"] ) def test_import_entities_flattened_error(): - client = DatastoreAdminClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAdminClient(credentials=ga_credentials.AnonymousCredentials(),) # Attempting to call a method with both a request object and flattened # fields is an error. @@ -733,7 +785,9 @@ def test_import_entities_flattened_error(): @pytest.mark.asyncio async def test_import_entities_flattened_async(): - client = DatastoreAdminAsyncClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAdminAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.import_entities), "__call__") as call: @@ -756,13 +810,9 @@ async def test_import_entities_flattened_async(): # request object values. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0].project_id == "project_id_value" - assert args[0].labels == {"key_value": "value_value"} - assert args[0].input_url == "input_url_value" - assert args[0].entity_filter == datastore_admin.EntityFilter( kinds=["kinds_value"] ) @@ -770,7 +820,9 @@ async def test_import_entities_flattened_async(): @pytest.mark.asyncio async def test_import_entities_flattened_error_async(): - client = DatastoreAdminAsyncClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAdminAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), + ) # Attempting to call a method with both a request object and flattened # fields is an error. @@ -784,11 +836,169 @@ async def test_import_entities_flattened_error_async(): ) +def test_create_index( + transport: str = "grpc", request_type=datastore_admin.CreateIndexRequest +): + client = DatastoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.create_index), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = operations_pb2.Operation(name="operations/spam") + response = client.create_index(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == datastore_admin.CreateIndexRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, future.Future) + + +def test_create_index_from_dict(): + test_create_index(request_type=dict) + + +def test_create_index_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = DatastoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.create_index), "__call__") as call: + client.create_index() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == datastore_admin.CreateIndexRequest() + + +@pytest.mark.asyncio +async def test_create_index_async( + transport: str = "grpc_asyncio", request_type=datastore_admin.CreateIndexRequest +): + client = DatastoreAdminAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.create_index), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + operations_pb2.Operation(name="operations/spam") + ) + response = await client.create_index(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == datastore_admin.CreateIndexRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, future.Future) + + +@pytest.mark.asyncio +async def test_create_index_async_from_dict(): + await test_create_index_async(request_type=dict) + + +def test_delete_index( + transport: str = "grpc", request_type=datastore_admin.DeleteIndexRequest +): + client = DatastoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.delete_index), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = operations_pb2.Operation(name="operations/spam") + response = client.delete_index(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == datastore_admin.DeleteIndexRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, future.Future) + + +def test_delete_index_from_dict(): + test_delete_index(request_type=dict) + + +def test_delete_index_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = DatastoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.delete_index), "__call__") as call: + client.delete_index() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == datastore_admin.DeleteIndexRequest() + + +@pytest.mark.asyncio +async def test_delete_index_async( + transport: str = "grpc_asyncio", request_type=datastore_admin.DeleteIndexRequest +): + client = DatastoreAdminAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.delete_index), "__call__") as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( + operations_pb2.Operation(name="operations/spam") + ) + response = await client.delete_index(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == datastore_admin.DeleteIndexRequest() + + # Establish that the response is the type that we expect. + assert isinstance(response, future.Future) + + +@pytest.mark.asyncio +async def test_delete_index_async_from_dict(): + await test_delete_index_async(request_type=dict) + + def test_get_index( transport: str = "grpc", request_type=datastore_admin.GetIndexRequest ): client = DatastoreAdminClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -805,27 +1015,19 @@ def test_get_index( ancestor=index.Index.AncestorMode.NONE, state=index.Index.State.CREATING, ) - response = client.get_index(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0] == datastore_admin.GetIndexRequest() # Establish that the response is the type that we expect. - assert isinstance(response, index.Index) - assert response.project_id == "project_id_value" - assert response.index_id == "index_id_value" - assert response.kind == "kind_value" - assert response.ancestor == index.Index.AncestorMode.NONE - assert response.state == index.Index.State.CREATING @@ -833,12 +1035,27 @@ def test_get_index_from_dict(): test_get_index(request_type=dict) +def test_get_index_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = DatastoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.get_index), "__call__") as call: + client.get_index() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == datastore_admin.GetIndexRequest() + + @pytest.mark.asyncio async def test_get_index_async( transport: str = "grpc_asyncio", request_type=datastore_admin.GetIndexRequest ): client = DatastoreAdminAsyncClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -857,26 +1074,19 @@ async def test_get_index_async( state=index.Index.State.CREATING, ) ) - response = await client.get_index(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0] == datastore_admin.GetIndexRequest() # Establish that the response is the type that we expect. assert isinstance(response, index.Index) - assert response.project_id == "project_id_value" - assert response.index_id == "index_id_value" - assert response.kind == "kind_value" - assert response.ancestor == index.Index.AncestorMode.NONE - assert response.state == index.Index.State.CREATING @@ -889,7 +1099,7 @@ def test_list_indexes( transport: str = "grpc", request_type=datastore_admin.ListIndexesRequest ): client = DatastoreAdminClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -902,19 +1112,15 @@ def test_list_indexes( call.return_value = datastore_admin.ListIndexesResponse( next_page_token="next_page_token_value", ) - response = client.list_indexes(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0] == datastore_admin.ListIndexesRequest() # Establish that the response is the type that we expect. - assert isinstance(response, pagers.ListIndexesPager) - assert response.next_page_token == "next_page_token_value" @@ -922,12 +1128,27 @@ def test_list_indexes_from_dict(): test_list_indexes(request_type=dict) +def test_list_indexes_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = DatastoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.list_indexes), "__call__") as call: + client.list_indexes() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == datastore_admin.ListIndexesRequest() + + @pytest.mark.asyncio async def test_list_indexes_async( transport: str = "grpc_asyncio", request_type=datastore_admin.ListIndexesRequest ): client = DatastoreAdminAsyncClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -942,18 +1163,15 @@ async def test_list_indexes_async( next_page_token="next_page_token_value", ) ) - response = await client.list_indexes(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0] == datastore_admin.ListIndexesRequest() # Establish that the response is the type that we expect. assert isinstance(response, pagers.ListIndexesAsyncPager) - assert response.next_page_token == "next_page_token_value" @@ -963,7 +1181,7 @@ async def test_list_indexes_async_from_dict(): def test_list_indexes_pager(): - client = DatastoreAdminClient(credentials=credentials.AnonymousCredentials,) + client = DatastoreAdminClient(credentials=ga_credentials.AnonymousCredentials,) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.list_indexes), "__call__") as call: @@ -994,7 +1212,7 @@ def test_list_indexes_pager(): def test_list_indexes_pages(): - client = DatastoreAdminClient(credentials=credentials.AnonymousCredentials,) + client = DatastoreAdminClient(credentials=ga_credentials.AnonymousCredentials,) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.list_indexes), "__call__") as call: @@ -1020,7 +1238,7 @@ def test_list_indexes_pages(): @pytest.mark.asyncio async def test_list_indexes_async_pager(): - client = DatastoreAdminAsyncClient(credentials=credentials.AnonymousCredentials,) + client = DatastoreAdminAsyncClient(credentials=ga_credentials.AnonymousCredentials,) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object( @@ -1053,7 +1271,7 @@ async def test_list_indexes_async_pager(): @pytest.mark.asyncio async def test_list_indexes_async_pages(): - client = DatastoreAdminAsyncClient(credentials=credentials.AnonymousCredentials,) + client = DatastoreAdminAsyncClient(credentials=ga_credentials.AnonymousCredentials,) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object( @@ -1084,16 +1302,16 @@ async def test_list_indexes_async_pages(): def test_credentials_transport_error(): # It is an error to provide credentials and a transport instance. transport = transports.DatastoreAdminGrpcTransport( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), ) with pytest.raises(ValueError): client = DatastoreAdminClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # It is an error to provide a credentials file and a transport instance. transport = transports.DatastoreAdminGrpcTransport( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), ) with pytest.raises(ValueError): client = DatastoreAdminClient( @@ -1103,7 +1321,7 @@ def test_credentials_transport_error(): # It is an error to provide scopes and a transport instance. transport = transports.DatastoreAdminGrpcTransport( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), ) with pytest.raises(ValueError): client = DatastoreAdminClient( @@ -1114,7 +1332,7 @@ def test_credentials_transport_error(): def test_transport_instance(): # A client may be instantiated with a custom transport instance. transport = transports.DatastoreAdminGrpcTransport( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), ) client = DatastoreAdminClient(transport=transport) assert client.transport is transport @@ -1123,13 +1341,13 @@ def test_transport_instance(): def test_transport_get_channel(): # A client may be instantiated with a custom transport instance. transport = transports.DatastoreAdminGrpcTransport( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), ) channel = transport.grpc_channel assert channel transport = transports.DatastoreAdminGrpcAsyncIOTransport( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), ) channel = transport.grpc_channel assert channel @@ -1144,23 +1362,23 @@ def test_transport_get_channel(): ) def test_transport_adc(transport_class): # Test default credentials are used if not provided. - with mock.patch.object(auth, "default") as adc: - adc.return_value = (credentials.AnonymousCredentials(), None) + with mock.patch.object(google.auth, "default") as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) transport_class() adc.assert_called_once() def test_transport_grpc_default(): # A client should use the gRPC transport by default. - client = DatastoreAdminClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAdminClient(credentials=ga_credentials.AnonymousCredentials(),) assert isinstance(client.transport, transports.DatastoreAdminGrpcTransport,) def test_datastore_admin_base_transport_error(): # Passing both a credentials object and credentials_file should raise an error - with pytest.raises(exceptions.DuplicateCredentialArgs): + with pytest.raises(core_exceptions.DuplicateCredentialArgs): transport = transports.DatastoreAdminTransport( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), credentials_file="credentials.json", ) @@ -1172,7 +1390,7 @@ def test_datastore_admin_base_transport(): ) as Transport: Transport.return_value = None transport = transports.DatastoreAdminTransport( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), ) # Every method on the transport should just blindly @@ -1180,6 +1398,8 @@ def test_datastore_admin_base_transport(): methods = ( "export_entities", "import_entities", + "create_index", + "delete_index", "get_index", "list_indexes", ) @@ -1187,6 +1407,9 @@ def test_datastore_admin_base_transport(): with pytest.raises(NotImplementedError): getattr(transport, method)(request=object()) + with pytest.raises(NotImplementedError): + transport.close() + # Additionally, the LRO client (a property) should # also raise NotImplementedError with pytest.raises(NotImplementedError): @@ -1196,18 +1419,19 @@ def test_datastore_admin_base_transport(): def test_datastore_admin_base_transport_with_credentials_file(): # Instantiate the base transport with a credentials file with mock.patch.object( - auth, "load_credentials_from_file" + google.auth, "load_credentials_from_file", autospec=True ) as load_creds, mock.patch( "google.cloud.datastore_admin_v1.services.datastore_admin.transports.DatastoreAdminTransport._prep_wrapped_messages" ) as Transport: Transport.return_value = None - load_creds.return_value = (credentials.AnonymousCredentials(), None) + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) transport = transports.DatastoreAdminTransport( credentials_file="credentials.json", quota_project_id="octopus", ) load_creds.assert_called_once_with( "credentials.json", - scopes=( + scopes=None, + default_scopes=( "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/datastore", ), @@ -1217,22 +1441,23 @@ def test_datastore_admin_base_transport_with_credentials_file(): def test_datastore_admin_base_transport_with_adc(): # Test the default credentials are used if credentials and credentials_file are None. - with mock.patch.object(auth, "default") as adc, mock.patch( + with mock.patch.object(google.auth, "default", autospec=True) as adc, mock.patch( "google.cloud.datastore_admin_v1.services.datastore_admin.transports.DatastoreAdminTransport._prep_wrapped_messages" ) as Transport: Transport.return_value = None - adc.return_value = (credentials.AnonymousCredentials(), None) + adc.return_value = (ga_credentials.AnonymousCredentials(), None) transport = transports.DatastoreAdminTransport() adc.assert_called_once() def test_datastore_admin_auth_adc(): # If no credentials are provided, we should use ADC credentials. - with mock.patch.object(auth, "default") as adc: - adc.return_value = (credentials.AnonymousCredentials(), None) + with mock.patch.object(google.auth, "default", autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) DatastoreAdminClient() adc.assert_called_once_with( - scopes=( + scopes=None, + default_scopes=( "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/datastore", ), @@ -1240,16 +1465,22 @@ def test_datastore_admin_auth_adc(): ) -def test_datastore_admin_transport_auth_adc(): +@pytest.mark.parametrize( + "transport_class", + [ + transports.DatastoreAdminGrpcTransport, + transports.DatastoreAdminGrpcAsyncIOTransport, + ], +) +def test_datastore_admin_transport_auth_adc(transport_class): # If credentials and host are not provided, the transport class should use # ADC credentials. - with mock.patch.object(auth, "default") as adc: - adc.return_value = (credentials.AnonymousCredentials(), None) - transports.DatastoreAdminGrpcTransport( - host="squid.clam.whelk", quota_project_id="octopus" - ) + with mock.patch.object(google.auth, "default", autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class(quota_project_id="octopus", scopes=["1", "2"]) adc.assert_called_once_with( - scopes=( + scopes=["1", "2"], + default_scopes=( "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/datastore", ), @@ -1257,9 +1488,92 @@ def test_datastore_admin_transport_auth_adc(): ) +@pytest.mark.parametrize( + "transport_class,grpc_helpers", + [ + (transports.DatastoreAdminGrpcTransport, grpc_helpers), + (transports.DatastoreAdminGrpcAsyncIOTransport, grpc_helpers_async), + ], +) +def test_datastore_admin_transport_create_channel(transport_class, grpc_helpers): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object( + google.auth, "default", autospec=True + ) as adc, mock.patch.object( + grpc_helpers, "create_channel", autospec=True + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + adc.return_value = (creds, None) + transport_class(quota_project_id="octopus", scopes=["1", "2"]) + + create_channel.assert_called_with( + "datastore.googleapis.com:443", + credentials=creds, + credentials_file=None, + quota_project_id="octopus", + default_scopes=( + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/datastore", + ), + scopes=["1", "2"], + default_host="datastore.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.DatastoreAdminGrpcTransport, + transports.DatastoreAdminGrpcAsyncIOTransport, + ], +) +def test_datastore_admin_grpc_transport_client_cert_source_for_mtls(transport_class): + cred = ga_credentials.AnonymousCredentials() + + # Check ssl_channel_credentials is used if provided. + with mock.patch.object(transport_class, "create_channel") as mock_create_channel: + mock_ssl_channel_creds = mock.Mock() + transport_class( + host="squid.clam.whelk", + credentials=cred, + ssl_channel_credentials=mock_ssl_channel_creds, + ) + mock_create_channel.assert_called_once_with( + "squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_channel_creds, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Check if ssl_channel_credentials is not provided, then client_cert_source_for_mtls + # is used. + with mock.patch.object(transport_class, "create_channel", return_value=mock.Mock()): + with mock.patch("grpc.ssl_channel_credentials") as mock_ssl_cred: + transport_class( + credentials=cred, + client_cert_source_for_mtls=client_cert_source_callback, + ) + expected_cert, expected_key = client_cert_source_callback() + mock_ssl_cred.assert_called_once_with( + certificate_chain=expected_cert, private_key=expected_key + ) + + def test_datastore_admin_host_no_port(): client = DatastoreAdminClient( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), client_options=client_options.ClientOptions( api_endpoint="datastore.googleapis.com" ), @@ -1269,7 +1583,7 @@ def test_datastore_admin_host_no_port(): def test_datastore_admin_host_with_port(): client = DatastoreAdminClient( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), client_options=client_options.ClientOptions( api_endpoint="datastore.googleapis.com:8000" ), @@ -1278,7 +1592,7 @@ def test_datastore_admin_host_with_port(): def test_datastore_admin_grpc_transport_channel(): - channel = grpc.insecure_channel("http://localhost/") + channel = grpc.secure_channel("http://localhost/", grpc.local_channel_credentials()) # Check that channel is used if provided. transport = transports.DatastoreAdminGrpcTransport( @@ -1290,7 +1604,7 @@ def test_datastore_admin_grpc_transport_channel(): def test_datastore_admin_grpc_asyncio_transport_channel(): - channel = aio.insecure_channel("http://localhost/") + channel = aio.secure_channel("http://localhost/", grpc.local_channel_credentials()) # Check that channel is used if provided. transport = transports.DatastoreAdminGrpcAsyncIOTransport( @@ -1301,6 +1615,8 @@ def test_datastore_admin_grpc_asyncio_transport_channel(): assert transport._ssl_channel_credentials == None +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. @pytest.mark.parametrize( "transport_class", [ @@ -1315,7 +1631,7 @@ def test_datastore_admin_transport_channel_mtls_with_client_cert_source( "grpc.ssl_channel_credentials", autospec=True ) as grpc_ssl_channel_cred: with mock.patch.object( - transport_class, "create_channel", autospec=True + transport_class, "create_channel" ) as grpc_create_channel: mock_ssl_cred = mock.Mock() grpc_ssl_channel_cred.return_value = mock_ssl_cred @@ -1323,9 +1639,9 @@ def test_datastore_admin_transport_channel_mtls_with_client_cert_source( mock_grpc_channel = mock.Mock() grpc_create_channel.return_value = mock_grpc_channel - cred = credentials.AnonymousCredentials() + cred = ga_credentials.AnonymousCredentials() with pytest.warns(DeprecationWarning): - with mock.patch.object(auth, "default") as adc: + with mock.patch.object(google.auth, "default") as adc: adc.return_value = (cred, None) transport = transport_class( host="squid.clam.whelk", @@ -1341,17 +1657,20 @@ def test_datastore_admin_transport_channel_mtls_with_client_cert_source( "mtls.squid.clam.whelk:443", credentials=cred, credentials_file=None, - scopes=( - "https://www.googleapis.com/auth/cloud-platform", - "https://www.googleapis.com/auth/datastore", - ), + scopes=None, ssl_credentials=mock_ssl_cred, quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) assert transport.grpc_channel == mock_grpc_channel assert transport._ssl_channel_credentials == mock_ssl_cred +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. @pytest.mark.parametrize( "transport_class", [ @@ -1367,7 +1686,7 @@ def test_datastore_admin_transport_channel_mtls_with_adc(transport_class): ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred), ): with mock.patch.object( - transport_class, "create_channel", autospec=True + transport_class, "create_channel" ) as grpc_create_channel: mock_grpc_channel = mock.Mock() grpc_create_channel.return_value = mock_grpc_channel @@ -1385,19 +1704,20 @@ def test_datastore_admin_transport_channel_mtls_with_adc(transport_class): "mtls.squid.clam.whelk:443", credentials=mock_cred, credentials_file=None, - scopes=( - "https://www.googleapis.com/auth/cloud-platform", - "https://www.googleapis.com/auth/datastore", - ), + scopes=None, ssl_credentials=mock_ssl_cred, quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) assert transport.grpc_channel == mock_grpc_channel def test_datastore_admin_grpc_lro_client(): client = DatastoreAdminClient( - credentials=credentials.AnonymousCredentials(), transport="grpc", + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", ) transport = client.transport @@ -1410,7 +1730,7 @@ def test_datastore_admin_grpc_lro_client(): def test_datastore_admin_grpc_lro_async_client(): client = DatastoreAdminAsyncClient( - credentials=credentials.AnonymousCredentials(), transport="grpc_asyncio", + credentials=ga_credentials.AnonymousCredentials(), transport="grpc_asyncio", ) transport = client.transport @@ -1423,7 +1743,6 @@ def test_datastore_admin_grpc_lro_async_client(): def test_common_billing_account_path(): billing_account = "squid" - expected = "billingAccounts/{billing_account}".format( billing_account=billing_account, ) @@ -1444,7 +1763,6 @@ def test_parse_common_billing_account_path(): def test_common_folder_path(): folder = "whelk" - expected = "folders/{folder}".format(folder=folder,) actual = DatastoreAdminClient.common_folder_path(folder) assert expected == actual @@ -1463,7 +1781,6 @@ def test_parse_common_folder_path(): def test_common_organization_path(): organization = "oyster" - expected = "organizations/{organization}".format(organization=organization,) actual = DatastoreAdminClient.common_organization_path(organization) assert expected == actual @@ -1482,7 +1799,6 @@ def test_parse_common_organization_path(): def test_common_project_path(): project = "cuttlefish" - expected = "projects/{project}".format(project=project,) actual = DatastoreAdminClient.common_project_path(project) assert expected == actual @@ -1502,7 +1818,6 @@ def test_parse_common_project_path(): def test_common_location_path(): project = "winkle" location = "nautilus" - expected = "projects/{project}/locations/{location}".format( project=project, location=location, ) @@ -1529,7 +1844,7 @@ def test_client_withDEFAULT_CLIENT_INFO(): transports.DatastoreAdminTransport, "_prep_wrapped_messages" ) as prep: client = DatastoreAdminClient( - credentials=credentials.AnonymousCredentials(), client_info=client_info, + credentials=ga_credentials.AnonymousCredentials(), client_info=client_info, ) prep.assert_called_once_with(client_info) @@ -1538,6 +1853,52 @@ def test_client_withDEFAULT_CLIENT_INFO(): ) as prep: transport_class = DatastoreAdminClient.get_transport_class() transport = transport_class( - credentials=credentials.AnonymousCredentials(), client_info=client_info, + credentials=ga_credentials.AnonymousCredentials(), client_info=client_info, ) prep.assert_called_once_with(client_info) + + +@pytest.mark.asyncio +async def test_transport_close_async(): + client = DatastoreAdminAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc_asyncio", + ) + with mock.patch.object( + type(getattr(client.transport, "grpc_channel")), "close" + ) as close: + async with client: + close.assert_not_called() + close.assert_called_once() + + +def test_transport_close(): + transports = { + "grpc": "_grpc_channel", + } + + for transport, close_name in transports.items(): + client = DatastoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport + ) + with mock.patch.object( + type(getattr(client.transport, close_name)), "close" + ) as close: + with client: + close.assert_not_called() + close.assert_called_once() + + +def test_client_ctx(): + transports = [ + "grpc", + ] + for transport in transports: + client = DatastoreAdminClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport + ) + # Test client calls underlying transport. + with mock.patch.object(type(client.transport), "close") as close: + close.assert_not_called() + with client: + pass + close.assert_called() diff --git a/tests/unit/gapic/datastore_v1/__init__.py b/tests/unit/gapic/datastore_v1/__init__.py index 8b137891..4de65971 100644 --- a/tests/unit/gapic/datastore_v1/__init__.py +++ b/tests/unit/gapic/datastore_v1/__init__.py @@ -1 +1,15 @@ - +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/tests/unit/gapic/datastore_v1/test_datastore.py b/tests/unit/gapic/datastore_v1/test_datastore.py index 32faab36..04ced96f 100644 --- a/tests/unit/gapic/datastore_v1/test_datastore.py +++ b/tests/unit/gapic/datastore_v1/test_datastore.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - # Copyright 2020 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # - import os import mock @@ -24,13 +22,14 @@ import pytest from proto.marshal.rules.dates import DurationRule, TimestampRule -from google import auth + from google.api_core import client_options -from google.api_core import exceptions +from google.api_core import exceptions as core_exceptions from google.api_core import gapic_v1 from google.api_core import grpc_helpers from google.api_core import grpc_helpers_async -from google.auth import credentials +from google.api_core import path_template +from google.auth import credentials as ga_credentials from google.auth.exceptions import MutualTLSChannelError from google.cloud.datastore_v1.services.datastore import DatastoreAsyncClient from google.cloud.datastore_v1.services.datastore import DatastoreClient @@ -39,10 +38,11 @@ from google.cloud.datastore_v1.types import entity from google.cloud.datastore_v1.types import query from google.oauth2 import service_account -from google.protobuf import struct_pb2 as struct # type: ignore -from google.protobuf import timestamp_pb2 as timestamp # type: ignore -from google.protobuf import wrappers_pb2 as wrappers # type: ignore -from google.type import latlng_pb2 as latlng # type: ignore +from google.protobuf import struct_pb2 # type: ignore +from google.protobuf import timestamp_pb2 # type: ignore +from google.protobuf import wrappers_pb2 # type: ignore +from google.type import latlng_pb2 # type: ignore +import google.auth def client_cert_source_callback(): @@ -84,25 +84,70 @@ def test__get_default_mtls_endpoint(): assert DatastoreClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi -@pytest.mark.parametrize("client_class", [DatastoreClient, DatastoreAsyncClient]) +@pytest.mark.parametrize("client_class", [DatastoreClient, DatastoreAsyncClient,]) +def test_datastore_client_from_service_account_info(client_class): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object( + service_account.Credentials, "from_service_account_info" + ) as factory: + factory.return_value = creds + info = {"valid": True} + client = client_class.from_service_account_info(info) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == "datastore.googleapis.com:443" + + +@pytest.mark.parametrize( + "transport_class,transport_name", + [ + (transports.DatastoreGrpcTransport, "grpc"), + (transports.DatastoreGrpcAsyncIOTransport, "grpc_asyncio"), + ], +) +def test_datastore_client_service_account_always_use_jwt( + transport_class, transport_name +): + with mock.patch.object( + service_account.Credentials, "with_always_use_jwt_access", create=True + ) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=True) + use_jwt.assert_called_once_with(True) + + with mock.patch.object( + service_account.Credentials, "with_always_use_jwt_access", create=True + ) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=False) + use_jwt.assert_not_called() + + +@pytest.mark.parametrize("client_class", [DatastoreClient, DatastoreAsyncClient,]) def test_datastore_client_from_service_account_file(client_class): - creds = credentials.AnonymousCredentials() + creds = ga_credentials.AnonymousCredentials() with mock.patch.object( service_account.Credentials, "from_service_account_file" ) as factory: factory.return_value = creds client = client_class.from_service_account_file("dummy/file/path.json") assert client.transport._credentials == creds + assert isinstance(client, client_class) client = client_class.from_service_account_json("dummy/file/path.json") assert client.transport._credentials == creds + assert isinstance(client, client_class) assert client.transport._host == "datastore.googleapis.com:443" def test_datastore_client_get_transport_class(): transport = DatastoreClient.get_transport_class() - assert transport == transports.DatastoreGrpcTransport + available_transports = [ + transports.DatastoreGrpcTransport, + ] + assert transport in available_transports transport = DatastoreClient.get_transport_class("grpc") assert transport == transports.DatastoreGrpcTransport @@ -130,7 +175,7 @@ def test_datastore_client_get_transport_class(): def test_datastore_client_client_options(client_class, transport_class, transport_name): # Check that if channel is provided we won't create a new one. with mock.patch.object(DatastoreClient, "get_transport_class") as gtc: - transport = transport_class(credentials=credentials.AnonymousCredentials()) + transport = transport_class(credentials=ga_credentials.AnonymousCredentials()) client = client_class(transport=transport) gtc.assert_not_called() @@ -143,15 +188,16 @@ def test_datastore_client_client_options(client_class, transport_class, transpor options = client_options.ClientOptions(api_endpoint="squid.clam.whelk") with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class(client_options=options) + client = client_class(transport=transport_name, client_options=options) patched.assert_called_once_with( credentials=None, credentials_file=None, host="squid.clam.whelk", scopes=None, - ssl_channel_credentials=None, + client_cert_source_for_mtls=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, ) # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is @@ -159,15 +205,16 @@ def test_datastore_client_client_options(client_class, transport_class, transpor with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class() + client = client_class(transport=transport_name) patched.assert_called_once_with( credentials=None, credentials_file=None, host=client.DEFAULT_ENDPOINT, scopes=None, - ssl_channel_credentials=None, + client_cert_source_for_mtls=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, ) # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is @@ -175,15 +222,16 @@ def test_datastore_client_client_options(client_class, transport_class, transpor with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class() + client = client_class(transport=transport_name) patched.assert_called_once_with( credentials=None, credentials_file=None, host=client.DEFAULT_MTLS_ENDPOINT, scopes=None, - ssl_channel_credentials=None, + client_cert_source_for_mtls=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, ) # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has @@ -203,15 +251,16 @@ def test_datastore_client_client_options(client_class, transport_class, transpor options = client_options.ClientOptions(quota_project_id="octopus") with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class(client_options=options) + client = client_class(transport=transport_name, client_options=options) patched.assert_called_once_with( credentials=None, credentials_file=None, host=client.DEFAULT_ENDPOINT, scopes=None, - ssl_channel_credentials=None, + client_cert_source_for_mtls=None, quota_project_id="octopus", client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, ) @@ -258,29 +307,26 @@ def test_datastore_client_mtls_env_auto( client_cert_source=client_cert_source_callback ) with mock.patch.object(transport_class, "__init__") as patched: - ssl_channel_creds = mock.Mock() - with mock.patch( - "grpc.ssl_channel_credentials", return_value=ssl_channel_creds - ): - patched.return_value = None - client = client_class(client_options=options) + patched.return_value = None + client = client_class(transport=transport_name, client_options=options) - if use_client_cert_env == "false": - expected_ssl_channel_creds = None - expected_host = client.DEFAULT_ENDPOINT - else: - expected_ssl_channel_creds = ssl_channel_creds - expected_host = client.DEFAULT_MTLS_ENDPOINT + if use_client_cert_env == "false": + expected_client_cert_source = None + expected_host = client.DEFAULT_ENDPOINT + else: + expected_client_cert_source = client_cert_source_callback + expected_host = client.DEFAULT_MTLS_ENDPOINT - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=expected_host, - scopes=None, - ssl_channel_credentials=expected_ssl_channel_creds, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - ) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) # Check the case ADC client cert is provided. Whether client cert is used depends on # GOOGLE_API_USE_CLIENT_CERTIFICATE value. @@ -289,66 +335,55 @@ def test_datastore_client_mtls_env_auto( ): with mock.patch.object(transport_class, "__init__") as patched: with mock.patch( - "google.auth.transport.grpc.SslCredentials.__init__", return_value=None + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=True, ): with mock.patch( - "google.auth.transport.grpc.SslCredentials.is_mtls", - new_callable=mock.PropertyMock, - ) as is_mtls_mock: - with mock.patch( - "google.auth.transport.grpc.SslCredentials.ssl_credentials", - new_callable=mock.PropertyMock, - ) as ssl_credentials_mock: - if use_client_cert_env == "false": - is_mtls_mock.return_value = False - ssl_credentials_mock.return_value = None - expected_host = client.DEFAULT_ENDPOINT - expected_ssl_channel_creds = None - else: - is_mtls_mock.return_value = True - ssl_credentials_mock.return_value = mock.Mock() - expected_host = client.DEFAULT_MTLS_ENDPOINT - expected_ssl_channel_creds = ( - ssl_credentials_mock.return_value - ) + "google.auth.transport.mtls.default_client_cert_source", + return_value=client_cert_source_callback, + ): + if use_client_cert_env == "false": + expected_host = client.DEFAULT_ENDPOINT + expected_client_cert_source = None + else: + expected_host = client.DEFAULT_MTLS_ENDPOINT + expected_client_cert_source = client_cert_source_callback - patched.return_value = None - client = client_class() - patched.assert_called_once_with( - credentials=None, - credentials_file=None, - host=expected_host, - scopes=None, - ssl_channel_credentials=expected_ssl_channel_creds, - quota_project_id=None, - client_info=transports.base.DEFAULT_CLIENT_INFO, - ) - - # Check the case client_cert_source and ADC client cert are not provided. - with mock.patch.dict( - os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env} - ): - with mock.patch.object(transport_class, "__init__") as patched: - with mock.patch( - "google.auth.transport.grpc.SslCredentials.__init__", return_value=None - ): - with mock.patch( - "google.auth.transport.grpc.SslCredentials.is_mtls", - new_callable=mock.PropertyMock, - ) as is_mtls_mock: - is_mtls_mock.return_value = False patched.return_value = None - client = client_class() + client = client_class(transport=transport_name) patched.assert_called_once_with( credentials=None, credentials_file=None, - host=client.DEFAULT_ENDPOINT, + host=expected_host, scopes=None, - ssl_channel_credentials=None, + client_cert_source_for_mtls=expected_client_cert_source, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, ) + # Check the case client_cert_source and ADC client cert are not provided. + with mock.patch.dict( + os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env} + ): + with mock.patch.object(transport_class, "__init__") as patched: + with mock.patch( + "google.auth.transport.mtls.has_default_client_cert_source", + return_value=False, + ): + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + @pytest.mark.parametrize( "client_class,transport_class,transport_name", @@ -368,15 +403,16 @@ def test_datastore_client_client_options_scopes( options = client_options.ClientOptions(scopes=["1", "2"],) with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class(client_options=options) + client = client_class(transport=transport_name, client_options=options) patched.assert_called_once_with( credentials=None, credentials_file=None, host=client.DEFAULT_ENDPOINT, scopes=["1", "2"], - ssl_channel_credentials=None, + client_cert_source_for_mtls=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, ) @@ -398,15 +434,16 @@ def test_datastore_client_client_options_credentials_file( options = client_options.ClientOptions(credentials_file="credentials.json") with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None - client = client_class(client_options=options) + client = client_class(transport=transport_name, client_options=options) patched.assert_called_once_with( credentials=None, credentials_file="credentials.json", host=client.DEFAULT_ENDPOINT, scopes=None, - ssl_channel_credentials=None, + client_cert_source_for_mtls=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, ) @@ -421,15 +458,16 @@ def test_datastore_client_client_options_from_dict(): credentials_file=None, host="squid.clam.whelk", scopes=None, - ssl_channel_credentials=None, + client_cert_source_for_mtls=None, quota_project_id=None, client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, ) def test_lookup(transport: str = "grpc", request_type=datastore.LookupRequest): client = DatastoreClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -440,17 +478,14 @@ def test_lookup(transport: str = "grpc", request_type=datastore.LookupRequest): with mock.patch.object(type(client.transport.lookup), "__call__") as call: # Designate an appropriate return value for the call. call.return_value = datastore.LookupResponse() - response = client.lookup(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0] == datastore.LookupRequest() # Establish that the response is the type that we expect. - assert isinstance(response, datastore.LookupResponse) @@ -458,12 +493,27 @@ def test_lookup_from_dict(): test_lookup(request_type=dict) +def test_lookup_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = DatastoreClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.lookup), "__call__") as call: + client.lookup() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == datastore.LookupRequest() + + @pytest.mark.asyncio async def test_lookup_async( transport: str = "grpc_asyncio", request_type=datastore.LookupRequest ): client = DatastoreAsyncClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -476,13 +526,11 @@ async def test_lookup_async( call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( datastore.LookupResponse() ) - response = await client.lookup(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0] == datastore.LookupRequest() # Establish that the response is the type that we expect. @@ -495,13 +543,12 @@ async def test_lookup_async_from_dict(): def test_lookup_flattened(): - client = DatastoreClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreClient(credentials=ga_credentials.AnonymousCredentials(),) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.lookup), "__call__") as call: # Designate an appropriate return value for the call. call.return_value = datastore.LookupResponse() - # Call the method with a truthy value for each flattened field, # using the keyword arguments to the method. client.lookup( @@ -520,20 +567,17 @@ def test_lookup_flattened(): # request object values. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0].project_id == "project_id_value" - assert args[0].read_options == datastore.ReadOptions( read_consistency=datastore.ReadOptions.ReadConsistency.STRONG ) - assert args[0].keys == [ entity.Key(partition_id=entity.PartitionId(project_id="project_id_value")) ] def test_lookup_flattened_error(): - client = DatastoreClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreClient(credentials=ga_credentials.AnonymousCredentials(),) # Attempting to call a method with both a request object and flattened # fields is an error. @@ -554,7 +598,7 @@ def test_lookup_flattened_error(): @pytest.mark.asyncio async def test_lookup_flattened_async(): - client = DatastoreAsyncClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAsyncClient(credentials=ga_credentials.AnonymousCredentials(),) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.lookup), "__call__") as call: @@ -582,13 +626,10 @@ async def test_lookup_flattened_async(): # request object values. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0].project_id == "project_id_value" - assert args[0].read_options == datastore.ReadOptions( read_consistency=datastore.ReadOptions.ReadConsistency.STRONG ) - assert args[0].keys == [ entity.Key(partition_id=entity.PartitionId(project_id="project_id_value")) ] @@ -596,7 +637,7 @@ async def test_lookup_flattened_async(): @pytest.mark.asyncio async def test_lookup_flattened_error_async(): - client = DatastoreAsyncClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAsyncClient(credentials=ga_credentials.AnonymousCredentials(),) # Attempting to call a method with both a request object and flattened # fields is an error. @@ -617,7 +658,7 @@ async def test_lookup_flattened_error_async(): def test_run_query(transport: str = "grpc", request_type=datastore.RunQueryRequest): client = DatastoreClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -628,17 +669,14 @@ def test_run_query(transport: str = "grpc", request_type=datastore.RunQueryReque with mock.patch.object(type(client.transport.run_query), "__call__") as call: # Designate an appropriate return value for the call. call.return_value = datastore.RunQueryResponse() - response = client.run_query(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0] == datastore.RunQueryRequest() # Establish that the response is the type that we expect. - assert isinstance(response, datastore.RunQueryResponse) @@ -646,12 +684,27 @@ def test_run_query_from_dict(): test_run_query(request_type=dict) +def test_run_query_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = DatastoreClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.run_query), "__call__") as call: + client.run_query() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == datastore.RunQueryRequest() + + @pytest.mark.asyncio async def test_run_query_async( transport: str = "grpc_asyncio", request_type=datastore.RunQueryRequest ): client = DatastoreAsyncClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -664,13 +717,11 @@ async def test_run_query_async( call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( datastore.RunQueryResponse() ) - response = await client.run_query(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0] == datastore.RunQueryRequest() # Establish that the response is the type that we expect. @@ -686,7 +737,7 @@ def test_begin_transaction( transport: str = "grpc", request_type=datastore.BeginTransactionRequest ): client = DatastoreClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -701,19 +752,15 @@ def test_begin_transaction( call.return_value = datastore.BeginTransactionResponse( transaction=b"transaction_blob", ) - response = client.begin_transaction(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0] == datastore.BeginTransactionRequest() # Establish that the response is the type that we expect. - assert isinstance(response, datastore.BeginTransactionResponse) - assert response.transaction == b"transaction_blob" @@ -721,12 +768,29 @@ def test_begin_transaction_from_dict(): test_begin_transaction(request_type=dict) +def test_begin_transaction_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = DatastoreClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.begin_transaction), "__call__" + ) as call: + client.begin_transaction() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == datastore.BeginTransactionRequest() + + @pytest.mark.asyncio async def test_begin_transaction_async( transport: str = "grpc_asyncio", request_type=datastore.BeginTransactionRequest ): client = DatastoreAsyncClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -741,18 +805,15 @@ async def test_begin_transaction_async( call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( datastore.BeginTransactionResponse(transaction=b"transaction_blob",) ) - response = await client.begin_transaction(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0] == datastore.BeginTransactionRequest() # Establish that the response is the type that we expect. assert isinstance(response, datastore.BeginTransactionResponse) - assert response.transaction == b"transaction_blob" @@ -762,7 +823,7 @@ async def test_begin_transaction_async_from_dict(): def test_begin_transaction_flattened(): - client = DatastoreClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreClient(credentials=ga_credentials.AnonymousCredentials(),) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object( @@ -770,7 +831,6 @@ def test_begin_transaction_flattened(): ) as call: # Designate an appropriate return value for the call. call.return_value = datastore.BeginTransactionResponse() - # Call the method with a truthy value for each flattened field, # using the keyword arguments to the method. client.begin_transaction(project_id="project_id_value",) @@ -779,12 +839,11 @@ def test_begin_transaction_flattened(): # request object values. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0].project_id == "project_id_value" def test_begin_transaction_flattened_error(): - client = DatastoreClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreClient(credentials=ga_credentials.AnonymousCredentials(),) # Attempting to call a method with both a request object and flattened # fields is an error. @@ -796,7 +855,7 @@ def test_begin_transaction_flattened_error(): @pytest.mark.asyncio async def test_begin_transaction_flattened_async(): - client = DatastoreAsyncClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAsyncClient(credentials=ga_credentials.AnonymousCredentials(),) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object( @@ -816,13 +875,12 @@ async def test_begin_transaction_flattened_async(): # request object values. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0].project_id == "project_id_value" @pytest.mark.asyncio async def test_begin_transaction_flattened_error_async(): - client = DatastoreAsyncClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAsyncClient(credentials=ga_credentials.AnonymousCredentials(),) # Attempting to call a method with both a request object and flattened # fields is an error. @@ -834,7 +892,7 @@ async def test_begin_transaction_flattened_error_async(): def test_commit(transport: str = "grpc", request_type=datastore.CommitRequest): client = DatastoreClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -845,19 +903,15 @@ def test_commit(transport: str = "grpc", request_type=datastore.CommitRequest): with mock.patch.object(type(client.transport.commit), "__call__") as call: # Designate an appropriate return value for the call. call.return_value = datastore.CommitResponse(index_updates=1389,) - response = client.commit(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0] == datastore.CommitRequest() # Establish that the response is the type that we expect. - assert isinstance(response, datastore.CommitResponse) - assert response.index_updates == 1389 @@ -865,12 +919,27 @@ def test_commit_from_dict(): test_commit(request_type=dict) +def test_commit_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = DatastoreClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.commit), "__call__") as call: + client.commit() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == datastore.CommitRequest() + + @pytest.mark.asyncio async def test_commit_async( transport: str = "grpc_asyncio", request_type=datastore.CommitRequest ): client = DatastoreAsyncClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -883,18 +952,15 @@ async def test_commit_async( call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( datastore.CommitResponse(index_updates=1389,) ) - response = await client.commit(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0] == datastore.CommitRequest() # Establish that the response is the type that we expect. assert isinstance(response, datastore.CommitResponse) - assert response.index_updates == 1389 @@ -904,13 +970,12 @@ async def test_commit_async_from_dict(): def test_commit_flattened(): - client = DatastoreClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreClient(credentials=ga_credentials.AnonymousCredentials(),) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.commit), "__call__") as call: # Designate an appropriate return value for the call. call.return_value = datastore.CommitResponse() - # Call the method with a truthy value for each flattened field, # using the keyword arguments to the method. client.commit( @@ -934,11 +999,8 @@ def test_commit_flattened(): # request object values. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0].project_id == "project_id_value" - assert args[0].mode == datastore.CommitRequest.Mode.TRANSACTIONAL - assert args[0].mutations == [ datastore.Mutation( insert=entity.Entity( @@ -948,12 +1010,11 @@ def test_commit_flattened(): ) ) ] - assert args[0].transaction == b"transaction_blob" def test_commit_flattened_error(): - client = DatastoreClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreClient(credentials=ga_credentials.AnonymousCredentials(),) # Attempting to call a method with both a request object and flattened # fields is an error. @@ -979,7 +1040,7 @@ def test_commit_flattened_error(): @pytest.mark.asyncio async def test_commit_flattened_async(): - client = DatastoreAsyncClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAsyncClient(credentials=ga_credentials.AnonymousCredentials(),) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.commit), "__call__") as call: @@ -1012,11 +1073,8 @@ async def test_commit_flattened_async(): # request object values. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0].project_id == "project_id_value" - assert args[0].mode == datastore.CommitRequest.Mode.TRANSACTIONAL - assert args[0].mutations == [ datastore.Mutation( insert=entity.Entity( @@ -1026,13 +1084,12 @@ async def test_commit_flattened_async(): ) ) ] - assert args[0].transaction == b"transaction_blob" @pytest.mark.asyncio async def test_commit_flattened_error_async(): - client = DatastoreAsyncClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAsyncClient(credentials=ga_credentials.AnonymousCredentials(),) # Attempting to call a method with both a request object and flattened # fields is an error. @@ -1058,7 +1115,7 @@ async def test_commit_flattened_error_async(): def test_rollback(transport: str = "grpc", request_type=datastore.RollbackRequest): client = DatastoreClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -1069,17 +1126,14 @@ def test_rollback(transport: str = "grpc", request_type=datastore.RollbackReques with mock.patch.object(type(client.transport.rollback), "__call__") as call: # Designate an appropriate return value for the call. call.return_value = datastore.RollbackResponse() - response = client.rollback(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0] == datastore.RollbackRequest() # Establish that the response is the type that we expect. - assert isinstance(response, datastore.RollbackResponse) @@ -1087,12 +1141,27 @@ def test_rollback_from_dict(): test_rollback(request_type=dict) +def test_rollback_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = DatastoreClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.rollback), "__call__") as call: + client.rollback() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == datastore.RollbackRequest() + + @pytest.mark.asyncio async def test_rollback_async( transport: str = "grpc_asyncio", request_type=datastore.RollbackRequest ): client = DatastoreAsyncClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -1105,13 +1174,11 @@ async def test_rollback_async( call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( datastore.RollbackResponse() ) - response = await client.rollback(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0] == datastore.RollbackRequest() # Establish that the response is the type that we expect. @@ -1124,13 +1191,12 @@ async def test_rollback_async_from_dict(): def test_rollback_flattened(): - client = DatastoreClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreClient(credentials=ga_credentials.AnonymousCredentials(),) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.rollback), "__call__") as call: # Designate an appropriate return value for the call. call.return_value = datastore.RollbackResponse() - # Call the method with a truthy value for each flattened field, # using the keyword arguments to the method. client.rollback( @@ -1141,14 +1207,12 @@ def test_rollback_flattened(): # request object values. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0].project_id == "project_id_value" - assert args[0].transaction == b"transaction_blob" def test_rollback_flattened_error(): - client = DatastoreClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreClient(credentials=ga_credentials.AnonymousCredentials(),) # Attempting to call a method with both a request object and flattened # fields is an error. @@ -1162,7 +1226,7 @@ def test_rollback_flattened_error(): @pytest.mark.asyncio async def test_rollback_flattened_async(): - client = DatastoreAsyncClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAsyncClient(credentials=ga_credentials.AnonymousCredentials(),) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.rollback), "__call__") as call: @@ -1182,15 +1246,13 @@ async def test_rollback_flattened_async(): # request object values. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0].project_id == "project_id_value" - assert args[0].transaction == b"transaction_blob" @pytest.mark.asyncio async def test_rollback_flattened_error_async(): - client = DatastoreAsyncClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAsyncClient(credentials=ga_credentials.AnonymousCredentials(),) # Attempting to call a method with both a request object and flattened # fields is an error. @@ -1206,7 +1268,7 @@ def test_allocate_ids( transport: str = "grpc", request_type=datastore.AllocateIdsRequest ): client = DatastoreClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -1217,17 +1279,14 @@ def test_allocate_ids( with mock.patch.object(type(client.transport.allocate_ids), "__call__") as call: # Designate an appropriate return value for the call. call.return_value = datastore.AllocateIdsResponse() - response = client.allocate_ids(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0] == datastore.AllocateIdsRequest() # Establish that the response is the type that we expect. - assert isinstance(response, datastore.AllocateIdsResponse) @@ -1235,12 +1294,27 @@ def test_allocate_ids_from_dict(): test_allocate_ids(request_type=dict) +def test_allocate_ids_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = DatastoreClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.allocate_ids), "__call__") as call: + client.allocate_ids() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == datastore.AllocateIdsRequest() + + @pytest.mark.asyncio async def test_allocate_ids_async( transport: str = "grpc_asyncio", request_type=datastore.AllocateIdsRequest ): client = DatastoreAsyncClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -1253,13 +1327,11 @@ async def test_allocate_ids_async( call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( datastore.AllocateIdsResponse() ) - response = await client.allocate_ids(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0] == datastore.AllocateIdsRequest() # Establish that the response is the type that we expect. @@ -1272,13 +1344,12 @@ async def test_allocate_ids_async_from_dict(): def test_allocate_ids_flattened(): - client = DatastoreClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreClient(credentials=ga_credentials.AnonymousCredentials(),) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.allocate_ids), "__call__") as call: # Designate an appropriate return value for the call. call.return_value = datastore.AllocateIdsResponse() - # Call the method with a truthy value for each flattened field, # using the keyword arguments to the method. client.allocate_ids( @@ -1294,16 +1365,14 @@ def test_allocate_ids_flattened(): # request object values. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0].project_id == "project_id_value" - assert args[0].keys == [ entity.Key(partition_id=entity.PartitionId(project_id="project_id_value")) ] def test_allocate_ids_flattened_error(): - client = DatastoreClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreClient(credentials=ga_credentials.AnonymousCredentials(),) # Attempting to call a method with both a request object and flattened # fields is an error. @@ -1321,7 +1390,7 @@ def test_allocate_ids_flattened_error(): @pytest.mark.asyncio async def test_allocate_ids_flattened_async(): - client = DatastoreAsyncClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAsyncClient(credentials=ga_credentials.AnonymousCredentials(),) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.allocate_ids), "__call__") as call: @@ -1346,9 +1415,7 @@ async def test_allocate_ids_flattened_async(): # request object values. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0].project_id == "project_id_value" - assert args[0].keys == [ entity.Key(partition_id=entity.PartitionId(project_id="project_id_value")) ] @@ -1356,7 +1423,7 @@ async def test_allocate_ids_flattened_async(): @pytest.mark.asyncio async def test_allocate_ids_flattened_error_async(): - client = DatastoreAsyncClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAsyncClient(credentials=ga_credentials.AnonymousCredentials(),) # Attempting to call a method with both a request object and flattened # fields is an error. @@ -1374,7 +1441,7 @@ async def test_allocate_ids_flattened_error_async(): def test_reserve_ids(transport: str = "grpc", request_type=datastore.ReserveIdsRequest): client = DatastoreClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -1385,17 +1452,14 @@ def test_reserve_ids(transport: str = "grpc", request_type=datastore.ReserveIdsR with mock.patch.object(type(client.transport.reserve_ids), "__call__") as call: # Designate an appropriate return value for the call. call.return_value = datastore.ReserveIdsResponse() - response = client.reserve_ids(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0] == datastore.ReserveIdsRequest() # Establish that the response is the type that we expect. - assert isinstance(response, datastore.ReserveIdsResponse) @@ -1403,12 +1467,27 @@ def test_reserve_ids_from_dict(): test_reserve_ids(request_type=dict) +def test_reserve_ids_empty_call(): + # This test is a coverage failsafe to make sure that totally empty calls, + # i.e. request == None and no flattened fields passed, work. + client = DatastoreClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc", + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object(type(client.transport.reserve_ids), "__call__") as call: + client.reserve_ids() + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == datastore.ReserveIdsRequest() + + @pytest.mark.asyncio async def test_reserve_ids_async( transport: str = "grpc_asyncio", request_type=datastore.ReserveIdsRequest ): client = DatastoreAsyncClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # Everything is optional in proto3 as far as the runtime is concerned, @@ -1421,13 +1500,11 @@ async def test_reserve_ids_async( call.return_value = grpc_helpers_async.FakeUnaryUnaryCall( datastore.ReserveIdsResponse() ) - response = await client.reserve_ids(request) # Establish that the underlying gRPC stub method was called. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0] == datastore.ReserveIdsRequest() # Establish that the response is the type that we expect. @@ -1440,13 +1517,12 @@ async def test_reserve_ids_async_from_dict(): def test_reserve_ids_flattened(): - client = DatastoreClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreClient(credentials=ga_credentials.AnonymousCredentials(),) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.reserve_ids), "__call__") as call: # Designate an appropriate return value for the call. call.return_value = datastore.ReserveIdsResponse() - # Call the method with a truthy value for each flattened field, # using the keyword arguments to the method. client.reserve_ids( @@ -1462,16 +1538,14 @@ def test_reserve_ids_flattened(): # request object values. assert len(call.mock_calls) == 1 _, args, _ = call.mock_calls[0] - assert args[0].project_id == "project_id_value" - assert args[0].keys == [ entity.Key(partition_id=entity.PartitionId(project_id="project_id_value")) ] def test_reserve_ids_flattened_error(): - client = DatastoreClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreClient(credentials=ga_credentials.AnonymousCredentials(),) # Attempting to call a method with both a request object and flattened # fields is an error. @@ -1489,7 +1563,7 @@ def test_reserve_ids_flattened_error(): @pytest.mark.asyncio async def test_reserve_ids_flattened_async(): - client = DatastoreAsyncClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAsyncClient(credentials=ga_credentials.AnonymousCredentials(),) # Mock the actual call within the gRPC stub, and fake the request. with mock.patch.object(type(client.transport.reserve_ids), "__call__") as call: @@ -1514,9 +1588,7 @@ async def test_reserve_ids_flattened_async(): # request object values. assert len(call.mock_calls) _, args, _ = call.mock_calls[0] - assert args[0].project_id == "project_id_value" - assert args[0].keys == [ entity.Key(partition_id=entity.PartitionId(project_id="project_id_value")) ] @@ -1524,7 +1596,7 @@ async def test_reserve_ids_flattened_async(): @pytest.mark.asyncio async def test_reserve_ids_flattened_error_async(): - client = DatastoreAsyncClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreAsyncClient(credentials=ga_credentials.AnonymousCredentials(),) # Attempting to call a method with both a request object and flattened # fields is an error. @@ -1543,16 +1615,16 @@ async def test_reserve_ids_flattened_error_async(): def test_credentials_transport_error(): # It is an error to provide credentials and a transport instance. transport = transports.DatastoreGrpcTransport( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), ) with pytest.raises(ValueError): client = DatastoreClient( - credentials=credentials.AnonymousCredentials(), transport=transport, + credentials=ga_credentials.AnonymousCredentials(), transport=transport, ) # It is an error to provide a credentials file and a transport instance. transport = transports.DatastoreGrpcTransport( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), ) with pytest.raises(ValueError): client = DatastoreClient( @@ -1562,7 +1634,7 @@ def test_credentials_transport_error(): # It is an error to provide scopes and a transport instance. transport = transports.DatastoreGrpcTransport( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), ) with pytest.raises(ValueError): client = DatastoreClient( @@ -1573,7 +1645,7 @@ def test_credentials_transport_error(): def test_transport_instance(): # A client may be instantiated with a custom transport instance. transport = transports.DatastoreGrpcTransport( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), ) client = DatastoreClient(transport=transport) assert client.transport is transport @@ -1582,13 +1654,13 @@ def test_transport_instance(): def test_transport_get_channel(): # A client may be instantiated with a custom transport instance. transport = transports.DatastoreGrpcTransport( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), ) channel = transport.grpc_channel assert channel transport = transports.DatastoreGrpcAsyncIOTransport( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), ) channel = transport.grpc_channel assert channel @@ -1596,27 +1668,27 @@ def test_transport_get_channel(): @pytest.mark.parametrize( "transport_class", - [transports.DatastoreGrpcTransport, transports.DatastoreGrpcAsyncIOTransport], + [transports.DatastoreGrpcTransport, transports.DatastoreGrpcAsyncIOTransport,], ) def test_transport_adc(transport_class): # Test default credentials are used if not provided. - with mock.patch.object(auth, "default") as adc: - adc.return_value = (credentials.AnonymousCredentials(), None) + with mock.patch.object(google.auth, "default") as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) transport_class() adc.assert_called_once() def test_transport_grpc_default(): # A client should use the gRPC transport by default. - client = DatastoreClient(credentials=credentials.AnonymousCredentials(),) + client = DatastoreClient(credentials=ga_credentials.AnonymousCredentials(),) assert isinstance(client.transport, transports.DatastoreGrpcTransport,) def test_datastore_base_transport_error(): # Passing both a credentials object and credentials_file should raise an error - with pytest.raises(exceptions.DuplicateCredentialArgs): + with pytest.raises(core_exceptions.DuplicateCredentialArgs): transport = transports.DatastoreTransport( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), credentials_file="credentials.json", ) @@ -1628,7 +1700,7 @@ def test_datastore_base_transport(): ) as Transport: Transport.return_value = None transport = transports.DatastoreTransport( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), ) # Every method on the transport should just blindly @@ -1646,22 +1718,26 @@ def test_datastore_base_transport(): with pytest.raises(NotImplementedError): getattr(transport, method)(request=object()) + with pytest.raises(NotImplementedError): + transport.close() + def test_datastore_base_transport_with_credentials_file(): # Instantiate the base transport with a credentials file with mock.patch.object( - auth, "load_credentials_from_file" + google.auth, "load_credentials_from_file", autospec=True ) as load_creds, mock.patch( "google.cloud.datastore_v1.services.datastore.transports.DatastoreTransport._prep_wrapped_messages" ) as Transport: Transport.return_value = None - load_creds.return_value = (credentials.AnonymousCredentials(), None) + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) transport = transports.DatastoreTransport( credentials_file="credentials.json", quota_project_id="octopus", ) load_creds.assert_called_once_with( "credentials.json", - scopes=( + scopes=None, + default_scopes=( "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/datastore", ), @@ -1671,22 +1747,23 @@ def test_datastore_base_transport_with_credentials_file(): def test_datastore_base_transport_with_adc(): # Test the default credentials are used if credentials and credentials_file are None. - with mock.patch.object(auth, "default") as adc, mock.patch( + with mock.patch.object(google.auth, "default", autospec=True) as adc, mock.patch( "google.cloud.datastore_v1.services.datastore.transports.DatastoreTransport._prep_wrapped_messages" ) as Transport: Transport.return_value = None - adc.return_value = (credentials.AnonymousCredentials(), None) + adc.return_value = (ga_credentials.AnonymousCredentials(), None) transport = transports.DatastoreTransport() adc.assert_called_once() def test_datastore_auth_adc(): # If no credentials are provided, we should use ADC credentials. - with mock.patch.object(auth, "default") as adc: - adc.return_value = (credentials.AnonymousCredentials(), None) + with mock.patch.object(google.auth, "default", autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) DatastoreClient() adc.assert_called_once_with( - scopes=( + scopes=None, + default_scopes=( "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/datastore", ), @@ -1694,16 +1771,19 @@ def test_datastore_auth_adc(): ) -def test_datastore_transport_auth_adc(): +@pytest.mark.parametrize( + "transport_class", + [transports.DatastoreGrpcTransport, transports.DatastoreGrpcAsyncIOTransport,], +) +def test_datastore_transport_auth_adc(transport_class): # If credentials and host are not provided, the transport class should use # ADC credentials. - with mock.patch.object(auth, "default") as adc: - adc.return_value = (credentials.AnonymousCredentials(), None) - transports.DatastoreGrpcTransport( - host="squid.clam.whelk", quota_project_id="octopus" - ) + with mock.patch.object(google.auth, "default", autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class(quota_project_id="octopus", scopes=["1", "2"]) adc.assert_called_once_with( - scopes=( + scopes=["1", "2"], + default_scopes=( "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/datastore", ), @@ -1711,9 +1791,89 @@ def test_datastore_transport_auth_adc(): ) +@pytest.mark.parametrize( + "transport_class,grpc_helpers", + [ + (transports.DatastoreGrpcTransport, grpc_helpers), + (transports.DatastoreGrpcAsyncIOTransport, grpc_helpers_async), + ], +) +def test_datastore_transport_create_channel(transport_class, grpc_helpers): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object( + google.auth, "default", autospec=True + ) as adc, mock.patch.object( + grpc_helpers, "create_channel", autospec=True + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + adc.return_value = (creds, None) + transport_class(quota_project_id="octopus", scopes=["1", "2"]) + + create_channel.assert_called_with( + "datastore.googleapis.com:443", + credentials=creds, + credentials_file=None, + quota_project_id="octopus", + default_scopes=( + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/datastore", + ), + scopes=["1", "2"], + default_host="datastore.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + +@pytest.mark.parametrize( + "transport_class", + [transports.DatastoreGrpcTransport, transports.DatastoreGrpcAsyncIOTransport], +) +def test_datastore_grpc_transport_client_cert_source_for_mtls(transport_class): + cred = ga_credentials.AnonymousCredentials() + + # Check ssl_channel_credentials is used if provided. + with mock.patch.object(transport_class, "create_channel") as mock_create_channel: + mock_ssl_channel_creds = mock.Mock() + transport_class( + host="squid.clam.whelk", + credentials=cred, + ssl_channel_credentials=mock_ssl_channel_creds, + ) + mock_create_channel.assert_called_once_with( + "squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_channel_creds, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Check if ssl_channel_credentials is not provided, then client_cert_source_for_mtls + # is used. + with mock.patch.object(transport_class, "create_channel", return_value=mock.Mock()): + with mock.patch("grpc.ssl_channel_credentials") as mock_ssl_cred: + transport_class( + credentials=cred, + client_cert_source_for_mtls=client_cert_source_callback, + ) + expected_cert, expected_key = client_cert_source_callback() + mock_ssl_cred.assert_called_once_with( + certificate_chain=expected_cert, private_key=expected_key + ) + + def test_datastore_host_no_port(): client = DatastoreClient( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), client_options=client_options.ClientOptions( api_endpoint="datastore.googleapis.com" ), @@ -1723,7 +1883,7 @@ def test_datastore_host_no_port(): def test_datastore_host_with_port(): client = DatastoreClient( - credentials=credentials.AnonymousCredentials(), + credentials=ga_credentials.AnonymousCredentials(), client_options=client_options.ClientOptions( api_endpoint="datastore.googleapis.com:8000" ), @@ -1732,7 +1892,7 @@ def test_datastore_host_with_port(): def test_datastore_grpc_transport_channel(): - channel = grpc.insecure_channel("http://localhost/") + channel = grpc.secure_channel("http://localhost/", grpc.local_channel_credentials()) # Check that channel is used if provided. transport = transports.DatastoreGrpcTransport( @@ -1744,7 +1904,7 @@ def test_datastore_grpc_transport_channel(): def test_datastore_grpc_asyncio_transport_channel(): - channel = aio.insecure_channel("http://localhost/") + channel = aio.secure_channel("http://localhost/", grpc.local_channel_credentials()) # Check that channel is used if provided. transport = transports.DatastoreGrpcAsyncIOTransport( @@ -1755,6 +1915,8 @@ def test_datastore_grpc_asyncio_transport_channel(): assert transport._ssl_channel_credentials == None +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. @pytest.mark.parametrize( "transport_class", [transports.DatastoreGrpcTransport, transports.DatastoreGrpcAsyncIOTransport], @@ -1764,7 +1926,7 @@ def test_datastore_transport_channel_mtls_with_client_cert_source(transport_clas "grpc.ssl_channel_credentials", autospec=True ) as grpc_ssl_channel_cred: with mock.patch.object( - transport_class, "create_channel", autospec=True + transport_class, "create_channel" ) as grpc_create_channel: mock_ssl_cred = mock.Mock() grpc_ssl_channel_cred.return_value = mock_ssl_cred @@ -1772,9 +1934,9 @@ def test_datastore_transport_channel_mtls_with_client_cert_source(transport_clas mock_grpc_channel = mock.Mock() grpc_create_channel.return_value = mock_grpc_channel - cred = credentials.AnonymousCredentials() + cred = ga_credentials.AnonymousCredentials() with pytest.warns(DeprecationWarning): - with mock.patch.object(auth, "default") as adc: + with mock.patch.object(google.auth, "default") as adc: adc.return_value = (cred, None) transport = transport_class( host="squid.clam.whelk", @@ -1790,17 +1952,20 @@ def test_datastore_transport_channel_mtls_with_client_cert_source(transport_clas "mtls.squid.clam.whelk:443", credentials=cred, credentials_file=None, - scopes=( - "https://www.googleapis.com/auth/cloud-platform", - "https://www.googleapis.com/auth/datastore", - ), + scopes=None, ssl_credentials=mock_ssl_cred, quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) assert transport.grpc_channel == mock_grpc_channel assert transport._ssl_channel_credentials == mock_ssl_cred +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. @pytest.mark.parametrize( "transport_class", [transports.DatastoreGrpcTransport, transports.DatastoreGrpcAsyncIOTransport], @@ -1813,7 +1978,7 @@ def test_datastore_transport_channel_mtls_with_adc(transport_class): ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred), ): with mock.patch.object( - transport_class, "create_channel", autospec=True + transport_class, "create_channel" ) as grpc_create_channel: mock_grpc_channel = mock.Mock() grpc_create_channel.return_value = mock_grpc_channel @@ -1831,19 +1996,19 @@ def test_datastore_transport_channel_mtls_with_adc(transport_class): "mtls.squid.clam.whelk:443", credentials=mock_cred, credentials_file=None, - scopes=( - "https://www.googleapis.com/auth/cloud-platform", - "https://www.googleapis.com/auth/datastore", - ), + scopes=None, ssl_credentials=mock_ssl_cred, quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) assert transport.grpc_channel == mock_grpc_channel def test_common_billing_account_path(): billing_account = "squid" - expected = "billingAccounts/{billing_account}".format( billing_account=billing_account, ) @@ -1864,7 +2029,6 @@ def test_parse_common_billing_account_path(): def test_common_folder_path(): folder = "whelk" - expected = "folders/{folder}".format(folder=folder,) actual = DatastoreClient.common_folder_path(folder) assert expected == actual @@ -1883,7 +2047,6 @@ def test_parse_common_folder_path(): def test_common_organization_path(): organization = "oyster" - expected = "organizations/{organization}".format(organization=organization,) actual = DatastoreClient.common_organization_path(organization) assert expected == actual @@ -1902,7 +2065,6 @@ def test_parse_common_organization_path(): def test_common_project_path(): project = "cuttlefish" - expected = "projects/{project}".format(project=project,) actual = DatastoreClient.common_project_path(project) assert expected == actual @@ -1922,7 +2084,6 @@ def test_parse_common_project_path(): def test_common_location_path(): project = "winkle" location = "nautilus" - expected = "projects/{project}/locations/{location}".format( project=project, location=location, ) @@ -1949,7 +2110,7 @@ def test_client_withDEFAULT_CLIENT_INFO(): transports.DatastoreTransport, "_prep_wrapped_messages" ) as prep: client = DatastoreClient( - credentials=credentials.AnonymousCredentials(), client_info=client_info, + credentials=ga_credentials.AnonymousCredentials(), client_info=client_info, ) prep.assert_called_once_with(client_info) @@ -1958,6 +2119,52 @@ def test_client_withDEFAULT_CLIENT_INFO(): ) as prep: transport_class = DatastoreClient.get_transport_class() transport = transport_class( - credentials=credentials.AnonymousCredentials(), client_info=client_info, + credentials=ga_credentials.AnonymousCredentials(), client_info=client_info, ) prep.assert_called_once_with(client_info) + + +@pytest.mark.asyncio +async def test_transport_close_async(): + client = DatastoreAsyncClient( + credentials=ga_credentials.AnonymousCredentials(), transport="grpc_asyncio", + ) + with mock.patch.object( + type(getattr(client.transport, "grpc_channel")), "close" + ) as close: + async with client: + close.assert_not_called() + close.assert_called_once() + + +def test_transport_close(): + transports = { + "grpc": "_grpc_channel", + } + + for transport, close_name in transports.items(): + client = DatastoreClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport + ) + with mock.patch.object( + type(getattr(client.transport, close_name)), "close" + ) as close: + with client: + close.assert_not_called() + close.assert_called_once() + + +def test_client_ctx(): + transports = [ + "grpc", + ] + for transport in transports: + client = DatastoreClient( + credentials=ga_credentials.AnonymousCredentials(), transport=transport + ) + # Test client calls underlying transport. + with mock.patch.object(type(client.transport), "close") as close: + close.assert_not_called() + with client: + pass + close.assert_called()