From 2c2cd1b8c0977698357760c3b31ea2757ee55ab5 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Tue, 8 Jun 2021 12:49:19 -0400 Subject: [PATCH] refactor: add / use 'Client._post_resource' method (#443) Toward #38. --- google/cloud/storage/blob.py | 19 +- google/cloud/storage/bucket.py | 34 +- google/cloud/storage/client.py | 103 ++- google/cloud/storage/notification.py | 25 +- tests/unit/test_blob.py | 1234 +++++++++++++------------- tests/unit/test_bucket.py | 618 ++++++------- tests/unit/test_client.py | 520 +++++------ tests/unit/test_notification.py | 83 +- 8 files changed, 1400 insertions(+), 1236 deletions(-) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 391ced253..73851ea02 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -3164,14 +3164,13 @@ def compose( "sourceObjects": source_objects, "destination": self._properties.copy(), } - api_response = client._connection.api_request( - method="POST", - path=self.path + "/compose", + api_response = client._post_resource( + "{}/compose".format(self.path), + request, query_params=query_params, - data=request, - _target_object=self, timeout=timeout, retry=retry, + _target_object=self, ) self._set_properties(api_response) @@ -3315,15 +3314,15 @@ def rewrite( if_source_metageneration_not_match=if_source_metageneration_not_match, ) - api_response = client._connection.api_request( - method="POST", - path=source.path + "/rewriteTo" + self.path, + path = "{}/rewriteTo{}".format(source.path, self.path) + api_response = client._post_resource( + path, + self._properties, query_params=query_params, - data=self._properties, headers=headers, - _target_object=self, timeout=timeout, retry=retry, + _target_object=self, ) rewritten = int(api_response["totalBytesRewritten"]) size = int(api_response["objectSize"]) diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index c019b2f12..7703dc234 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -1986,13 +1986,13 @@ def copy_blob( new_blob = Blob(bucket=destination_bucket, name=new_name) api_path = blob.path + "/copyTo" + new_blob.path - copy_result = client._connection.api_request( - method="POST", - path=api_path, + copy_result = client._post_resource( + api_path, + None, query_params=query_params, - _target_object=new_blob, timeout=timeout, retry=retry, + _target_object=new_blob, ) if not preserve_acl: @@ -2006,7 +2006,6 @@ def rename_blob( blob, new_name, client=None, - timeout=_DEFAULT_TIMEOUT, if_generation_match=None, if_generation_not_match=None, if_metageneration_match=None, @@ -2015,6 +2014,7 @@ def rename_blob( if_source_generation_not_match=None, if_source_metageneration_match=None, if_source_metageneration_not_match=None, + timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, ): """Rename the given blob using copy and delete operations. @@ -2044,14 +2044,6 @@ def rename_blob( :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. - :type timeout: float or tuple - :param timeout: (Optional) The amount of time, in seconds, to wait - for the server response. The timeout applies to each individual - request. - - Can also be passed as a tuple (connect_timeout, read_timeout). - See :meth:`requests.Session.request` documentation for details. - :type if_generation_match: long :param if_generation_match: (Optional) Makes the operation conditional on whether the destination @@ -2113,6 +2105,14 @@ def rename_blob( does not match the given value. Also used in the delete request. + :type timeout: float or tuple + :param timeout: (Optional) The amount of time, in seconds, to wait + for the server response. The timeout applies to each individual + request. + + Can also be passed as a tuple (connect_timeout, read_timeout). + See :meth:`requests.Session.request` documentation for details. + :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy :param retry: (Optional) How to retry the RPC. A None value will disable retries. A google.api_core.retry.Retry value will enable retries, and the object will @@ -3311,13 +3311,13 @@ def lock_retention_policy( query_params["userProject"] = self.user_project path = "/b/{}/lockRetentionPolicy".format(self.name) - api_response = client._connection.api_request( - method="POST", - path=path, + api_response = client._post_resource( + path, + None, query_params=query_params, - _target_object=self, timeout=timeout, retry=retry, + _target_object=self, ) self._set_properties(api_response) diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index effcddddd..57c5b4103 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -528,6 +528,77 @@ def _put_resource( _target_object=_target_object, ) + def _post_resource( + self, + path, + data, + query_params=None, + headers=None, + timeout=_DEFAULT_TIMEOUT, + retry=None, + _target_object=None, + ): + """Helper for bucket / blob methods making API 'POST' calls. + + Args: + path str: + The path of the resource to which to post. + + data dict: + The data to be posted. + + query_params Optional[dict]: + HTTP query parameters to be passed + + headers Optional[dict]: + HTTP headers to be passed + + timeout (Optional[Union[float, Tuple[float, float]]]): + The amount of time, in seconds, to wait for the server response. + + Can also be passed as a tuple (connect_timeout, read_timeout). + See :meth:`requests.Session.request` documentation for details. + + retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]): + How to retry the RPC. A None value will disable retries. + A google.api_core.retry.Retry value will enable retries, and the object will + define retriable response codes and errors and configure backoff and timeout options. + + A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and + activates it only if certain conditions are met. This class exists to provide safe defaults + for RPC calls that are not technically safe to retry normally (due to potential data + duplication or other side-effects) but become safe to retry if a condition such as + if_metageneration_match is set. + + See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for + information on retry types and how to configure them. + + _target_object (Union[ \ + :class:`~google.cloud.storage.bucket.Bucket`, \ + :class:`~google.cloud.storage.bucket.blob`, \ + ]): + Object to which future data is to be applied -- only relevant + in the context of a batch. + + Returns: + dict + The JSON resource returned from the post. + + Raises: + google.cloud.exceptions.NotFound + If the bucket is not found. + """ + return self._connection.api_request( + method="POST", + path=path, + data=data, + query_params=query_params, + headers=headers, + timeout=timeout, + retry=retry, + _target_object=_target_object, + ) + def _delete_resource( self, path, @@ -875,14 +946,13 @@ def create_bucket( if location is not None: properties["location"] = location - api_response = self._connection.api_request( - method="POST", - path="/b", + api_response = self._post_resource( + "/b", + properties, query_params=query_params, - data=properties, - _target_object=bucket, timeout=timeout, retry=retry, + _target_object=bucket, ) bucket._set_properties(api_response) @@ -1278,6 +1348,7 @@ def create_hmac_key( project_id=None, user_project=None, timeout=_DEFAULT_TIMEOUT, + retry=None, ): """Create an HMAC key for a service account. @@ -1298,6 +1369,20 @@ def create_hmac_key( Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy + :param retry: (Optional) How to retry the RPC. A None value will disable retries. + A google.api_core.retry.Retry value will enable retries, and the object will + define retriable response codes and errors and configure backoff and timeout options. + + A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and + activates it only if certain conditions are met. This class exists to provide safe defaults + for RPC calls that are not technically safe to retry normally (due to potential data + duplication or other side-effects) but become safe to retry if a condition such as + if_metageneration_match is set. + + See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for + information on retry types and how to configure them. + :rtype: Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str] :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string. @@ -1311,12 +1396,8 @@ def create_hmac_key( if user_project is not None: qs_params["userProject"] = user_project - api_response = self._connection.api_request( - method="POST", - path=path, - query_params=qs_params, - timeout=timeout, - retry=None, + api_response = self._post_resource( + path, None, query_params=qs_params, timeout=timeout, retry=retry, ) metadata = HMACKeyMetadata(self) metadata._properties = api_response["metadata"] diff --git a/google/cloud/storage/notification.py b/google/cloud/storage/notification.py index 2f5661fce..e86859466 100644 --- a/google/cloud/storage/notification.py +++ b/google/cloud/storage/notification.py @@ -233,7 +233,7 @@ def _set_properties(self, response): self._properties.clear() self._properties.update(response) - def create(self, client=None, timeout=_DEFAULT_TIMEOUT): + def create(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=None): """API wrapper: create the notification. See: @@ -251,6 +251,20 @@ def create(self, client=None, timeout=_DEFAULT_TIMEOUT): Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + + :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy + :param retry: (Optional) How to retry the RPC. A None value will disable retries. + A google.api_core.retry.Retry value will enable retries, and the object will + define retriable response codes and errors and configure backoff and timeout options. + + A google.cloud.storage.retry.ConditionalRetryPolicy value wraps a Retry object and + activates it only if certain conditions are met. This class exists to provide safe defaults + for RPC calls that are not technically safe to retry normally (due to potential data + duplication or other side-effects) but become safe to retry if a condition such as + if_metageneration_match is set. + + See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for + information on retry types and how to configure them. """ if self.notification_id is not None: raise ValueError( @@ -266,13 +280,8 @@ def create(self, client=None, timeout=_DEFAULT_TIMEOUT): path = "/b/{}/notificationConfigs".format(self.bucket.name) properties = self._properties.copy() properties["topic"] = _TOPIC_REF_FMT.format(self.topic_project, self.topic_name) - self._properties = client._connection.api_request( - method="POST", - path=path, - query_params=query_params, - data=properties, - timeout=timeout, - retry=None, + self._properties = client._post_resource( + path, properties, query_params=query_params, timeout=timeout, retry=retry, ) def exists(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index 548ab8619..071033a45 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -3540,744 +3540,805 @@ def test_make_private_w_timeout_w_retry(self): ) def test_compose_wo_content_type_set(self): - SOURCE_1 = "source-1" - SOURCE_2 = "source-2" - DESTINATION = "destination" - RESOURCE = {} - after = ({"status": http_client.OK}, RESOURCE) - connection = _Connection(after) - client = _Client(connection) + source_1_name = "source-1" + source_2_name = "source-2" + destination_name = "destination" + api_response = {} + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response bucket = _Bucket(client=client) - source_1 = self._make_one(SOURCE_1, bucket=bucket) - source_2 = self._make_one(SOURCE_2, bucket=bucket) - destination = self._make_one(DESTINATION, bucket=bucket) + source_1 = self._make_one(source_1_name, bucket=bucket) + source_2 = self._make_one(source_2_name, bucket=bucket) + destination = self._make_one(destination_name, bucket=bucket) # no destination.content_type set destination.compose(sources=[source_1, source_2]) self.assertIsNone(destination.content_type) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "POST", - "path": "/b/name/o/%s/compose" % DESTINATION, - "query_params": {}, - "data": { - "sourceObjects": [{"name": source_1.name}, {"name": source_2.name}], - "destination": {}, - }, - "_target_object": destination, - "timeout": self._get_default_timeout(), - "retry": DEFAULT_RETRY_IF_GENERATION_SPECIFIED, - }, + expected_path = "/b/name/o/%s/compose" % destination_name + expected_data = { + "sourceObjects": [{"name": source_1_name}, {"name": source_2_name}], + "destination": {}, + } + expected_query_params = {} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + _target_object=destination, ) - def test_compose_minimal_w_user_project(self): - SOURCE_1 = "source-1" - SOURCE_2 = "source-2" - DESTINATION = "destination" - RESOURCE = {"etag": "DEADBEEF"} - USER_PROJECT = "user-project-123" - after = ({"status": http_client.OK}, RESOURCE) - connection = _Connection(after) - client = _Client(connection) - bucket = _Bucket(client=client, user_project=USER_PROJECT) - source_1 = self._make_one(SOURCE_1, bucket=bucket) - source_2 = self._make_one(SOURCE_2, bucket=bucket) - destination = self._make_one(DESTINATION, bucket=bucket) + def test_compose_minimal_w_user_project_w_timeout(self): + source_1_name = "source-1" + source_2_name = "source-2" + destination_name = "destination" + api_response = {"etag": "DEADBEEF"} + user_project = "user-project-123" + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response + bucket = _Bucket(client=client, user_project=user_project) + source_1 = self._make_one(source_1_name, bucket=bucket) + source_2 = self._make_one(source_2_name, bucket=bucket) + destination = self._make_one(destination_name, bucket=bucket) destination.content_type = "text/plain" + timeout = 42 - destination.compose(sources=[source_1, source_2], timeout=42) + destination.compose(sources=[source_1, source_2], timeout=timeout) self.assertEqual(destination.etag, "DEADBEEF") - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "POST", - "path": "/b/name/o/%s/compose" % DESTINATION, - "query_params": {"userProject": USER_PROJECT}, - "data": { - "sourceObjects": [{"name": source_1.name}, {"name": source_2.name}], - "destination": {"contentType": "text/plain"}, - }, - "_target_object": destination, - "timeout": 42, - "retry": DEFAULT_RETRY_IF_GENERATION_SPECIFIED, - }, + expected_path = "/b/name/o/%s/compose" % destination_name + expected_data = { + "sourceObjects": [{"name": source_1_name}, {"name": source_2_name}], + "destination": {"contentType": "text/plain"}, + } + expected_query_params = {"userProject": user_project} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=timeout, + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + _target_object=destination, ) - def test_compose_w_additional_property_changes(self): - SOURCE_1 = "source-1" - SOURCE_2 = "source-2" - DESTINATION = "destination" - RESOURCE = {"etag": "DEADBEEF"} - after = ({"status": http_client.OK}, RESOURCE) - connection = _Connection(after) - client = _Client(connection) + def test_compose_w_additional_property_changes_w_retry(self): + source_1_name = "source-1" + source_2_name = "source-2" + destination_name = "destination" + api_response = {"etag": "DEADBEEF"} + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response bucket = _Bucket(client=client) - source_1 = self._make_one(SOURCE_1, bucket=bucket) - source_2 = self._make_one(SOURCE_2, bucket=bucket) - destination = self._make_one(DESTINATION, bucket=bucket) + source_1 = self._make_one(source_1_name, bucket=bucket) + source_2 = self._make_one(source_2_name, bucket=bucket) + destination = self._make_one(destination_name, bucket=bucket) destination.content_type = "text/plain" destination.content_language = "en-US" destination.metadata = {"my-key": "my-value"} + retry = mock.Mock(spec=[]) - destination.compose(sources=[source_1, source_2]) + destination.compose(sources=[source_1, source_2], retry=retry) self.assertEqual(destination.etag, "DEADBEEF") - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "POST", - "path": "/b/name/o/%s/compose" % DESTINATION, - "query_params": {}, - "data": { - "sourceObjects": [{"name": source_1.name}, {"name": source_2.name}], - "destination": { - "contentType": "text/plain", - "contentLanguage": "en-US", - "metadata": {"my-key": "my-value"}, - }, - }, - "_target_object": destination, - "timeout": self._get_default_timeout(), - "retry": DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + expected_path = "/b/name/o/%s/compose" % destination_name + expected_data = { + "sourceObjects": [{"name": source_1_name}, {"name": source_2_name}], + "destination": { + "contentType": "text/plain", + "contentLanguage": "en-US", + "metadata": {"my-key": "my-value"}, }, + } + expected_query_params = {} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=self._get_default_timeout(), + retry=retry, + _target_object=destination, ) def test_compose_w_generation_match(self): - SOURCE_1 = "source-1" - SOURCE_2 = "source-2" - DESTINATION = "destination" - RESOURCE = {} - GENERATION_NUMBERS = [6, 9] - METAGENERATION_NUMBERS = [7, 1] - - after = ({"status": http_client.OK}, RESOURCE) - connection = _Connection(after) - client = _Client(connection) + source_1_name = "source-1" + source_2_name = "source-2" + destination_name = "destination" + api_response = {} + generation_numbers = [6, 9] + metageneration_numbers = [7, 1] + + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response bucket = _Bucket(client=client) - source_1 = self._make_one(SOURCE_1, bucket=bucket) - source_2 = self._make_one(SOURCE_2, bucket=bucket) + source_1 = self._make_one(source_1_name, bucket=bucket) + source_2 = self._make_one(source_2_name, bucket=bucket) - destination = self._make_one(DESTINATION, bucket=bucket) + destination = self._make_one(destination_name, bucket=bucket) destination.compose( sources=[source_1, source_2], - if_generation_match=GENERATION_NUMBERS, - if_metageneration_match=METAGENERATION_NUMBERS, + if_generation_match=generation_numbers, + if_metageneration_match=metageneration_numbers, ) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "POST", - "path": "/b/name/o/%s/compose" % DESTINATION, - "query_params": {}, - "data": { - "sourceObjects": [ - { - "name": source_1.name, - "objectPreconditions": { - "ifGenerationMatch": GENERATION_NUMBERS[0], - "ifMetagenerationMatch": METAGENERATION_NUMBERS[0], - }, - }, - { - "name": source_2.name, - "objectPreconditions": { - "ifGenerationMatch": GENERATION_NUMBERS[1], - "ifMetagenerationMatch": METAGENERATION_NUMBERS[1], - }, - }, - ], - "destination": {}, + expected_path = "/b/name/o/%s/compose" % destination_name + expected_data = { + "sourceObjects": [ + { + "name": source_1_name, + "objectPreconditions": { + "ifGenerationMatch": generation_numbers[0], + "ifMetagenerationMatch": metageneration_numbers[0], + }, }, - "_target_object": destination, - "timeout": self._get_default_timeout(), - "retry": DEFAULT_RETRY_IF_GENERATION_SPECIFIED, - }, + { + "name": source_2_name, + "objectPreconditions": { + "ifGenerationMatch": generation_numbers[1], + "ifMetagenerationMatch": metageneration_numbers[1], + }, + }, + ], + "destination": {}, + } + expected_query_params = {} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + _target_object=destination, ) def test_compose_w_generation_match_bad_length(self): - SOURCE_1 = "source-1" - SOURCE_2 = "source-2" - DESTINATION = "destination" - GENERATION_NUMBERS = [6] - METAGENERATION_NUMBERS = [7] - - after = ({"status": http_client.OK}, {}) - connection = _Connection(after) - client = _Client(connection) + source_1_name = "source-1" + source_2_name = "source-2" + destination_name = "destination" + generation_numbers = [6] + client = mock.Mock(spec=["_post_resource"]) bucket = _Bucket(client=client) - source_1 = self._make_one(SOURCE_1, bucket=bucket) - source_2 = self._make_one(SOURCE_2, bucket=bucket) + source_1 = self._make_one(source_1_name, bucket=bucket) + source_2 = self._make_one(source_2_name, bucket=bucket) - destination = self._make_one(DESTINATION, bucket=bucket) + destination = self._make_one(destination_name, bucket=bucket) with self.assertRaises(ValueError): destination.compose( - sources=[source_1, source_2], if_generation_match=GENERATION_NUMBERS + sources=[source_1, source_2], if_generation_match=generation_numbers ) + + client._post_resource.assert_not_called() + + def test_compose_w_metageneration_match_bad_length(self): + source_1_name = "source-1" + source_2_name = "source-2" + destination_name = "destination" + metageneration_numbers = [7] + client = mock.Mock(spec=["_post_resource"]) + bucket = _Bucket(client=client) + source_1 = self._make_one(source_1_name, bucket=bucket) + source_2 = self._make_one(source_2_name, bucket=bucket) + destination = self._make_one(destination_name, bucket=bucket) + with self.assertRaises(ValueError): destination.compose( sources=[source_1, source_2], - if_metageneration_match=METAGENERATION_NUMBERS, + if_metageneration_match=metageneration_numbers, ) - def test_compose_w_generation_match_nones(self): - SOURCE_1 = "source-1" - SOURCE_2 = "source-2" - DESTINATION = "destination" - GENERATION_NUMBERS = [6, None] + client._post_resource.assert_not_called() - after = ({"status": http_client.OK}, {}) - connection = _Connection(after) - client = _Client(connection) + def test_compose_w_generation_match_nones(self): + source_1_name = "source-1" + source_2_name = "source-2" + destination_name = "destination" + generation_numbers = [6, None] + api_response = {} + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response bucket = _Bucket(client=client) - source_1 = self._make_one(SOURCE_1, bucket=bucket) - source_2 = self._make_one(SOURCE_2, bucket=bucket) + source_1 = self._make_one(source_1_name, bucket=bucket) + source_2 = self._make_one(source_2_name, bucket=bucket) + destination = self._make_one(destination_name, bucket=bucket) - destination = self._make_one(DESTINATION, bucket=bucket) destination.compose( - sources=[source_1, source_2], if_generation_match=GENERATION_NUMBERS + sources=[source_1, source_2], if_generation_match=generation_numbers ) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "POST", - "path": "/b/name/o/%s/compose" % DESTINATION, - "query_params": {}, - "data": { - "sourceObjects": [ - { - "name": source_1.name, - "objectPreconditions": { - "ifGenerationMatch": GENERATION_NUMBERS[0] - }, - }, - {"name": source_2.name}, - ], - "destination": {}, + expected_path = "/b/name/o/%s/compose" % destination_name + expected_data = { + "sourceObjects": [ + { + "name": source_1_name, + "objectPreconditions": { + "ifGenerationMatch": generation_numbers[0], + }, }, - "_target_object": destination, - "timeout": self._get_default_timeout(), - "retry": DEFAULT_RETRY_IF_GENERATION_SPECIFIED, - }, + {"name": source_2_name}, + ], + "destination": {}, + } + expected_query_params = {} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + _target_object=destination, ) - def test_rewrite_response_without_resource(self): - SOURCE_BLOB = "source" - DEST_BLOB = "dest" - DEST_BUCKET = "other-bucket" - TOKEN = "TOKEN" - RESPONSE = { - "totalBytesRewritten": 33, - "objectSize": 42, + def test_rewrite_w_response_wo_resource(self): + source_name = "source" + dest_name = "dest" + other_bucket_name = "other-bucket" + bytes_rewritten = 33 + object_size = 52 + rewrite_token = "TOKEN" + api_response = { + "totalBytesRewritten": bytes_rewritten, + "objectSize": object_size, "done": False, - "rewriteToken": TOKEN, + "rewriteToken": rewrite_token, } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response source_bucket = _Bucket(client=client) - source_blob = self._make_one(SOURCE_BLOB, bucket=source_bucket) - dest_bucket = _Bucket(client=client, name=DEST_BUCKET) - dest_blob = self._make_one(DEST_BLOB, bucket=dest_bucket) + source_blob = self._make_one(source_name, bucket=source_bucket) + dest_bucket = _Bucket(client=client, name=other_bucket_name) + dest_blob = self._make_one(dest_name, bucket=dest_bucket) token, rewritten, size = dest_blob.rewrite(source_blob) - self.assertEqual(token, TOKEN) - self.assertEqual(rewritten, 33) - self.assertEqual(size, 42) - - def test_rewrite_w_generations(self): - SOURCE_BLOB = "source" - SOURCE_GENERATION = 42 - DEST_BLOB = "dest" - DEST_BUCKET = "other-bucket" - DEST_GENERATION = 43 - TOKEN = "TOKEN" - RESPONSE = { - "totalBytesRewritten": 33, - "objectSize": 42, + self.assertEqual(token, rewrite_token) + self.assertEqual(rewritten, bytes_rewritten) + self.assertEqual(size, object_size) + + expected_path = "/b/%s/o/%s/rewriteTo/b/%s/o/%s" % ( + source_bucket.name, + source_name, + other_bucket_name, + dest_name, + ) + expected_data = {} + expected_query_params = {} + expected_headers = {} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + headers=expected_headers, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + _target_object=dest_blob, + ) + + def test_rewrite_w_generations_w_timeout(self): + source_name = "source" + source_generation = 22 + dest_name = "dest" + other_bucket_name = "other-bucket" + dest_generation = 23 + bytes_rewritten = 33 + object_size = 52 + rewrite_token = "TOKEN" + api_response = { + "totalBytesRewritten": bytes_rewritten, + "objectSize": object_size, "done": False, - "rewriteToken": TOKEN, + "rewriteToken": rewrite_token, } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response source_bucket = _Bucket(client=client) source_blob = self._make_one( - SOURCE_BLOB, bucket=source_bucket, generation=SOURCE_GENERATION + source_name, bucket=source_bucket, generation=source_generation ) - dest_bucket = _Bucket(client=client, name=DEST_BUCKET) + dest_bucket = _Bucket(client=client, name=other_bucket_name) dest_blob = self._make_one( - DEST_BLOB, bucket=dest_bucket, generation=DEST_GENERATION + dest_name, bucket=dest_bucket, generation=dest_generation ) + timeout = 42 - token, rewritten, size = dest_blob.rewrite(source_blob, timeout=42) + token, rewritten, size = dest_blob.rewrite(source_blob, timeout=timeout) - self.assertEqual(token, TOKEN) - self.assertEqual(rewritten, 33) - self.assertEqual(size, 42) + self.assertEqual(token, rewrite_token) + self.assertEqual(rewritten, bytes_rewritten) + self.assertEqual(size, object_size) - (kw,) = connection._requested - self.assertEqual(kw["method"], "POST") - self.assertEqual( - kw["path"], - "/b/%s/o/%s/rewriteTo/b/%s/o/%s" - % ( - (source_bucket.name, source_blob.name, dest_bucket.name, dest_blob.name) - ), - ) - self.assertEqual(kw["query_params"], {"sourceGeneration": SOURCE_GENERATION}) - self.assertEqual(kw["timeout"], 42) - - def test_rewrite_w_generation_match(self): - SOURCE_BLOB = "source" - SOURCE_GENERATION_NUMBER = 42 - DEST_BLOB = "dest" - DEST_BUCKET = "other-bucket" - DEST_GENERATION_NUMBER = 16 - TOKEN = "TOKEN" - RESPONSE = { - "totalBytesRewritten": 33, - "objectSize": 42, + expected_path = "/b/%s/o/%s/rewriteTo/b/%s/o/%s" % ( + source_bucket.name, + source_name, + other_bucket_name, + dest_name, + ) + expected_data = {"generation": dest_generation} + expected_query_params = {"sourceGeneration": source_generation} + expected_headers = {} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + headers=expected_headers, + timeout=timeout, + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + _target_object=dest_blob, + ) + + def test_rewrite_w_generation_match_w_retry(self): + source_name = "source" + source_generation = 42 + dest_name = "dest" + other_bucket_name = "other-bucket" + dest_generation = 16 + bytes_rewritten = 33 + object_size = 52 + rewrite_token = "TOKEN" + api_response = { + "totalBytesRewritten": bytes_rewritten, + "objectSize": object_size, "done": False, - "rewriteToken": TOKEN, + "rewriteToken": rewrite_token, } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response source_bucket = _Bucket(client=client) source_blob = self._make_one( - SOURCE_BLOB, bucket=source_bucket, generation=SOURCE_GENERATION_NUMBER + source_name, bucket=source_bucket, generation=source_generation ) - dest_bucket = _Bucket(client=client, name=DEST_BUCKET) + dest_bucket = _Bucket(client=client, name=other_bucket_name) dest_blob = self._make_one( - DEST_BLOB, bucket=dest_bucket, generation=DEST_GENERATION_NUMBER + dest_name, bucket=dest_bucket, generation=dest_generation ) + retry = mock.Mock(spec=[]) + token, rewritten, size = dest_blob.rewrite( source_blob, - timeout=42, if_generation_match=dest_blob.generation, if_source_generation_match=source_blob.generation, + retry=retry, ) - (kw,) = connection._requested - self.assertEqual(kw["method"], "POST") - self.assertEqual( - kw["path"], - "/b/%s/o/%s/rewriteTo/b/%s/o/%s" - % ( - (source_bucket.name, source_blob.name, dest_bucket.name, dest_blob.name) - ), + + self.assertEqual(token, rewrite_token) + self.assertEqual(rewritten, bytes_rewritten) + self.assertEqual(size, object_size) + + expected_path = "/b/%s/o/%s/rewriteTo/b/%s/o/%s" % ( + source_bucket.name, + source_name, + other_bucket_name, + dest_name, ) - self.assertEqual( - kw["query_params"], - { - "ifSourceGenerationMatch": SOURCE_GENERATION_NUMBER, - "ifGenerationMatch": DEST_GENERATION_NUMBER, - "sourceGeneration": SOURCE_GENERATION_NUMBER, - }, + expected_data = {"generation": dest_generation} + expected_query_params = { + "ifSourceGenerationMatch": source_generation, + "ifGenerationMatch": dest_generation, + "sourceGeneration": source_generation, + } + expected_headers = {} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + headers=expected_headers, + timeout=self._get_default_timeout(), + retry=retry, + _target_object=dest_blob, ) - self.assertEqual(kw["timeout"], 42) def test_rewrite_other_bucket_other_name_no_encryption_partial(self): - SOURCE_BLOB = "source" - DEST_BLOB = "dest" - DEST_BUCKET = "other-bucket" - TOKEN = "TOKEN" - RESPONSE = { - "totalBytesRewritten": 33, - "objectSize": 42, + source_name = "source" + dest_name = "dest" + other_bucket_name = "other-bucket" + bytes_rewritten = 33 + object_size = 52 + rewrite_token = "TOKEN" + api_response = { + "totalBytesRewritten": bytes_rewritten, + "objectSize": object_size, "done": False, - "rewriteToken": TOKEN, - "resource": {"etag": "DEADBEEF"}, + "rewriteToken": rewrite_token, } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response source_bucket = _Bucket(client=client) - source_blob = self._make_one(SOURCE_BLOB, bucket=source_bucket) - dest_bucket = _Bucket(client=client, name=DEST_BUCKET) - dest_blob = self._make_one(DEST_BLOB, bucket=dest_bucket) + source_blob = self._make_one(source_name, bucket=source_bucket) + dest_bucket = _Bucket(client=client, name=other_bucket_name) + dest_blob = self._make_one(dest_name, bucket=dest_bucket) token, rewritten, size = dest_blob.rewrite(source_blob) - self.assertEqual(token, TOKEN) - self.assertEqual(rewritten, 33) - self.assertEqual(size, 42) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "POST") - PATH = "/b/name/o/%s/rewriteTo/b/%s/o/%s" % ( - SOURCE_BLOB, - DEST_BUCKET, - DEST_BLOB, - ) - self.assertEqual(kw[0]["path"], PATH) - self.assertEqual(kw[0]["query_params"], {}) - SENT = {} - self.assertEqual(kw[0]["data"], SENT) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - - headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} - self.assertNotIn("X-Goog-Copy-Source-Encryption-Algorithm", headers) - self.assertNotIn("X-Goog-Copy-Source-Encryption-Key", headers) - self.assertNotIn("X-Goog-Copy-Source-Encryption-Key-Sha256", headers) - self.assertNotIn("X-Goog-Encryption-Algorithm", headers) - self.assertNotIn("X-Goog-Encryption-Key", headers) - self.assertNotIn("X-Goog-Encryption-Key-Sha256", headers) + self.assertEqual(token, rewrite_token) + self.assertEqual(rewritten, bytes_rewritten) + self.assertEqual(size, object_size) + + expected_path = "/b/name/o/%s/rewriteTo/b/%s/o/%s" % ( + source_name, + other_bucket_name, + dest_name, + ) + expected_query_params = {} + expected_data = {} + expected_headers = {} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + headers=expected_headers, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + _target_object=dest_blob, + ) def test_rewrite_same_name_no_old_key_new_key_done_w_user_project(self): - KEY = b"01234567890123456789012345678901" # 32 bytes - KEY_B64 = base64.b64encode(KEY).rstrip().decode("ascii") - KEY_HASH = hashlib.sha256(KEY).digest() - KEY_HASH_B64 = base64.b64encode(KEY_HASH).rstrip().decode("ascii") - BLOB_NAME = "blob" - USER_PROJECT = "user-project-123" - RESPONSE = { - "totalBytesRewritten": 42, - "objectSize": 42, + blob_name = "blob" + user_project = "user-project-123" + key = b"01234567890123456789012345678901" # 32 bytes + key_b64 = base64.b64encode(key).rstrip().decode("ascii") + key_hash = hashlib.sha256(key).digest() + key_hash_b64 = base64.b64encode(key_hash).rstrip().decode("ascii") + bytes_rewritten = object_size = 52 + api_response = { + "totalBytesRewritten": bytes_rewritten, + "objectSize": object_size, "done": True, "resource": {"etag": "DEADBEEF"}, } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) - bucket = _Bucket(client=client, user_project=USER_PROJECT) - plain = self._make_one(BLOB_NAME, bucket=bucket) - encrypted = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=KEY) + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response + bucket = _Bucket(client=client, user_project=user_project) + plain = self._make_one(blob_name, bucket=bucket) + encrypted = self._make_one(blob_name, bucket=bucket, encryption_key=key) token, rewritten, size = encrypted.rewrite(plain) self.assertIsNone(token) - self.assertEqual(rewritten, 42) - self.assertEqual(size, 42) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "POST") - PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) - self.assertEqual(kw[0]["path"], PATH) - self.assertEqual(kw[0]["query_params"], {"userProject": USER_PROJECT}) - SENT = {} - self.assertEqual(kw[0]["data"], SENT) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - - headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} - self.assertNotIn("X-Goog-Copy-Source-Encryption-Algorithm", headers) - self.assertNotIn("X-Goog-Copy-Source-Encryption-Key", headers) - self.assertNotIn("X-Goog-Copy-Source-Encryption-Key-Sha256", headers) - self.assertEqual(headers["X-Goog-Encryption-Algorithm"], "AES256") - self.assertEqual(headers["X-Goog-Encryption-Key"], KEY_B64) - self.assertEqual(headers["X-Goog-Encryption-Key-Sha256"], KEY_HASH_B64) + self.assertEqual(rewritten, bytes_rewritten) + self.assertEqual(size, object_size) + + expected_path = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (blob_name, blob_name) + expected_query_params = {"userProject": user_project} + expected_data = {} + expected_headers = { + "X-Goog-Encryption-Algorithm": "AES256", + "X-Goog-Encryption-Key": key_b64, + "X-Goog-Encryption-Key-Sha256": key_hash_b64, + } + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + headers=expected_headers, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + _target_object=encrypted, + ) def test_rewrite_same_name_no_key_new_key_w_token(self): - SOURCE_KEY = b"01234567890123456789012345678901" # 32 bytes - SOURCE_KEY_B64 = base64.b64encode(SOURCE_KEY).rstrip().decode("ascii") - SOURCE_KEY_HASH = hashlib.sha256(SOURCE_KEY).digest() - SOURCE_KEY_HASH_B64 = base64.b64encode(SOURCE_KEY_HASH).rstrip().decode("ascii") - DEST_KEY = b"90123456789012345678901234567890" # 32 bytes - DEST_KEY_B64 = base64.b64encode(DEST_KEY).rstrip().decode("ascii") - DEST_KEY_HASH = hashlib.sha256(DEST_KEY).digest() - DEST_KEY_HASH_B64 = base64.b64encode(DEST_KEY_HASH).rstrip().decode("ascii") - BLOB_NAME = "blob" - TOKEN = "TOKEN" - RESPONSE = { - "totalBytesRewritten": 42, - "objectSize": 42, + blob_name = "blob" + source_key = b"01234567890123456789012345678901" # 32 bytes + source_key_b64 = base64.b64encode(source_key).rstrip().decode("ascii") + source_key_hash = hashlib.sha256(source_key).digest() + source_key_hash_b64 = base64.b64encode(source_key_hash).rstrip().decode("ascii") + dest_key = b"90123456789012345678901234567890" # 32 bytes + dest_key_b64 = base64.b64encode(dest_key).rstrip().decode("ascii") + dest_key_hash = hashlib.sha256(dest_key).digest() + dest_key_hash_b64 = base64.b64encode(dest_key_hash).rstrip().decode("ascii") + previous_token = "TOKEN" + bytes_rewritten = object_size = 52 + api_response = { + "totalBytesRewritten": bytes_rewritten, + "objectSize": object_size, "done": True, "resource": {"etag": "DEADBEEF"}, } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response bucket = _Bucket(client=client) - source = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=SOURCE_KEY) - dest = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=DEST_KEY) + source = self._make_one(blob_name, bucket=bucket, encryption_key=source_key) + dest = self._make_one(blob_name, bucket=bucket, encryption_key=dest_key) - token, rewritten, size = dest.rewrite(source, token=TOKEN) + token, rewritten, size = dest.rewrite(source, token=previous_token) self.assertIsNone(token) - self.assertEqual(rewritten, 42) - self.assertEqual(size, 42) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "POST") - PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) - self.assertEqual(kw[0]["path"], PATH) - self.assertEqual(kw[0]["query_params"], {"rewriteToken": TOKEN}) - SENT = {} - self.assertEqual(kw[0]["data"], SENT) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - - headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} - self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Algorithm"], "AES256") - self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Key"], SOURCE_KEY_B64) - self.assertEqual( - headers["X-Goog-Copy-Source-Encryption-Key-Sha256"], SOURCE_KEY_HASH_B64 + self.assertEqual(rewritten, bytes_rewritten) + self.assertEqual(size, object_size) + + expected_path = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (blob_name, blob_name) + expected_data = {} + expected_query_params = {"rewriteToken": previous_token} + expected_headers = { + "X-Goog-Copy-Source-Encryption-Algorithm": "AES256", + "X-Goog-Copy-Source-Encryption-Key": source_key_b64, + "X-Goog-Copy-Source-Encryption-Key-Sha256": source_key_hash_b64, + "X-Goog-Encryption-Algorithm": "AES256", + "X-Goog-Encryption-Key": dest_key_b64, + "X-Goog-Encryption-Key-Sha256": dest_key_hash_b64, + } + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + headers=expected_headers, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + _target_object=dest, ) - self.assertEqual(headers["X-Goog-Encryption-Algorithm"], "AES256") - self.assertEqual(headers["X-Goog-Encryption-Key"], DEST_KEY_B64) - self.assertEqual(headers["X-Goog-Encryption-Key-Sha256"], DEST_KEY_HASH_B64) def test_rewrite_same_name_w_old_key_new_kms_key(self): - SOURCE_KEY = b"01234567890123456789012345678901" # 32 bytes - SOURCE_KEY_B64 = base64.b64encode(SOURCE_KEY).rstrip().decode("ascii") - SOURCE_KEY_HASH = hashlib.sha256(SOURCE_KEY).digest() - SOURCE_KEY_HASH_B64 = base64.b64encode(SOURCE_KEY_HASH).rstrip().decode("ascii") - DEST_KMS_RESOURCE = ( + blob_name = "blob" + source_key = b"01234567890123456789012345678901" # 32 bytes + source_key_b64 = base64.b64encode(source_key).rstrip().decode("ascii") + source_key_hash = hashlib.sha256(source_key).digest() + source_key_hash_b64 = base64.b64encode(source_key_hash).rstrip().decode("ascii") + dest_kms_resource = ( "projects/test-project-123/" "locations/us/" "keyRings/test-ring/" "cryptoKeys/test-key" ) - BLOB_NAME = "blob" - RESPONSE = { - "totalBytesRewritten": 42, - "objectSize": 42, + bytes_rewritten = object_size = 42 + api_response = { + "totalBytesRewritten": bytes_rewritten, + "objectSize": object_size, "done": True, "resource": {"etag": "DEADBEEF"}, } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response bucket = _Bucket(client=client) - source = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=SOURCE_KEY) - dest = self._make_one(BLOB_NAME, bucket=bucket, kms_key_name=DEST_KMS_RESOURCE) + source = self._make_one(blob_name, bucket=bucket, encryption_key=source_key) + dest = self._make_one(blob_name, bucket=bucket, kms_key_name=dest_kms_resource) token, rewritten, size = dest.rewrite(source) self.assertIsNone(token) - self.assertEqual(rewritten, 42) - self.assertEqual(size, 42) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "POST") - PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) - self.assertEqual(kw[0]["path"], PATH) - self.assertEqual( - kw[0]["query_params"], {"destinationKmsKeyName": DEST_KMS_RESOURCE} - ) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - SENT = {"kmsKeyName": DEST_KMS_RESOURCE} - self.assertEqual(kw[0]["data"], SENT) + self.assertEqual(rewritten, bytes_rewritten) + self.assertEqual(size, object_size) - headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} - self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Algorithm"], "AES256") - self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Key"], SOURCE_KEY_B64) - self.assertEqual( - headers["X-Goog-Copy-Source-Encryption-Key-Sha256"], SOURCE_KEY_HASH_B64 + expected_path = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (blob_name, blob_name) + expected_data = {"kmsKeyName": dest_kms_resource} + expected_query_params = {"destinationKmsKeyName": dest_kms_resource} + expected_headers = { + "X-Goog-Copy-Source-Encryption-Algorithm": "AES256", + "X-Goog-Copy-Source-Encryption-Key": source_key_b64, + "X-Goog-Copy-Source-Encryption-Key-Sha256": source_key_hash_b64, + } + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + headers=expected_headers, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + _target_object=dest, ) def test_update_storage_class_invalid(self): - BLOB_NAME = "blob-name" + blob_name = "blob-name" bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) + blob = self._make_one(blob_name, bucket=bucket) + blob.rewrite = mock.Mock(spec=[]) + with self.assertRaises(ValueError): blob.update_storage_class(u"BOGUS") - def test_update_storage_class_large_file(self): - BLOB_NAME = "blob-name" - STORAGE_CLASS = u"NEARLINE" - TOKEN = "TOKEN" - INCOMPLETE_RESPONSE = { - "totalBytesRewritten": 42, - "objectSize": 84, - "done": False, - "rewriteToken": TOKEN, - "resource": {"storageClass": STORAGE_CLASS}, - } - COMPLETE_RESPONSE = { - "totalBytesRewritten": 84, - "objectSize": 84, - "done": True, - "resource": {"storageClass": STORAGE_CLASS}, - } - response_1 = ({"status": http_client.OK}, INCOMPLETE_RESPONSE) - response_2 = ({"status": http_client.OK}, COMPLETE_RESPONSE) - connection = _Connection(response_1, response_2) - client = _Client(connection) + blob.rewrite.assert_not_called() + + def _update_storage_class_multi_pass_helper(self, **kw): + blob_name = "blob-name" + storage_class = u"NEARLINE" + rewrite_token = "TOKEN" + bytes_rewritten = 42 + object_size = 84 + client = mock.Mock(spec=[]) bucket = _Bucket(client=client) - blob = self._make_one(BLOB_NAME, bucket=bucket) + blob = self._make_one(blob_name, bucket=bucket) + blob.rewrite = mock.Mock(spec=[]) + blob.rewrite.side_effect = [ + (rewrite_token, bytes_rewritten, object_size), + (None, object_size, object_size), + ] - blob.update_storage_class("NEARLINE") + expected_i_g_m = kw.get("if_generation_match") + expected_i_g_n_m = kw.get("if_generation_not_match") + expected_i_m_m = kw.get("if_metageneration_match") + expected_i_m_n_m = kw.get("if_metageneration_not_match") + expected_i_s_g_m = kw.get("if_source_generation_match") + expected_i_s_g_n_m = kw.get("if_source_generation_not_match") + expected_i_s_m_m = kw.get("if_source_metageneration_match") + expected_i_s_m_n_m = kw.get("if_source_metageneration_not_match") + expected_timeout = kw.get("timeout", self._get_default_timeout()) + expected_retry = kw.get("retry", DEFAULT_RETRY_IF_GENERATION_SPECIFIED) - self.assertEqual(blob.storage_class, "NEARLINE") + blob.update_storage_class(storage_class, **kw) - def test_update_storage_class_with_custom_timeout(self): - BLOB_NAME = "blob-name" - STORAGE_CLASS = u"NEARLINE" - TOKEN = "TOKEN" - INCOMPLETE_RESPONSE = { - "totalBytesRewritten": 42, - "objectSize": 84, - "done": False, - "rewriteToken": TOKEN, - "resource": {"storageClass": STORAGE_CLASS}, - } - COMPLETE_RESPONSE = { - "totalBytesRewritten": 84, - "objectSize": 84, - "done": True, - "resource": {"storageClass": STORAGE_CLASS}, - } - response_1 = ({"status": http_client.OK}, INCOMPLETE_RESPONSE) - response_2 = ({"status": http_client.OK}, COMPLETE_RESPONSE) - connection = _Connection(response_1, response_2) - client = _Client(connection) - bucket = _Bucket(client=client) - blob = self._make_one(BLOB_NAME, bucket=bucket) + self.assertEqual(blob.storage_class, storage_class) - blob.update_storage_class("NEARLINE", timeout=9.58) + call1 = mock.call( + blob, + if_generation_match=expected_i_g_m, + if_generation_not_match=expected_i_g_n_m, + if_metageneration_match=expected_i_m_m, + if_metageneration_not_match=expected_i_m_n_m, + if_source_generation_match=expected_i_s_g_m, + if_source_generation_not_match=expected_i_s_g_n_m, + if_source_metageneration_match=expected_i_s_m_m, + if_source_metageneration_not_match=expected_i_s_m_n_m, + timeout=expected_timeout, + retry=expected_retry, + ) + call2 = mock.call( + blob, + token=rewrite_token, + if_generation_match=expected_i_g_m, + if_generation_not_match=expected_i_g_n_m, + if_metageneration_match=expected_i_m_m, + if_metageneration_not_match=expected_i_m_n_m, + if_source_generation_match=expected_i_s_g_m, + if_source_generation_not_match=expected_i_s_g_n_m, + if_source_metageneration_match=expected_i_s_m_m, + if_source_metageneration_not_match=expected_i_s_m_n_m, + timeout=expected_timeout, + retry=expected_retry, + ) + blob.rewrite.assert_has_calls([call1, call2]) - self.assertEqual(blob.storage_class, "NEARLINE") + def test_update_storage_class_multi_pass_w_defaults(self): + self._update_storage_class_multi_pass_helper() - kw = connection._requested - self.assertEqual(len(kw), 2) + def test_update_storage_class_multi_pass_w_i_g_m(self): + generation = 16 + self._update_storage_class_multi_pass_helper(if_generation_match=generation) - for kw_item in kw: - self.assertIn("timeout", kw_item) - self.assertEqual(kw_item["timeout"], 9.58) + def test_update_storage_class_multi_pass_w_i_g_n_m(self): + generation = 16 + self._update_storage_class_multi_pass_helper(if_generation_not_match=generation) - def test_update_storage_class_wo_encryption_key(self): - BLOB_NAME = "blob-name" - STORAGE_CLASS = u"NEARLINE" - RESPONSE = { - "totalBytesRewritten": 42, - "objectSize": 42, - "done": True, - "resource": {"storageClass": STORAGE_CLASS}, - } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) + def test_update_storage_class_multi_pass_w_i_m_m(self): + metageneration = 16 + self._update_storage_class_multi_pass_helper( + if_metageneration_match=metageneration, + ) + + def test_update_storage_class_multi_pass_w_i_m_n_m(self): + metageneration = 16 + self._update_storage_class_multi_pass_helper( + if_metageneration_not_match=metageneration, + ) + + def test_update_storage_class_multi_pass_w_i_s_g_m(self): + generation = 16 + self._update_storage_class_multi_pass_helper( + if_source_generation_match=generation + ) + + def test_update_storage_class_multi_pass_w_i_s_g_n_m(self): + generation = 16 + self._update_storage_class_multi_pass_helper( + if_source_generation_not_match=generation + ) + + def test_update_storage_class_multi_pass_w_i_s_m_m(self): + metageneration = 16 + self._update_storage_class_multi_pass_helper( + if_source_metageneration_match=metageneration, + ) + + def test_update_storage_class_multi_pass_w_i_s_m_n_m(self): + metageneration = 16 + self._update_storage_class_multi_pass_helper( + if_source_metageneration_not_match=metageneration, + ) + + def test_update_storage_class_multi_pass_w_timeout(self): + timeout = 42 + self._update_storage_class_multi_pass_helper(timeout=timeout) + + def test_update_storage_class_multi_pass_w_retry(self): + retry = mock.Mock(spec=[]) + self._update_storage_class_multi_pass_helper(retry=retry) + + def _update_storage_class_single_pass_helper(self, **kw): + blob_name = "blob-name" + storage_class = u"NEARLINE" + object_size = 84 + client = mock.Mock(spec=[]) bucket = _Bucket(client=client) - blob = self._make_one(BLOB_NAME, bucket=bucket) + blob = self._make_one(blob_name, bucket=bucket) + blob.rewrite = mock.Mock(spec=[]) + blob.rewrite.return_value = (None, object_size, object_size) + + expected_i_g_m = kw.get("if_generation_match") + expected_i_g_n_m = kw.get("if_generation_not_match") + expected_i_m_m = kw.get("if_metageneration_match") + expected_i_m_n_m = kw.get("if_metageneration_not_match") + expected_i_s_g_m = kw.get("if_source_generation_match") + expected_i_s_g_n_m = kw.get("if_source_generation_not_match") + expected_i_s_m_m = kw.get("if_source_metageneration_match") + expected_i_s_m_n_m = kw.get("if_source_metageneration_not_match") + expected_timeout = kw.get("timeout", self._get_default_timeout()) + expected_retry = kw.get("retry", DEFAULT_RETRY_IF_GENERATION_SPECIFIED) + + blob.update_storage_class(storage_class, **kw) - blob.update_storage_class("NEARLINE") + self.assertEqual(blob.storage_class, storage_class) - self.assertEqual(blob.storage_class, "NEARLINE") + blob.rewrite.assert_called_once_with( + blob, + if_generation_match=expected_i_g_m, + if_generation_not_match=expected_i_g_n_m, + if_metageneration_match=expected_i_m_m, + if_metageneration_not_match=expected_i_m_n_m, + if_source_generation_match=expected_i_s_g_m, + if_source_generation_not_match=expected_i_s_g_n_m, + if_source_metageneration_match=expected_i_s_m_m, + if_source_metageneration_not_match=expected_i_s_m_n_m, + timeout=expected_timeout, + retry=expected_retry, + ) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "POST") - PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) - self.assertEqual(kw[0]["path"], PATH) - self.assertEqual(kw[0]["query_params"], {}) - SENT = {"storageClass": STORAGE_CLASS} - self.assertEqual(kw[0]["data"], SENT) - - headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} - # Blob has no key, and therefore the relevant headers are not sent. - self.assertNotIn("X-Goog-Copy-Source-Encryption-Algorithm", headers) - self.assertNotIn("X-Goog-Copy-Source-Encryption-Key", headers) - self.assertNotIn("X-Goog-Copy-Source-Encryption-Key-Sha256", headers) - self.assertNotIn("X-Goog-Encryption-Algorithm", headers) - self.assertNotIn("X-Goog-Encryption-Key", headers) - self.assertNotIn("X-Goog-Encryption-Key-Sha256", headers) - - def test_update_storage_class_w_encryption_key_w_user_project(self): - BLOB_NAME = "blob-name" - BLOB_KEY = b"01234567890123456789012345678901" # 32 bytes - BLOB_KEY_B64 = base64.b64encode(BLOB_KEY).rstrip().decode("ascii") - BLOB_KEY_HASH = hashlib.sha256(BLOB_KEY).digest() - BLOB_KEY_HASH_B64 = base64.b64encode(BLOB_KEY_HASH).rstrip().decode("ascii") - STORAGE_CLASS = u"NEARLINE" - USER_PROJECT = "user-project-123" - RESPONSE = { - "totalBytesRewritten": 42, - "objectSize": 42, - "done": True, - "resource": {"storageClass": STORAGE_CLASS}, - } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) - bucket = _Bucket(client=client, user_project=USER_PROJECT) - blob = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=BLOB_KEY) + def test_update_storage_class_single_pass_w_defaults(self): + self._update_storage_class_single_pass_helper() - blob.update_storage_class("NEARLINE") + def test_update_storage_class_single_pass_w_i_g_m(self): + generation = 16 + self._update_storage_class_single_pass_helper(if_generation_match=generation) - self.assertEqual(blob.storage_class, "NEARLINE") + def test_update_storage_class_single_pass_w_i_g_n_m(self): + generation = 16 + self._update_storage_class_single_pass_helper( + if_generation_not_match=generation + ) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "POST") - PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) - self.assertEqual(kw[0]["path"], PATH) - self.assertEqual(kw[0]["query_params"], {"userProject": USER_PROJECT}) - SENT = {"storageClass": STORAGE_CLASS} - self.assertEqual(kw[0]["data"], SENT) - - headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} - # Blob has key, and therefore the relevant headers are sent. - self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Algorithm"], "AES256") - self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Key"], BLOB_KEY_B64) - self.assertEqual( - headers["X-Goog-Copy-Source-Encryption-Key-Sha256"], BLOB_KEY_HASH_B64 + def test_update_storage_class_single_pass_w_i_m_m(self): + metageneration = 16 + self._update_storage_class_single_pass_helper( + if_metageneration_match=metageneration, ) - self.assertEqual(headers["X-Goog-Encryption-Algorithm"], "AES256") - self.assertEqual(headers["X-Goog-Encryption-Key"], BLOB_KEY_B64) - self.assertEqual(headers["X-Goog-Encryption-Key-Sha256"], BLOB_KEY_HASH_B64) - def test_update_storage_class_w_generation_match(self): - BLOB_NAME = "blob-name" - STORAGE_CLASS = u"NEARLINE" - GENERATION_NUMBER = 6 - SOURCE_GENERATION_NUMBER = 9 - RESPONSE = { - "totalBytesRewritten": 42, - "objectSize": 42, - "done": True, - "resource": {"storageClass": STORAGE_CLASS}, - } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) - bucket = _Bucket(client=client) - blob = self._make_one(BLOB_NAME, bucket=bucket) + def test_update_storage_class_single_pass_w_i_m_n_m(self): + metageneration = 16 + self._update_storage_class_single_pass_helper( + if_metageneration_not_match=metageneration, + ) - blob.update_storage_class( - "NEARLINE", - if_generation_match=GENERATION_NUMBER, - if_source_generation_match=SOURCE_GENERATION_NUMBER, + def test_update_storage_class_single_pass_w_i_s_g_m(self): + generation = 16 + self._update_storage_class_single_pass_helper( + if_source_generation_match=generation ) - self.assertEqual(blob.storage_class, "NEARLINE") + def test_update_storage_class_single_pass_w_i_s_g_n_m(self): + generation = 16 + self._update_storage_class_single_pass_helper( + if_source_generation_not_match=generation + ) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "POST") - PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) - self.assertEqual(kw[0]["path"], PATH) - self.assertEqual( - kw[0]["query_params"], - { - "ifGenerationMatch": GENERATION_NUMBER, - "ifSourceGenerationMatch": SOURCE_GENERATION_NUMBER, - }, + def test_update_storage_class_single_pass_w_i_s_m_m(self): + metageneration = 16 + self._update_storage_class_single_pass_helper( + if_source_metageneration_match=metageneration, ) - SENT = {"storageClass": STORAGE_CLASS} - self.assertEqual(kw[0]["data"], SENT) + + def test_update_storage_class_single_pass_w_i_s_m_n_m(self): + metageneration = 16 + self._update_storage_class_single_pass_helper( + if_source_metageneration_not_match=metageneration, + ) + + def test_update_storage_class_single_pass_w_timeout(self): + timeout = 42 + self._update_storage_class_single_pass_helper(timeout=timeout) + + def test_update_storage_class_single_pass_w_retry(self): + retry = mock.Mock(spec=[]) + self._update_storage_class_single_pass_helper(retry=retry) def test_cache_control_getter(self): BLOB_NAME = "blob-name" @@ -4956,19 +5017,8 @@ class _Connection(object): USER_AGENT = "testing 1.2.3" credentials = object() - def __init__(self, *responses): - self._responses = responses[:] - self._requested = [] - self._signed = [] - - def _respond(self, **kw): - self._requested.append(kw) - response, self._responses = self._responses[0], self._responses[1:] - return response - - def api_request(self, **kw): - info, content = self._respond(**kw) - return content + def __init__(self): + pass class _Bucket(object): @@ -5016,10 +5066,6 @@ class _Client(object): def __init__(self, connection): self._base_connection = connection - @property - def _connection(self): - return self._base_connection - @property def _credentials(self): return self._base_connection.credentials diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index 410c9d9b6..50fe02c0e 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -24,15 +24,6 @@ from google.cloud.storage.retry import DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED -def _make_connection(*responses): - import google.cloud.storage._http - - mock_connection = mock.create_autospec(google.cloud.storage._http.Connection) - mock_connection.user_agent = "testing 1.2.3" - mock_connection.api_request.side_effect = list(responses) - return mock_connection - - def _create_signing_credentials(): import google.auth.credentials @@ -1651,111 +1642,124 @@ def _make_blob(bucket_name, blob_name): return blob def test_copy_blobs_wo_name(self): - SOURCE = "source" - DEST = "dest" - BLOB_NAME = "blob-name" - connection = _Connection({}) - client = _Client(connection) - source = self._make_one(client=client, name=SOURCE) - dest = self._make_one(client=client, name=DEST) - blob = self._make_blob(SOURCE, BLOB_NAME) + source_name = "source" + dest_name = "dest" + blob_name = "blob-name" + api_response = {} + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response + source = self._make_one(client=client, name=source_name) + dest = self._make_one(client=client, name=dest_name) + blob = self._make_blob(source_name, blob_name) - new_blob = source.copy_blob(blob, dest, timeout=42) + new_blob = source.copy_blob(blob, dest) self.assertIs(new_blob.bucket, dest) - self.assertEqual(new_blob.name, BLOB_NAME) + self.assertEqual(new_blob.name, blob_name) - (kw,) = connection._requested - COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( - SOURCE, BLOB_NAME, DEST, BLOB_NAME + expected_path = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( + source_name, blob_name, dest_name, blob_name + ) + expected_data = None + expected_query_params = {} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + _target_object=new_blob, ) - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], COPY_PATH) - self.assertEqual(kw["query_params"], {}) - self.assertEqual(kw["timeout"], 42) - self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) - - def test_copy_blobs_source_generation(self): - SOURCE = "source" - DEST = "dest" - BLOB_NAME = "blob-name" - GENERATION = 1512565576797178 - connection = _Connection({}) - client = _Client(connection) - source = self._make_one(client=client, name=SOURCE) - dest = self._make_one(client=client, name=DEST) - blob = self._make_blob(SOURCE, BLOB_NAME) + def test_copy_blob_w_source_generation_w_timeout(self): + source_name = "source" + dest_name = "dest" + blob_name = "blob-name" + generation = 1512565576797178 + api_response = {} + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response + source = self._make_one(client=client, name=source_name) + dest = self._make_one(client=client, name=dest_name) + blob = self._make_blob(source_name, blob_name) + timeout = 42 - new_blob = source.copy_blob(blob, dest, source_generation=GENERATION) + new_blob = source.copy_blob( + blob, dest, source_generation=generation, timeout=timeout, + ) self.assertIs(new_blob.bucket, dest) - self.assertEqual(new_blob.name, BLOB_NAME) + self.assertEqual(new_blob.name, blob_name) - (kw,) = connection._requested - COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( - SOURCE, BLOB_NAME, DEST, BLOB_NAME + expected_path = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( + source_name, blob_name, dest_name, blob_name + ) + expected_data = None + expected_query_params = {"sourceGeneration": generation} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=timeout, + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + _target_object=new_blob, ) - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], COPY_PATH) - self.assertEqual(kw["query_params"], {"sourceGeneration": GENERATION}) - self.assertEqual(kw["timeout"], self._get_default_timeout()) - self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) - - def test_copy_blobs_w_generation_match(self): - SOURCE = "source" - DEST = "dest" - BLOB_NAME = "blob-name" - GENERATION_NUMBER = 6 - SOURCE_GENERATION_NUMBER = 9 - connection = _Connection({}) - client = _Client(connection) - source = self._make_one(client=client, name=SOURCE) - dest = self._make_one(client=client, name=DEST) - blob = self._make_blob(SOURCE, BLOB_NAME) + def test_copy_blob_w_generation_match_w_retry(self): + source_name = "source" + dest_name = "dest" + blob_name = "blob-name" + generation_number = 6 + source_generation_number = 9 + api_response = {} + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response + source = self._make_one(client=client, name=source_name) + dest = self._make_one(client=client, name=dest_name) + blob = self._make_blob(source_name, blob_name) + retry = mock.Mock(spec=[]) new_blob = source.copy_blob( blob, dest, - if_generation_match=GENERATION_NUMBER, - if_source_generation_match=SOURCE_GENERATION_NUMBER, + if_generation_match=generation_number, + if_source_generation_match=source_generation_number, + retry=retry, ) self.assertIs(new_blob.bucket, dest) - self.assertEqual(new_blob.name, BLOB_NAME) + self.assertEqual(new_blob.name, blob_name) - (kw,) = connection._requested - COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( - SOURCE, BLOB_NAME, DEST, BLOB_NAME + expected_path = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( + source_name, blob_name, dest_name, blob_name ) - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], COPY_PATH) - self.assertEqual( - kw["query_params"], - { - "ifGenerationMatch": GENERATION_NUMBER, - "ifSourceGenerationMatch": SOURCE_GENERATION_NUMBER, - }, + expected_data = None + expected_query_params = { + "ifGenerationMatch": generation_number, + "ifSourceGenerationMatch": source_generation_number, + } + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=self._get_default_timeout(), + retry=retry, + _target_object=new_blob, ) - self.assertEqual(kw["timeout"], self._get_default_timeout()) - self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) - def test_copy_blobs_preserve_acl(self): + def test_copy_blob_w_preserve_acl_false_w_explicit_client(self): from google.cloud.storage.acl import ObjectACL source_name = "source" dest_name = "dest" blob_name = "blob-name" new_name = "new_name" - - connection = _Connection({}, {}) - client = _Client(connection) - - # Temporary, until we get a real client in place. - client._patch_resource = mock.Mock(return_value={}) - - source = self._make_one(client=client, name=source_name) - dest = self._make_one(client=client, name=dest_name) + post_api_response = {} + patch_api_response = {} + client = mock.Mock(spec=["_post_resource", "_patch_resource"]) + client._post_resource.return_value = post_api_response + client._patch_resource.return_value = patch_api_response + source = self._make_one(client=None, name=source_name) + dest = self._make_one(client=None, name=dest_name) blob = self._make_blob(source_name, blob_name) new_blob = source.copy_blob( @@ -1766,170 +1770,159 @@ def test_copy_blobs_preserve_acl(self): self.assertEqual(new_blob.name, new_name) self.assertIsInstance(new_blob.acl, ObjectACL) - (kw1,) = connection._requested - copy_path = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( + expected_copy_path = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( source_name, blob_name, dest_name, new_name ) - self.assertEqual(kw1["method"], "POST") - self.assertEqual(kw1["path"], copy_path) - self.assertEqual(kw1["query_params"], {}) - self.assertEqual(kw1["timeout"], self._get_default_timeout()) - self.assertEqual(kw1["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) + expected_copy_data = None + expected_copy_query_params = {} + client._post_resource.assert_called_once_with( + expected_copy_path, + expected_copy_data, + query_params=expected_copy_query_params, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + _target_object=new_blob, + ) expected_patch_path = "/b/{}/o/{}".format(dest_name, new_name) - expected_data = {"acl": []} - expected_query_params = {"projection": "full"} + expected_patch_data = {"acl": []} + expected_patch_query_params = {"projection": "full"} client._patch_resource.assert_called_once_with( expected_patch_path, - expected_data, - query_params=expected_query_params, + expected_patch_data, + query_params=expected_patch_query_params, timeout=self._get_default_timeout(), retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, ) - def test_copy_blobs_w_name_and_user_project(self): - SOURCE = "source" - DEST = "dest" - BLOB_NAME = "blob-name" - NEW_NAME = "new_name" - USER_PROJECT = "user-project-123" - connection = _Connection({}) - client = _Client(connection) - source = self._make_one(client=client, name=SOURCE, user_project=USER_PROJECT) - dest = self._make_one(client=client, name=DEST) - blob = self._make_blob(SOURCE, BLOB_NAME) + def test_copy_blob_w_name_and_user_project(self): + source_name = "source" + dest_name = "dest" + blob_name = "blob-name" + new_name = "new_name" + user_project = "user-project-123" + api_response = {} + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response + source = self._make_one( + client=client, name=source_name, user_project=user_project + ) + dest = self._make_one(client=client, name=dest_name) + blob = self._make_blob(source_name, blob_name) - new_blob = source.copy_blob(blob, dest, NEW_NAME) + new_blob = source.copy_blob(blob, dest, new_name) self.assertIs(new_blob.bucket, dest) - self.assertEqual(new_blob.name, NEW_NAME) + self.assertEqual(new_blob.name, new_name) - COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( - SOURCE, BLOB_NAME, DEST, NEW_NAME + expected_path = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( + source_name, blob_name, dest_name, new_name ) - (kw,) = connection._requested - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], COPY_PATH) - self.assertEqual(kw["query_params"], {"userProject": USER_PROJECT}) - self.assertEqual(kw["timeout"], self._get_default_timeout()) - self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) - - def test_rename_blob(self): - BUCKET_NAME = "BUCKET_NAME" - BLOB_NAME = "blob-name" - NEW_BLOB_NAME = "new-blob-name" - DATA = {"name": NEW_BLOB_NAME} - connection = _Connection(DATA) - client = _Client(connection) - bucket = self._make_one(client=client, name=BUCKET_NAME) - blob = self._make_blob(BUCKET_NAME, BLOB_NAME) - - renamed_blob = bucket.rename_blob( - blob, NEW_BLOB_NAME, client=client, timeout=42 + expected_data = None + expected_query_params = {"userProject": user_project} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + _target_object=new_blob, ) - self.assertIs(renamed_blob.bucket, bucket) - self.assertEqual(renamed_blob.name, NEW_BLOB_NAME) + def _rename_blob_helper(self, explicit_client=False, same_name=False, **kw): + bucket_name = "BUCKET_NAME" + blob_name = "blob-name" - COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( - BUCKET_NAME, BLOB_NAME, BUCKET_NAME, NEW_BLOB_NAME - ) - (kw,) = connection._requested - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], COPY_PATH) - self.assertEqual(kw["query_params"], {}) - self.assertEqual(kw["timeout"], 42) - self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) + if same_name: + new_blob_name = blob_name + else: + new_blob_name = "new-blob-name" - blob.delete.assert_called_once_with( - client=client, - timeout=42, - if_generation_match=None, - if_generation_not_match=None, - if_metageneration_match=None, - if_metageneration_not_match=None, - retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, - ) + client = mock.Mock(spec=[]) + kw = kw.copy() - def test_rename_blob_with_generation_match(self): - BUCKET_NAME = "BUCKET_NAME" - BLOB_NAME = "blob-name" - NEW_BLOB_NAME = "new-blob-name" - DATA = {"name": NEW_BLOB_NAME} - GENERATION_NUMBER = 6 - SOURCE_GENERATION_NUMBER = 7 - SOURCE_METAGENERATION_NUMBER = 9 - - connection = _Connection(DATA) - client = _Client(connection) - bucket = self._make_one(client=client, name=BUCKET_NAME) - blob = self._make_blob(BUCKET_NAME, BLOB_NAME) + if explicit_client: + bucket = self._make_one(client=None, name=bucket_name) + expected_client = kw["client"] = client + else: + bucket = self._make_one(client=client, name=bucket_name) + expected_client = None - renamed_blob = bucket.rename_blob( - blob, - NEW_BLOB_NAME, - client=client, - timeout=42, - if_generation_match=GENERATION_NUMBER, - if_source_generation_match=SOURCE_GENERATION_NUMBER, - if_source_metageneration_not_match=SOURCE_METAGENERATION_NUMBER, - ) + expected_i_g_m = kw.get("if_generation_match") + expected_i_g_n_m = kw.get("if_generation_not_match") + expected_i_m_m = kw.get("if_metageneration_match") + expected_i_m_n_m = kw.get("if_metageneration_not_match") + expected_i_s_g_m = kw.get("if_source_generation_match") + expected_i_s_g_n_m = kw.get("if_source_generation_not_match") + expected_i_s_m_m = kw.get("if_source_metageneration_match") + expected_i_s_m_n_m = kw.get("if_source_metageneration_not_match") + expected_timeout = kw.get("timeout", self._get_default_timeout()) + expected_retry = kw.get("retry", DEFAULT_RETRY_IF_GENERATION_SPECIFIED) - self.assertIs(renamed_blob.bucket, bucket) - self.assertEqual(renamed_blob.name, NEW_BLOB_NAME) + bucket.copy_blob = mock.Mock(spec=[]) + blob = self._make_blob(bucket_name, blob_name) - COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( - BUCKET_NAME, BLOB_NAME, BUCKET_NAME, NEW_BLOB_NAME - ) - (kw,) = connection._requested - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], COPY_PATH) - self.assertEqual( - kw["query_params"], - { - "ifGenerationMatch": GENERATION_NUMBER, - "ifSourceGenerationMatch": SOURCE_GENERATION_NUMBER, - "ifSourceMetagenerationNotMatch": SOURCE_METAGENERATION_NUMBER, - }, - ) - self.assertEqual(kw["timeout"], 42) - self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) + renamed_blob = bucket.rename_blob(blob, new_blob_name, **kw) - blob.delete.assert_called_once_with( - client=client, - timeout=42, - if_generation_match=SOURCE_GENERATION_NUMBER, - if_generation_not_match=None, - if_metageneration_match=None, - if_metageneration_not_match=SOURCE_METAGENERATION_NUMBER, - retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, - ) + self.assertIs(renamed_blob, bucket.copy_blob.return_value) - def test_rename_blob_to_itself(self): - BUCKET_NAME = "BUCKET_NAME" - BLOB_NAME = "blob-name" - DATA = {"name": BLOB_NAME} - connection = _Connection(DATA) - client = _Client(connection) - bucket = self._make_one(client=client, name=BUCKET_NAME) - blob = self._make_blob(BUCKET_NAME, BLOB_NAME) + bucket.copy_blob.assert_called_once_with( + blob, + bucket, + new_blob_name, + client=expected_client, + if_generation_match=expected_i_g_m, + if_generation_not_match=expected_i_g_n_m, + if_metageneration_match=expected_i_m_m, + if_metageneration_not_match=expected_i_m_n_m, + if_source_generation_match=expected_i_s_g_m, + if_source_generation_not_match=expected_i_s_g_n_m, + if_source_metageneration_match=expected_i_s_m_m, + if_source_metageneration_not_match=expected_i_s_m_n_m, + timeout=expected_timeout, + retry=expected_retry, + ) + + if same_name: + blob.delete.assert_not_called() + else: + blob.delete.assert_called_once_with( + client=expected_client, + if_generation_match=expected_i_s_g_m, + if_generation_not_match=expected_i_s_g_n_m, + if_metageneration_match=expected_i_s_m_m, + if_metageneration_not_match=expected_i_s_m_n_m, + timeout=expected_timeout, + retry=expected_retry, + ) - renamed_blob = bucket.rename_blob(blob, BLOB_NAME) + def test_rename_blob_w_defaults(self): + self._rename_blob_helper() - self.assertIs(renamed_blob.bucket, bucket) - self.assertEqual(renamed_blob.name, BLOB_NAME) + def test_rename_blob_w_explicit_client(self): + self._rename_blob_helper(explicit_client=True) + + def test_rename_blob_w_generation_match(self): + generation_number = 6 + source_generation_number = 7 + source_metageneration_number = 9 - COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( - BUCKET_NAME, BLOB_NAME, BUCKET_NAME, BLOB_NAME + self._rename_blob_helper( + if_generation_match=generation_number, + if_source_generation_match=source_generation_number, + if_source_metageneration_not_match=source_metageneration_number, ) - (kw,) = connection._requested - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], COPY_PATH) - self.assertEqual(kw["query_params"], {}) - self.assertEqual(kw["timeout"], self._get_default_timeout()) - self.assertEqual(kw["retry"], DEFAULT_RETRY_IF_GENERATION_SPECIFIED) - blob.delete.assert_not_called() + def test_rename_blob_w_timeout(self): + timeout = 42 + self._rename_blob_helper(timeout=timeout) + + def test_rename_blob_w_retry(self): + retry = mock.Mock(spec={}) + self._rename_blob_helper(retry=retry) + + def test_rename_blob_to_itself(self): + self._rename_blob_helper(same_name=True) def test_etag(self): ETAG = "ETAG" @@ -2543,23 +2536,22 @@ def test_versioning_enabled_getter(self): self.assertEqual(bucket.versioning_enabled, True) @mock.patch("warnings.warn") - def test_create_deprecated(self, mock_warn): - PROJECT = "PROJECT" - BUCKET_NAME = "bucket-name" - DATA = {"name": BUCKET_NAME} - connection = _make_connection(DATA) - client = self._make_client(project=PROJECT) - client._base_connection = connection + def test_create_w_defaults_deprecated(self, mock_warn): + bucket_name = "bucket-name" + api_response = {"name": bucket_name} + client = mock.Mock(spec=["create_bucket"]) + client.create_bucket.return_value = api_response + bucket = self._make_one(client=client, name=bucket_name) - bucket = self._make_one(client=client, name=BUCKET_NAME) bucket.create() - connection.api_request.assert_called_once_with( - method="POST", - path="/b", - query_params={"project": PROJECT}, - data=DATA, - _target_object=bucket, + client.create_bucket.assert_called_once_with( + bucket_or_name=bucket, + project=None, + user_project=None, + location=None, + predefined_acl=None, + predefined_default_object_acl=None, timeout=self._get_default_timeout(), retry=DEFAULT_RETRY, ) @@ -2572,26 +2564,40 @@ def test_create_deprecated(self, mock_warn): ) @mock.patch("warnings.warn") - def test_create_w_user_project(self, mock_warn): - PROJECT = "PROJECT" - BUCKET_NAME = "bucket-name" - DATA = {"name": BUCKET_NAME} - connection = _make_connection(DATA) - client = self._make_client(project=PROJECT) - client._base_connection = connection + def test_create_w_explicit_deprecated(self, mock_warn): + project = "PROJECT" + location = "eu" + user_project = "USER_PROJECT" + bucket_name = "bucket-name" + predefined_acl = "authenticatedRead" + predefined_default_object_acl = "bucketOwnerFullControl" + api_response = {"name": bucket_name} + client = mock.Mock(spec=["create_bucket"]) + client.create_bucket.return_value = api_response + bucket = self._make_one(client=None, name=bucket_name) + bucket._user_project = user_project + timeout = 42 + retry = mock.Mock(spec=[]) - bucket = self._make_one(client=client, name=BUCKET_NAME) - bucket._user_project = "USER_PROJECT" - bucket.create() + bucket.create( + client=client, + project=project, + location=location, + predefined_acl=predefined_acl, + predefined_default_object_acl=predefined_default_object_acl, + timeout=timeout, + retry=retry, + ) - connection.api_request.assert_called_once_with( - method="POST", - path="/b", - query_params={"project": PROJECT, "userProject": "USER_PROJECT"}, - data=DATA, - _target_object=bucket, - timeout=self._get_default_timeout(), - retry=DEFAULT_RETRY, + client.create_bucket.assert_called_once_with( + bucket_or_name=bucket, + project=project, + user_project=user_project, + location=location, + predefined_acl=predefined_acl, + predefined_default_object_acl=predefined_default_object_acl, + timeout=timeout, + retry=retry, ) mock_warn.assert_called_with( @@ -3483,10 +3489,7 @@ def test_generate_upload_policy_bad_credentials(self): bucket.generate_upload_policy([]) def test_lock_retention_policy_no_policy_set(self): - credentials = object() - connection = _Connection() - connection.credentials = credentials - client = _Client(connection) + client = mock.Mock(spec=["_post_resource"]) name = "name" bucket = self._make_one(client=client, name=name) bucket._properties["metageneration"] = 1234 @@ -3494,11 +3497,10 @@ def test_lock_retention_policy_no_policy_set(self): with self.assertRaises(ValueError): bucket.lock_retention_policy() + client._post_resource.assert_not_called() + def test_lock_retention_policy_no_metageneration(self): - credentials = object() - connection = _Connection() - connection.credentials = credentials - client = _Client(connection) + client = mock.Mock(spec=["_post_resource"]) name = "name" bucket = self._make_one(client=client, name=name) bucket._properties["retentionPolicy"] = { @@ -3509,11 +3511,10 @@ def test_lock_retention_policy_no_metageneration(self): with self.assertRaises(ValueError): bucket.lock_retention_policy() + client._post_resource.assert_not_called() + def test_lock_retention_policy_already_locked(self): - credentials = object() - connection = _Connection() - connection.credentials = credentials - client = _Client(connection) + client = mock.Mock(spec=["_post_resource"]) name = "name" bucket = self._make_one(client=client, name=name) bucket._properties["metageneration"] = 1234 @@ -3526,69 +3527,88 @@ def test_lock_retention_policy_already_locked(self): with self.assertRaises(ValueError): bucket.lock_retention_policy() - def test_lock_retention_policy_ok(self): + client._post_resource.assert_not_called() + + def test_lock_retention_policy_ok_w_timeout_w_retry(self): name = "name" - response = { + effective_time = "2018-03-01T16:46:27.123456Z" + one_hundred_days = 86400 * 100 # seconds in 100 days + metageneration = 1234 + api_response = { "name": name, - "metageneration": 1235, + "metageneration": metageneration + 1, "retentionPolicy": { - "effectiveTime": "2018-03-01T16:46:27.123456Z", + "effectiveTime": effective_time, "isLocked": True, - "retentionPeriod": 86400 * 100, # 100 days + "retentionPeriod": one_hundred_days, }, } - credentials = object() - connection = _Connection(response) - connection.credentials = credentials - client = _Client(connection) + metageneration = 1234 + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response bucket = self._make_one(client=client, name=name) - bucket._properties["metageneration"] = 1234 + bucket._properties["metageneration"] = metageneration bucket._properties["retentionPolicy"] = { - "effectiveTime": "2018-03-01T16:46:27.123456Z", - "retentionPeriod": 86400 * 100, # 100 days + "effectiveTime": effective_time, + "retentionPeriod": one_hundred_days, } + timeout = 42 + retry = mock.Mock(spec=[]) - bucket.lock_retention_policy(timeout=42) + bucket.lock_retention_policy(timeout=timeout, retry=retry) - (kw,) = connection._requested - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], "/b/{}/lockRetentionPolicy".format(name)) - self.assertEqual(kw["query_params"], {"ifMetagenerationMatch": 1234}) - self.assertEqual(kw["timeout"], 42) + expected_path = "/b/{}/lockRetentionPolicy".format(name) + expected_data = None + expected_query_params = {"ifMetagenerationMatch": metageneration} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=timeout, + retry=retry, + _target_object=bucket, + ) def test_lock_retention_policy_w_user_project(self): name = "name" user_project = "user-project-123" - response = { + metageneration = 1234 + effective_time = "2018-03-01T16:46:27.123456Z" + one_hundred_days = 86400 * 100 # seconds in 100 days + api_response = { "name": name, - "metageneration": 1235, + "metageneration": metageneration + 1, "retentionPolicy": { - "effectiveTime": "2018-03-01T16:46:27.123456Z", + "effectiveTime": effective_time, "isLocked": True, - "retentionPeriod": 86400 * 100, # 100 days + "retentionPeriod": one_hundred_days, }, } - credentials = object() - connection = _Connection(response) - connection.credentials = credentials - client = _Client(connection) + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response bucket = self._make_one(client=client, name=name, user_project=user_project) bucket._properties["metageneration"] = 1234 bucket._properties["retentionPolicy"] = { - "effectiveTime": "2018-03-01T16:46:27.123456Z", - "retentionPeriod": 86400 * 100, # 100 days + "effectiveTime": effective_time, + "retentionPeriod": one_hundred_days, } bucket.lock_retention_policy() - (kw,) = connection._requested - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], "/b/{}/lockRetentionPolicy".format(name)) - self.assertEqual( - kw["query_params"], - {"ifMetagenerationMatch": 1234, "userProject": user_project}, + expected_path = "/b/{}/lockRetentionPolicy".format(name) + expected_data = None + expected_query_params = { + "ifMetagenerationMatch": metageneration, + "userProject": user_project, + } + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY, + _target_object=bucket, ) - self.assertEqual(kw["timeout"], self._get_default_timeout()) def test_generate_signed_url_w_invalid_version(self): expiration = "2014-10-16T20:34:37.000Z" diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 6d34d935a..7a9c0e880 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -593,6 +593,69 @@ def test__put_resource_hit_w_explicit(self): _target_object=target, ) + def test__post_resource_miss_w_defaults(self): + from google.cloud.exceptions import NotFound + + project = "PROJECT" + path = "/path/to/something" + credentials = _make_credentials() + data = {"baz": "Baz"} + + client = self._make_one(project=project, credentials=credentials) + connection = client._base_connection = _make_connection() + + with self.assertRaises(NotFound): + client._post_resource(path, data) + + connection.api_request.assert_called_once_with( + method="POST", + path=path, + data=data, + query_params=None, + headers=None, + timeout=self._get_default_timeout(), + retry=None, + _target_object=None, + ) + + def test__post_resource_hit_w_explicit(self): + project = "PROJECT" + path = "/path/to/something" + data = {"baz": "Baz"} + query_params = {"foo": "Foo"} + headers = {"bar": "Bar"} + timeout = 100 + retry = mock.Mock(spec=[]) + credentials = _make_credentials() + + client = self._make_one(project=project, credentials=credentials) + expected = mock.Mock(spec={}) + connection = client._base_connection = _make_connection(expected) + target = mock.Mock(spec={}) + + found = client._post_resource( + path, + data, + query_params=query_params, + headers=headers, + timeout=timeout, + retry=retry, + _target_object=target, + ) + + self.assertIs(found, expected) + + connection.api_request.assert_called_once_with( + method="POST", + path=path, + data=data, + query_params=query_params, + headers=headers, + timeout=timeout, + retry=retry, + _target_object=target, + ) + def test__delete_resource_miss_w_defaults(self): from google.cloud.exceptions import NotFound @@ -955,7 +1018,7 @@ def test_create_bucket_w_missing_client_project(self): with self.assertRaises(ValueError): client.create_bucket("bucket") - def test_create_bucket_w_conflict(self): + def test_create_bucket_w_conflict_w_user_project(self): from google.cloud.exceptions import Conflict project = "PROJECT" @@ -963,62 +1026,60 @@ def test_create_bucket_w_conflict(self): other_project = "OTHER_PROJECT" credentials = _make_credentials() client = self._make_one(project=project, credentials=credentials) - connection = _make_connection() - client._base_connection = connection - connection.api_request.side_effect = Conflict("testing") + client._post_resource = mock.Mock() + client._post_resource.side_effect = Conflict("testing") bucket_name = "bucket-name" - data = {"name": bucket_name} with self.assertRaises(Conflict): client.create_bucket( bucket_name, project=other_project, user_project=user_project ) - connection.api_request.assert_called_once_with( - method="POST", - path="/b", - query_params={"project": other_project, "userProject": user_project}, - data=data, - _target_object=mock.ANY, + expected_path = "/b" + expected_data = {"name": bucket_name} + expected_query_params = { + "project": other_project, + "userProject": user_project, + } + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, timeout=self._get_default_timeout(), retry=DEFAULT_RETRY, + _target_object=mock.ANY, ) @mock.patch("warnings.warn") - def test_create_requester_pays_deprecated(self, mock_warn): + def test_create_bucket_w_requester_pays_deprecated(self, mock_warn): from google.cloud.storage.bucket import Bucket + bucket_name = "bucket-name" project = "PROJECT" credentials = _make_credentials() + api_respone = {"name": bucket_name, "billing": {"requesterPays": True}} client = self._make_one(project=project, credentials=credentials) - bucket_name = "bucket-name" - json_expected = {"name": bucket_name, "billing": {"requesterPays": True}} - http = _make_requests_session([_make_json_response(json_expected)]) - client._http_internal = http + client._post_resource = mock.Mock() + client._post_resource.return_value = api_respone bucket = client.create_bucket(bucket_name, requester_pays=True) self.assertIsInstance(bucket, Bucket) self.assertEqual(bucket.name, bucket_name) self.assertTrue(bucket.requester_pays) - http.request.assert_called_once_with( - method="POST", - url=mock.ANY, - data=mock.ANY, - headers=mock.ANY, - timeout=mock.ANY, - ) - _, kwargs = http.request.call_args - scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) - self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) - self.assertEqual( - path, "/".join(["", "storage", client._connection.API_VERSION, "b"]) + + expected_path = "/b" + expected_data = api_respone + expected_query_params = {"project": project} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY, + _target_object=mock.ANY, ) - parms = dict(urlparse.parse_qsl(qs)) - self.assertEqual(parms["project"], project) - json_sent = http.request.call_args_list[0][1]["data"] - self.assertEqual(json_expected, json.loads(json_sent)) mock_warn.assert_called_with( "requester_pays arg is deprecated. Use Bucket().requester_pays instead.", @@ -1031,31 +1092,40 @@ def test_create_bucket_w_predefined_acl_invalid(self): bucket_name = "bucket-name" credentials = _make_credentials() client = self._make_one(project=project, credentials=credentials) + client._post_resource = mock.Mock() with self.assertRaises(ValueError): client.create_bucket(bucket_name, predefined_acl="bogus") - def test_create_bucket_w_predefined_acl_valid(self): + client._post_resource.assert_not_called() + + def test_create_bucket_w_predefined_acl_valid_w_timeout(self): project = "PROJECT" bucket_name = "bucket-name" - data = {"name": bucket_name} - + api_response = {"name": bucket_name} credentials = _make_credentials() client = self._make_one(project=project, credentials=credentials) - connection = _make_connection(data) - client._base_connection = connection + client._post_resource = mock.Mock() + client._post_resource.return_value = api_response + timeout = 42 + bucket = client.create_bucket( - bucket_name, predefined_acl="publicRead", timeout=42 + bucket_name, predefined_acl="publicRead", timeout=timeout, ) - connection.api_request.assert_called_once_with( - method="POST", - path="/b", - query_params={"project": project, "predefinedAcl": "publicRead"}, - data=data, - _target_object=bucket, - timeout=42, + expected_path = "/b" + expected_data = api_response + expected_query_params = { + "project": project, + "predefinedAcl": "publicRead", + } + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=timeout, retry=DEFAULT_RETRY, + _target_object=bucket, ) def test_create_bucket_w_predefined_default_object_acl_invalid(self): @@ -1064,93 +1134,98 @@ def test_create_bucket_w_predefined_default_object_acl_invalid(self): credentials = _make_credentials() client = self._make_one(project=project, credentials=credentials) + client._post_resource = mock.Mock() with self.assertRaises(ValueError): client.create_bucket(bucket_name, predefined_default_object_acl="bogus") - def test_create_bucket_w_predefined_default_object_acl_valid(self): + client._post_resource.assert_not_called() + + def test_create_bucket_w_predefined_default_object_acl_valid_w_retry(self): project = "PROJECT" bucket_name = "bucket-name" - data = {"name": bucket_name} - + api_response = {"name": bucket_name} credentials = _make_credentials() client = self._make_one(project=project, credentials=credentials) - connection = _make_connection(data) - client._base_connection = connection + client._post_resource = mock.Mock() + client._post_resource.return_value = api_response + retry = mock.Mock(spec=[]) + bucket = client.create_bucket( - bucket_name, predefined_default_object_acl="publicRead" + bucket_name, predefined_default_object_acl="publicRead", retry=retry, ) - connection.api_request.assert_called_once_with( - method="POST", - path="/b", - query_params={ - "project": project, - "predefinedDefaultObjectAcl": "publicRead", - }, - data=data, - _target_object=bucket, + expected_path = "/b" + expected_data = api_response + expected_query_params = { + "project": project, + "predefinedDefaultObjectAcl": "publicRead", + } + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, timeout=self._get_default_timeout(), - retry=DEFAULT_RETRY, + retry=retry, + _target_object=bucket, ) def test_create_bucket_w_explicit_location(self): project = "PROJECT" bucket_name = "bucket-name" location = "us-central1" - data = {"location": location, "name": bucket_name} - - connection = _make_connection( - data, "{'location': 'us-central1', 'name': 'bucket-name'}" - ) - + api_response = {"location": location, "name": bucket_name} credentials = _make_credentials() client = self._make_one(project=project, credentials=credentials) - client._base_connection = connection + client._post_resource = mock.Mock() + client._post_resource.return_value = api_response bucket = client.create_bucket(bucket_name, location=location) - connection.api_request.assert_called_once_with( - method="POST", - path="/b", - data=data, - _target_object=bucket, - query_params={"project": project}, + self.assertEqual(bucket.location, location) + + expected_path = "/b" + expected_data = {"location": location, "name": bucket_name} + expected_query_params = {"project": project} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, timeout=self._get_default_timeout(), retry=DEFAULT_RETRY, + _target_object=bucket, ) - self.assertEqual(bucket.location, location) def test_create_bucket_w_explicit_project(self): - from google.cloud.storage.client import Client - - PROJECT = "PROJECT" - OTHER_PROJECT = "other-project-123" - BUCKET_NAME = "bucket-name" - DATA = {"name": BUCKET_NAME} - connection = _make_connection(DATA) + project = "PROJECT" + other_project = "other-project-123" + bucket_name = "bucket-name" + api_response = {"name": bucket_name} + credentials = _make_credentials() + client = self._make_one(project=project, credentials=credentials) + client._post_resource = mock.Mock() + client._post_resource.return_value = api_response - client = Client(project=PROJECT) - client._base_connection = connection + bucket = client.create_bucket(bucket_name, project=other_project) - bucket = client.create_bucket(BUCKET_NAME, project=OTHER_PROJECT) - connection.api_request.assert_called_once_with( - method="POST", - path="/b", - query_params={"project": OTHER_PROJECT}, - data=DATA, - _target_object=bucket, + expected_path = "/b" + expected_data = api_response + expected_query_params = {"project": other_project} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, timeout=self._get_default_timeout(), retry=DEFAULT_RETRY, + _target_object=bucket, ) - def test_create_w_extra_properties(self): - from google.cloud.storage.client import Client + def test_create_bucket_w_extra_properties(self): from google.cloud.storage.bucket import Bucket - BUCKET_NAME = "bucket-name" - PROJECT = "PROJECT" - CORS = [ + bucket_name = "bucket-name" + project = "PROJECT" + cors = [ { "maxAgeSeconds": 60, "methods": ["*"], @@ -1158,144 +1233,69 @@ def test_create_w_extra_properties(self): "responseHeader": ["X-Custom-Header"], } ] - LIFECYCLE_RULES = [{"action": {"type": "Delete"}, "condition": {"age": 365}}] - LOCATION = "eu" - LABELS = {"color": "red", "flavor": "cherry"} - STORAGE_CLASS = "NEARLINE" - DATA = { - "name": BUCKET_NAME, - "cors": CORS, - "lifecycle": {"rule": LIFECYCLE_RULES}, - "location": LOCATION, - "storageClass": STORAGE_CLASS, + lifecycle_rules = [{"action": {"type": "Delete"}, "condition": {"age": 365}}] + location = "eu" + labels = {"color": "red", "flavor": "cherry"} + storage_class = "NEARLINE" + api_response = { + "name": bucket_name, + "cors": cors, + "lifecycle": {"rule": lifecycle_rules}, + "location": location, + "storageClass": storage_class, "versioning": {"enabled": True}, "billing": {"requesterPays": True}, - "labels": LABELS, + "labels": labels, } + credentials = _make_credentials() + client = self._make_one(project=project, credentials=credentials) + client._post_resource = mock.Mock() + client._post_resource.return_value = api_response - connection = _make_connection(DATA) - client = Client(project=PROJECT) - client._base_connection = connection - - bucket = Bucket(client=client, name=BUCKET_NAME) - bucket.cors = CORS - bucket.lifecycle_rules = LIFECYCLE_RULES - bucket.storage_class = STORAGE_CLASS + bucket = Bucket(client=client, name=bucket_name) + bucket.cors = cors + bucket.lifecycle_rules = lifecycle_rules + bucket.storage_class = storage_class bucket.versioning_enabled = True bucket.requester_pays = True - bucket.labels = LABELS - client.create_bucket(bucket, location=LOCATION) - - connection.api_request.assert_called_once_with( - method="POST", - path="/b", - query_params={"project": PROJECT}, - data=DATA, - _target_object=bucket, - timeout=self._get_default_timeout(), - retry=DEFAULT_RETRY, - ) - - def test_create_hit(self): - from google.cloud.storage.client import Client - - PROJECT = "PROJECT" - BUCKET_NAME = "bucket-name" - DATA = {"name": BUCKET_NAME} - connection = _make_connection(DATA) - client = Client(project=PROJECT) - client._base_connection = connection + bucket.labels = labels - bucket = client.create_bucket(BUCKET_NAME) + client.create_bucket(bucket, location=location) - connection.api_request.assert_called_once_with( - method="POST", - path="/b", - query_params={"project": PROJECT}, - data=DATA, - _target_object=bucket, + expected_path = "/b" + expected_data = api_response + expected_query_params = {"project": project} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, timeout=self._get_default_timeout(), retry=DEFAULT_RETRY, + _target_object=bucket, ) - def test_create_bucket_w_string_success(self): - from google.cloud.storage.bucket import Bucket - + def test_create_bucket_w_name_only(self): project = "PROJECT" - credentials = _make_credentials() - client = self._make_one(project=project, credentials=credentials) - bucket_name = "bucket-name" - json_expected = {"name": bucket_name} - data = json_expected - http = _make_requests_session([_make_json_response(data)]) - client._http_internal = http - - bucket = client.create_bucket(bucket_name) - - self.assertIsInstance(bucket, Bucket) - self.assertEqual(bucket.name, bucket_name) - http.request.assert_called_once_with( - method="POST", - url=mock.ANY, - data=mock.ANY, - headers=mock.ANY, - timeout=mock.ANY, - ) - _, kwargs = http.request.call_args - scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) - self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) - self.assertEqual( - path, "/".join(["", "storage", client._connection.API_VERSION, "b"]), - ) - parms = dict(urlparse.parse_qsl(qs)) - self.assertEqual(parms["project"], project) - json_sent = http.request.call_args_list[0][1]["data"] - self.assertEqual(json_expected, json.loads(json_sent)) - - def test_create_bucket_w_object_success(self): - from google.cloud.storage.bucket import Bucket - - project = "PROJECT" + api_response = {"name": bucket_name} credentials = _make_credentials() client = self._make_one(project=project, credentials=credentials) + client._post_resource = mock.Mock() + client._post_resource.return_value = api_response - bucket_name = "bucket-name" - bucket_obj = Bucket(client, bucket_name) - bucket_obj.storage_class = "COLDLINE" - bucket_obj.requester_pays = True - - json_expected = { - "name": bucket_name, - "billing": {"requesterPays": True}, - "storageClass": "COLDLINE", - } - data = json_expected - http = _make_requests_session([_make_json_response(data)]) - client._http_internal = http - - bucket = client.create_bucket(bucket_obj) + bucket = client.create_bucket(bucket_name) - self.assertIsInstance(bucket, Bucket) - self.assertEqual(bucket.name, bucket_name) - self.assertTrue(bucket.requester_pays) - http.request.assert_called_once_with( - method="POST", - url=mock.ANY, - data=mock.ANY, - headers=mock.ANY, - timeout=mock.ANY, - ) - _, kwargs = http.request.call_args - scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) - self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) - self.assertEqual( - path, "/".join(["", "storage", client._connection.API_VERSION, "b"]), + expected_path = "/b" + expected_data = api_response + expected_query_params = {"project": project} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY, + _target_object=bucket, ) - parms = dict(urlparse.parse_qsl(qs)) - self.assertEqual(parms["project"], project) - json_sent = http.request.call_args_list[0][1]["data"] - self.assertEqual(json_expected, json.loads(json_sent)) def test_download_blob_to_file_with_failure(self): from google.resumable_media import InvalidResponse @@ -1679,43 +1679,43 @@ def fake_response(): self.assertEqual(bucket.name, blob_name) def _create_hmac_key_helper( - self, explicit_project=None, user_project=None, timeout=None + self, explicit_project=None, user_project=None, timeout=None, retry=None, ): import datetime from pytz import UTC from google.cloud.storage.hmac_key import HMACKeyMetadata - PROJECT = "PROJECT" - ACCESS_ID = "ACCESS-ID" - CREDENTIALS = _make_credentials() - EMAIL = "storage-user-123@example.com" - SECRET = "a" * 40 + project = "PROJECT" + access_id = "ACCESS-ID" + credentials = _make_credentials() + email = "storage-user-123@example.com" + secret = "a" * 40 now = datetime.datetime.utcnow().replace(tzinfo=UTC) now_stamp = "{}Z".format(now.isoformat()) if explicit_project is not None: expected_project = explicit_project else: - expected_project = PROJECT + expected_project = project - RESOURCE = { + api_response = { "kind": "storage#hmacKey", "metadata": { - "accessId": ACCESS_ID, + "accessId": access_id, "etag": "ETAG", - "id": "projects/{}/hmacKeys/{}".format(PROJECT, ACCESS_ID), + "id": "projects/{}/hmacKeys/{}".format(project, access_id), "project": expected_project, "state": "ACTIVE", - "serviceAccountEmail": EMAIL, + "serviceAccountEmail": email, "timeCreated": now_stamp, "updated": now_stamp, }, - "secret": SECRET, + "secret": secret, } - client = self._make_one(project=PROJECT, credentials=CREDENTIALS) - http = _make_requests_session([_make_json_response(RESOURCE)]) - client._http_internal = http + client = self._make_one(project=project, credentials=credentials) + client._post_resource = mock.Mock() + client._post_resource.return_value = api_response kwargs = {} if explicit_project is not None: @@ -1725,43 +1725,37 @@ def _create_hmac_key_helper( kwargs["user_project"] = user_project if timeout is None: - timeout = self._get_default_timeout() - kwargs["timeout"] = timeout + expected_timeout = self._get_default_timeout() + else: + expected_timeout = kwargs["timeout"] = timeout + + if retry is None: + expected_retry = None + else: + expected_retry = kwargs["retry"] = retry - metadata, secret = client.create_hmac_key(service_account_email=EMAIL, **kwargs) + metadata, secret = client.create_hmac_key(service_account_email=email, **kwargs) self.assertIsInstance(metadata, HMACKeyMetadata) + self.assertIs(metadata._client, client) - self.assertEqual(metadata._properties, RESOURCE["metadata"]) - self.assertEqual(secret, RESOURCE["secret"]) + self.assertEqual(metadata._properties, api_response["metadata"]) + self.assertEqual(secret, api_response["secret"]) - qs_params = {"serviceAccountEmail": EMAIL} + expected_path = "/projects/{}/hmacKeys".format(expected_project) + expected_data = None + expected_query_params = {"serviceAccountEmail": email} if user_project is not None: - qs_params["userProject"] = user_project + expected_query_params["userProject"] = user_project - http.request.assert_called_once_with( - method="POST", url=mock.ANY, data=None, headers=mock.ANY, timeout=timeout - ) - _, kwargs = http.request.call_args - scheme, netloc, path, qs, _ = urlparse.urlsplit(kwargs.get("url")) - self.assertEqual("%s://%s" % (scheme, netloc), client._connection.API_BASE_URL) - self.assertEqual( - path, - "/".join( - [ - "", - "storage", - client._connection.API_VERSION, - "projects", - expected_project, - "hmacKeys", - ] - ), + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=expected_timeout, + retry=expected_retry, ) - parms = dict(urlparse.parse_qsl(qs)) - for param, expected in qs_params.items(): - self.assertEqual(parms[param], expected) def test_create_hmac_key_defaults(self): self._create_hmac_key_helper() @@ -1769,8 +1763,14 @@ def test_create_hmac_key_defaults(self): def test_create_hmac_key_explicit_project(self): self._create_hmac_key_helper(explicit_project="other-project-456") - def test_create_hmac_key_user_project(self): - self._create_hmac_key_helper(user_project="billed-project", timeout=42) + def test_create_hmac_key_w_user_project(self): + self._create_hmac_key_helper(user_project="billed-project") + + def test_create_hmac_key_w_timeout(self): + self._create_hmac_key_helper(timeout=42) + + def test_create_hmac_key_w_retry(self): + self._create_hmac_key_helper(retry=mock.Mock(spec=[])) def test_list_hmac_keys_defaults_empty(self): PROJECT = "PROJECT" diff --git a/tests/unit/test_notification.py b/tests/unit/test_notification.py index ae8924b08..04ffd68a1 100644 --- a/tests/unit/test_notification.py +++ b/tests/unit/test_notification.py @@ -231,7 +231,8 @@ def test_self_link(self): self.assertEqual(notification.self_link, self.SELF_LINK) def test_create_w_existing_notification_id(self): - client = self._make_client() + client = mock.Mock(spec=["_post_resource", "project"]) + client.project = self.BUCKET_PROJECT bucket = self._make_bucket(client) notification = self._make_one(bucket, self.TOPIC_NAME) notification._properties["id"] = self.NOTIFICATION_ID @@ -239,20 +240,23 @@ def test_create_w_existing_notification_id(self): with self.assertRaises(ValueError): notification.create() + client._post_resource.assert_not_called() + def test_create_w_defaults(self): from google.cloud.storage.notification import NONE_PAYLOAD_FORMAT - client = self._make_client() - bucket = self._make_bucket(client) - notification = self._make_one(bucket, self.TOPIC_NAME) - api_request = client._connection.api_request - api_request.return_value = { + api_response = { "topic": self.TOPIC_REF, "id": self.NOTIFICATION_ID, "etag": self.ETAG, "selfLink": self.SELF_LINK, "payload_format": NONE_PAYLOAD_FORMAT, } + client = mock.Mock(spec=["_post_resource", "project"]) + client.project = self.BUCKET_PROJECT + client._post_resource.return_value = api_response + bucket = self._make_bucket(client) + notification = self._make_one(bucket, self.TOPIC_NAME) notification.create() @@ -264,32 +268,22 @@ def test_create_w_defaults(self): self.assertIsNone(notification.blob_name_prefix) self.assertEqual(notification.payload_format, NONE_PAYLOAD_FORMAT) - data = {"topic": self.TOPIC_REF, "payload_format": NONE_PAYLOAD_FORMAT} - api_request.assert_called_once_with( - method="POST", - path=self.CREATE_PATH, - query_params={}, - data=data, + expected_data = { + "topic": self.TOPIC_REF, + "payload_format": NONE_PAYLOAD_FORMAT, + } + expected_query_params = {} + client._post_resource.assert_called_once_with( + self.CREATE_PATH, + expected_data, + query_params=expected_query_params, timeout=self._get_default_timeout(), retry=None, ) - def test_create_w_explicit_client(self): - USER_PROJECT = "user-project-123" - client = self._make_client() - alt_client = self._make_client() - bucket = self._make_bucket(client, user_project=USER_PROJECT) - notification = self._make_one( - bucket, - self.TOPIC_NAME, - topic_project=self.TOPIC_ALT_PROJECT, - custom_attributes=self.CUSTOM_ATTRIBUTES, - event_types=self.event_types(), - blob_name_prefix=self.BLOB_NAME_PREFIX, - payload_format=self.payload_format(), - ) - api_request = alt_client._connection.api_request - api_request.return_value = { + def test_create_w_explicit_client_w_timeout_w_retry(self): + user_project = "user-project-123" + api_response = { "topic": self.TOPIC_ALT_REF, "custom_attributes": self.CUSTOM_ATTRIBUTES, "event_types": self.event_types(), @@ -299,8 +293,23 @@ def test_create_w_explicit_client(self): "etag": self.ETAG, "selfLink": self.SELF_LINK, } + bucket = self._make_bucket(client=None, user_project=user_project) + notification = self._make_one( + bucket, + self.TOPIC_NAME, + topic_project=self.TOPIC_ALT_PROJECT, + custom_attributes=self.CUSTOM_ATTRIBUTES, + event_types=self.event_types(), + blob_name_prefix=self.BLOB_NAME_PREFIX, + payload_format=self.payload_format(), + ) + client = mock.Mock(spec=["_post_resource", "project"]) + client.project = self.BUCKET_PROJECT + client._post_resource.return_value = api_response + timeout = 42 + retry = mock.Mock(spec=[]) - notification.create(client=alt_client, timeout=42) + notification.create(client=client, timeout=timeout, retry=retry) self.assertEqual(notification.custom_attributes, self.CUSTOM_ATTRIBUTES) self.assertEqual(notification.event_types, self.event_types()) @@ -310,20 +319,20 @@ def test_create_w_explicit_client(self): self.assertEqual(notification.etag, self.ETAG) self.assertEqual(notification.self_link, self.SELF_LINK) - data = { + expected_data = { "topic": self.TOPIC_ALT_REF, "custom_attributes": self.CUSTOM_ATTRIBUTES, "event_types": self.event_types(), "object_name_prefix": self.BLOB_NAME_PREFIX, "payload_format": self.payload_format(), } - api_request.assert_called_once_with( - method="POST", - path=self.CREATE_PATH, - query_params={"userProject": USER_PROJECT}, - data=data, - timeout=42, - retry=None, + expected_query_params = {"userProject": user_project} + client._post_resource.assert_called_once_with( + self.CREATE_PATH, + expected_data, + query_params=expected_query_params, + timeout=timeout, + retry=retry, ) def test_exists_wo_notification_id(self):