From 094444280dd7b7735e24071e5381508cbd392260 Mon Sep 17 00:00:00 2001 From: Gurov Ilya Date: Fri, 15 May 2020 03:46:06 +0300 Subject: [PATCH] feat: add if*generation*Match support, pt1 (#123) * feat: add ifMetageneration*Match support, pt1 * fix unit tests, add test for helper * fix unit tests * add generation match args into more methods * feat: add if*generation*Match support, pt2 * Lint fix. * delete "more than one set "checks * del excess import * delete "more than one set" checks * rename the helper; add error raising in case of wront parameters type * add more system tests * system tests fixes * cleanup system test * fix comments * delete excess checks Co-authored-by: Frank Natividad --- google/cloud/storage/_helpers.py | 183 ++++++++++++- google/cloud/storage/blob.py | 449 +++++++++++++++++++++++++++---- google/cloud/storage/bucket.py | 237 ++++++++++++++-- google/cloud/storage/client.py | 47 +++- tests/system/test_system.py | 185 +++++++++++++ tests/unit/test__helpers.py | 158 +++++++++++ tests/unit/test_blob.py | 267 +++++++++++++++++- tests/unit/test_bucket.py | 141 ++++++++++ tests/unit/test_client.py | 96 +++++++ 9 files changed, 1686 insertions(+), 77 deletions(-) diff --git a/google/cloud/storage/_helpers.py b/google/cloud/storage/_helpers.py index b649384f7..dee5cbc42 100644 --- a/google/cloud/storage/_helpers.py +++ b/google/cloud/storage/_helpers.py @@ -30,6 +30,18 @@ _DEFAULT_STORAGE_HOST = u"https://storage.googleapis.com" +# generation match parameters in camel and snake cases +_GENERATION_MATCH_PARAMETERS = ( + ("if_generation_match", "ifGenerationMatch"), + ("if_generation_not_match", "ifGenerationNotMatch"), + ("if_metageneration_match", "ifMetagenerationMatch"), + ("if_metageneration_not_match", "ifMetagenerationNotMatch"), + ("if_source_generation_match", "ifSourceGenerationMatch"), + ("if_source_generation_not_match", "ifSourceGenerationNotMatch"), + ("if_source_metageneration_match", "ifSourceMetagenerationMatch"), + ("if_source_metageneration_not_match", "ifSourceMetagenerationNotMatch"), +) + def _get_storage_host(): return os.environ.get(STORAGE_EMULATOR_ENV_VAR, _DEFAULT_STORAGE_HOST) @@ -121,27 +133,64 @@ def _query_params(self): params["userProject"] = self.user_project return params - def reload(self, client=None, timeout=_DEFAULT_TIMEOUT): + def reload( + self, + client=None, + timeout=_DEFAULT_TIMEOUT, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, + ): """Reload properties from Cloud Storage. If :attr:`user_project` is set, bills the API request to that project. :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: the client to use. If not passed, falls back to the + :param client: the client to use. If not passed, falls back to the ``client`` stored on the current object. + :type timeout: float or tuple :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + + :type if_generation_match: long + :param if_generation_match: (Optional) Make the operation conditional on whether + the blob's current generation matches the given value. + Setting to 0 makes the operation succeed only if there + are no live versions of the blob. + + :type if_generation_not_match: long + :param if_generation_not_match: (Optional) Make the operation conditional on whether + the blob's current generation does not match the given + value. If no live blob exists, the precondition fails. + Setting to 0 makes the operation succeed only if there + is a live version of the blob. + + :type if_metageneration_match: long + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. """ client = self._require_client(client) query_params = self._query_params # Pass only '?projection=noAcl' here because 'acl' and related # are handled via custom endpoints. query_params["projection"] = "noAcl" + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) api_response = client._connection.api_request( method="GET", path=self.path, @@ -180,7 +229,15 @@ def _set_properties(self, value): # If the values are reset, the changes must as well. self._changes = set() - def patch(self, client=None, timeout=_DEFAULT_TIMEOUT): + def patch( + self, + client=None, + timeout=_DEFAULT_TIMEOUT, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, + ): """Sends all changed properties in a PATCH request. Updates the ``_properties`` with the response from the backend. @@ -189,20 +246,49 @@ def patch(self, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: the client to use. If not passed, falls back to the + :param client: the client to use. If not passed, falls back to the ``client`` stored on the current object. + :type timeout: float or tuple :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + + :type if_generation_match: long + :param if_generation_match: (Optional) Make the operation conditional on whether + the blob's current generation matches the given value. + Setting to 0 makes the operation succeed only if there + are no live versions of the blob. + + :type if_generation_not_match: long + :param if_generation_not_match: (Optional) Make the operation conditional on whether + the blob's current generation does not match the given + value. If no live blob exists, the precondition fails. + Setting to 0 makes the operation succeed only if there + is a live version of the blob. + + :type if_metageneration_match: long + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. """ client = self._require_client(client) query_params = self._query_params # Pass '?projection=full' here because 'PATCH' documented not # to work properly w/ 'noAcl'. query_params["projection"] = "full" + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) update_properties = {key: self._properties[key] for key in self._changes} # Make the API call. @@ -216,7 +302,15 @@ def patch(self, client=None, timeout=_DEFAULT_TIMEOUT): ) self._set_properties(api_response) - def update(self, client=None, timeout=_DEFAULT_TIMEOUT): + def update( + self, + client=None, + timeout=_DEFAULT_TIMEOUT, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, + ): """Sends all properties in a PUT request. Updates the ``_properties`` with the response from the backend. @@ -225,18 +319,46 @@ def update(self, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: the client to use. If not passed, falls back to the + :param client: the client to use. If not passed, falls back to the ``client`` stored on the current object. + :type timeout: float or tuple :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + + :type if_generation_match: long + :param if_generation_match: (Optional) Make the operation conditional on whether + the blob's current generation matches the given value. + Setting to 0 makes the operation succeed only if there + are no live versions of the blob. + + :type if_generation_not_match: long + :param if_generation_not_match: (Optional) Make the operation conditional on whether + the blob's current generation does not match the given + value. If no live blob exists, the precondition fails. + Setting to 0 makes the operation succeed only if there + is a live version of the blob. + + :type if_metageneration_match: long + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. """ client = self._require_client(client) + query_params = self._query_params query_params["projection"] = "full" + _add_generation_match_parameters( + query_params, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) api_response = client._connection.api_request( method="PUT", path=self.path, @@ -312,3 +434,52 @@ def _convert_to_timestamp(value): utc_naive = value.replace(tzinfo=None) - value.utcoffset() mtime = (utc_naive - datetime(1970, 1, 1)).total_seconds() return mtime + + +def _add_generation_match_parameters(parameters, **match_parameters): + """Add generation match parameters into the given parameters list. + + :type parameters: list or dict + :param parameters: Parameters list or dict. + + :type match_parameters: dict + :param match_parameters: if*generation*match parameters to add. + + :raises: :exc:`ValueError` if ``parameters`` is not a ``list()`` + or a ``dict()``. + """ + for snakecase_name, camelcase_name in _GENERATION_MATCH_PARAMETERS: + value = match_parameters.get(snakecase_name) + + if value is not None: + if isinstance(parameters, list): + parameters.append((camelcase_name, value)) + + elif isinstance(parameters, dict): + parameters[camelcase_name] = value + + else: + raise ValueError( + "`parameters` argument should be a dict() or a list()." + ) + + +def _raise_if_more_than_one_set(**kwargs): + """Raise ``ValueError`` exception if more than one parameter was set. + + :type error: :exc:`ValueError` + :param error: Description of which fields were set + + :raises: :class:`~ValueError` containing the fields that were set + """ + if sum(arg is not None for arg in kwargs.values()) > 1: + escaped_keys = ["'%s'" % name for name in kwargs.keys()] + + keys_but_last = ", ".join(escaped_keys[:-1]) + last_key = escaped_keys[-1] + + msg = "Pass at most one of {keys_but_last} and {last_key}".format( + keys_but_last=keys_but_last, last_key=last_key + ) + + raise ValueError(msg) diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index d3416616b..9567c1096 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -54,9 +54,11 @@ from google.cloud._helpers import _rfc3339_to_datetime from google.cloud._helpers import _to_bytes from google.cloud.exceptions import NotFound +from google.cloud.storage._helpers import _add_generation_match_parameters from google.cloud.storage._helpers import _PropertyMixin from google.cloud.storage._helpers import _scalar_property from google.cloud.storage._helpers import _convert_to_timestamp +from google.cloud.storage._helpers import _raise_if_more_than_one_set from google.cloud.storage._signing import generate_signed_url_v2 from google.cloud.storage._signing import generate_signed_url_v4 from google.cloud.storage.acl import ACL @@ -183,7 +185,7 @@ def __init__( self.chunk_size = chunk_size # Check that setter accepts value. self._bucket = bucket self._acl = ObjectACL(self) - _raise_for_more_than_one_none( + _raise_if_more_than_one_set( encryption_key=encryption_key, kms_key_name=kms_key_name, ) @@ -565,7 +567,15 @@ def generate_signed_url( access_token=access_token, ) - def exists(self, client=None, timeout=_DEFAULT_TIMEOUT): + def exists( + self, + client=None, + timeout=_DEFAULT_TIMEOUT, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, + ): """Determines whether or not this blob exists. If :attr:`user_project` is set on the bucket, bills the API request @@ -582,6 +592,27 @@ def exists(self, client=None, timeout=_DEFAULT_TIMEOUT): Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + :type if_generation_match: long + :param if_generation_match: (Optional) Make the operation conditional on whether + the blob's current generation matches the given value. + Setting to 0 makes the operation succeed only if there + are no live versions of the blob. + + :type if_generation_not_match: long + :param if_generation_not_match: (Optional) Make the operation conditional on whether + the blob's current generation does not match the given + value. If no live blob exists, the precondition fails. + Setting to 0 makes the operation succeed only if there + is a live version of the blob. + + :type if_metageneration_match: long + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. + :rtype: bool :returns: True if the blob exists in Cloud Storage. """ @@ -591,6 +622,13 @@ def exists(self, client=None, timeout=_DEFAULT_TIMEOUT): query_params = self._query_params query_params["fields"] = "name" + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) try: # We intentionally pass `_target_object=None` since fields=name # would limit the local properties. @@ -608,7 +646,15 @@ def exists(self, client=None, timeout=_DEFAULT_TIMEOUT): except NotFound: return False - def delete(self, client=None, timeout=_DEFAULT_TIMEOUT): + def delete( + self, + client=None, + timeout=_DEFAULT_TIMEOUT, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, + ): """Deletes a blob from Cloud Storage. If :attr:`user_project` is set on the bucket, bills the API request @@ -616,8 +662,9 @@ def delete(self, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: (Optional) The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. + :type timeout: float or tuple :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. @@ -625,12 +672,40 @@ def delete(self, client=None, timeout=_DEFAULT_TIMEOUT): Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + :type if_generation_match: long + :param if_generation_match: (Optional) Make the operation conditional on whether + the blob's current generation matches the given value. + Setting to 0 makes the operation succeed only if there + are no live versions of the blob. + + :type if_generation_not_match: long + :param if_generation_not_match: (Optional) Make the operation conditional on whether + the blob's current generation does not match the given + value. If no live blob exists, the precondition fails. + Setting to 0 makes the operation succeed only if there + is a live version of the blob. + + :type if_metageneration_match: long + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. + :raises: :class:`google.cloud.exceptions.NotFound` (propagated from :meth:`google.cloud.storage.bucket.Bucket.delete_blob`). """ self.bucket.delete_blob( - self.name, client=client, generation=self.generation, timeout=timeout + self.name, + client=client, + generation=self.generation, + timeout=timeout, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, ) def _get_transport(self, client): @@ -648,7 +723,14 @@ def _get_transport(self, client): client = self._require_client(client) return client._http - def _get_download_url(self, client): + def _get_download_url( + self, + client, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, + ): """Get the download URL for the current blob. If the ``media_link`` has been loaded, it will be used, otherwise @@ -658,6 +740,26 @@ def _get_download_url(self, client): :type client: :class:`~google.cloud.storage.client.Client` :param client: The client to use. + :type if_generation_match: long + :param if_generation_match: (Optional) Make the operation conditional on whether + the blob's current generation matches the given value. + Setting to 0 makes the operation succeed only if there + are no live versions of the blob. + + :type if_generation_not_match: long + :param if_generation_not_match: (Optional) Make the operation conditional on whether + the blob's current generation does not match the given + value. If no live blob exists, the precondition fails. + Setting to 0 makes the operation succeed only if there + is a live version of the blob. + + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. + :rtype: str :returns: The download URL for the current blob. """ @@ -674,6 +776,13 @@ def _get_download_url(self, client): if self.user_project is not None: name_value_pairs.append(("userProject", self.user_project)) + _add_generation_match_parameters( + name_value_pairs, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) return _add_query_parameters(base_url, name_value_pairs) def _do_download( @@ -746,7 +855,16 @@ def _do_download( download.consume_next_chunk(transport) def download_to_file( - self, file_obj, client=None, start=None, end=None, raw_download=False + self, + file_obj, + client=None, + start=None, + end=None, + raw_download=False, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, ): """Download the contents of this blob into a file-like object. @@ -791,10 +909,37 @@ def download_to_file( :param raw_download: (Optional) If true, download the object without any expansion. + :type if_generation_match: long + :param if_generation_match: (Optional) Make the operation conditional on whether + the blob's current generation matches the given value. + Setting to 0 makes the operation succeed only if there + are no live versions of the blob. + + :type if_generation_not_match: long + :param if_generation_not_match: (Optional) Make the operation conditional on whether + the blob's current generation does not match the given + value. If no live blob exists, the precondition fails. + Setting to 0 makes the operation succeed only if there + is a live version of the blob. + + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. + :raises: :class:`google.cloud.exceptions.NotFound` """ client = self._require_client(client) - download_url = self._get_download_url(client) + + download_url = self._get_download_url( + client, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) headers = _get_encryption_headers(self._encryption_key) headers["accept-encoding"] = "gzip" @@ -807,7 +952,16 @@ def download_to_file( _raise_from_invalid_response(exc) def download_to_filename( - self, filename, client=None, start=None, end=None, raw_download=False + self, + filename, + client=None, + start=None, + end=None, + raw_download=False, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, ): """Download the contents of this blob into a named file. @@ -819,7 +973,7 @@ def download_to_filename( :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: (Optional) The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. :type start: int @@ -832,6 +986,26 @@ def download_to_filename( :param raw_download: (Optional) If true, download the object without any expansion. + :type if_generation_match: long + :param if_generation_match: (Optional) Make the operation conditional on whether + the blob's current generation matches the given value. + Setting to 0 makes the operation succeed only if there + are no live versions of the blob. + + :type if_generation_not_match: long + :param if_generation_not_match: (Optional) Make the operation conditional on whether + the blob's current generation does not match the given + value. If no live blob exists, the precondition fails. + Setting to 0 makes the operation succeed only if there + is a live version of the blob. + + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. + :raises: :class:`google.cloud.exceptions.NotFound` """ try: @@ -842,6 +1016,10 @@ def download_to_filename( start=start, end=end, raw_download=raw_download, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, ) except resumable_media.DataCorruption: # Delete the corrupt downloaded file. @@ -856,7 +1034,17 @@ def download_to_filename( mtime = updated.timestamp() os.utime(file_obj.name, (mtime, mtime)) - def download_as_string(self, client=None, start=None, end=None, raw_download=False): + def download_as_string( + self, + client=None, + start=None, + end=None, + raw_download=False, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, + ): """Download the contents of this blob as a bytes object. If :attr:`user_project` is set on the bucket, bills the API request @@ -864,7 +1052,7 @@ def download_as_string(self, client=None, start=None, end=None, raw_download=Fal :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: (Optional) The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. :type start: int @@ -877,8 +1065,29 @@ def download_as_string(self, client=None, start=None, end=None, raw_download=Fal :param raw_download: (Optional) If true, download the object without any expansion. + :type if_generation_match: long + :param if_generation_match: (Optional) Make the operation conditional on whether + the blob's current generation matches the given value. + Setting to 0 makes the operation succeed only if there + are no live versions of the blob. + + :type if_generation_not_match: long + :param if_generation_not_match: (Optional) Make the operation conditional on whether + the blob's current generation does not match the given + value. If no live blob exists, the precondition fails. + Setting to 0 makes the operation succeed only if there + is a live version of the blob. + + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. + :rtype: bytes :returns: The data stored in this blob. + :raises: :class:`google.cloud.exceptions.NotFound` """ string_buffer = BytesIO() @@ -888,6 +1097,10 @@ def download_as_string(self, client=None, start=None, end=None, raw_download=Fal start=start, end=end, raw_download=raw_download, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, ) return string_buffer.getvalue() @@ -1540,16 +1753,6 @@ def upload_from_file( if num_retries is not None: warnings.warn(_NUM_RETRIES_MESSAGE, DeprecationWarning, stacklevel=2) - _raise_for_more_than_one_none( - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - ) - - _raise_for_more_than_one_none( - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) - _maybe_rewind(file_obj, rewind=rewind) predefined_acl = ACL.validate_predefined(predefined_acl) @@ -2046,7 +2249,21 @@ def compose(self, sources, client=None, timeout=_DEFAULT_TIMEOUT): ) self._set_properties(api_response) - def rewrite(self, source, token=None, client=None, timeout=_DEFAULT_TIMEOUT): + def rewrite( + self, + source, + token=None, + client=None, + timeout=_DEFAULT_TIMEOUT, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, + if_source_generation_match=None, + if_source_generation_not_match=None, + if_source_metageneration_match=None, + if_source_metageneration_not_match=None, + ): """Rewrite source blob into this one. If :attr:`user_project` is set on the bucket, bills the API request @@ -2072,6 +2289,63 @@ def rewrite(self, source, token=None, client=None, timeout=_DEFAULT_TIMEOUT): Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + :type if_generation_match: long + :param if_generation_match: (Optional) Makes the operation + conditional on whether the destination + object's current generation matches the + given value. Setting to 0 makes the + operation succeed only if there are no + live versions of the object. + + :type if_generation_not_match: long + :param if_generation_not_match: (Optional) Makes the operation + conditional on whether the + destination object's current + generation does not match the given + value. If no live object exists, + the precondition fails. Setting to + 0 makes the operation succeed only + if there is a live version + of the object. + + :type if_metageneration_match: long + :param if_metageneration_match: (Optional) Makes the operation + conditional on whether the + destination object's current + metageneration matches the given + value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Makes the operation + conditional on whether the + destination object's current + metageneration does not match + the given value. + + :type if_source_generation_match: long + :param if_source_generation_match: (Optional) Makes the operation + conditional on whether the source + object's generation matches the + given value. + + :type if_source_generation_not_match: long + :param if_source_generation_not_match: (Optional) Makes the operation + conditional on whether the source + object's generation does not match + the given value. + + :type if_source_metageneration_match: long + :param if_source_metageneration_match: (Optional) Makes the operation + conditional on whether the source + object's current metageneration + matches the given value. + + :type if_source_metageneration_not_match: long + :param if_source_metageneration_not_match: (Optional) Makes the operation + conditional on whether the source + object's current metageneration + does not match the given value. + :rtype: tuple :returns: ``(token, bytes_rewritten, total_bytes)``, where ``token`` is a rewrite token (``None`` if the rewrite is complete), @@ -2096,6 +2370,18 @@ def rewrite(self, source, token=None, client=None, timeout=_DEFAULT_TIMEOUT): if self.kms_key_name is not None: query_params["destinationKmsKeyName"] = self.kms_key_name + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + if_source_generation_match=if_source_generation_match, + if_source_generation_not_match=if_source_generation_not_match, + if_source_metageneration_match=if_source_metageneration_match, + if_source_metageneration_not_match=if_source_metageneration_not_match, + ) + api_response = client._connection.api_request( method="POST", path=source.path + "/rewriteTo" + self.path, @@ -2117,7 +2403,19 @@ def rewrite(self, source, token=None, client=None, timeout=_DEFAULT_TIMEOUT): return api_response["rewriteToken"], rewritten, size - def update_storage_class(self, new_class, client=None): + def update_storage_class( + self, + new_class, + client=None, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, + if_source_generation_match=None, + if_source_generation_not_match=None, + if_source_metageneration_match=None, + if_source_metageneration_not_match=None, + ): """Update blob's storage class via a rewrite-in-place. This helper will wait for the rewrite to complete before returning, so it may take some time for large files. @@ -2142,6 +2440,63 @@ def update_storage_class(self, new_class, client=None): :type client: :class:`~google.cloud.storage.client.Client` :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the blob's bucket. + + :type if_generation_match: long + :param if_generation_match: (Optional) Makes the operation + conditional on whether the destination + object's current generation matches the + given value. Setting to 0 makes the + operation succeed only if there are no + live versions of the object. + + :type if_generation_not_match: long + :param if_generation_not_match: (Optional) Makes the operation + conditional on whether the + destination object's current + generation does not match the given + value. If no live object exists, + the precondition fails. Setting to + 0 makes the operation succeed only + if there is a live version + of the object. + + :type if_metageneration_match: long + :param if_metageneration_match: (Optional) Makes the operation + conditional on whether the + destination object's current + metageneration matches the given + value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Makes the operation + conditional on whether the + destination object's current + metageneration does not match + the given value. + + :type if_source_generation_match: long + :param if_source_generation_match: (Optional) Makes the operation + conditional on whether the source + object's generation matches the + given value. + + :type if_source_generation_not_match: long + :param if_source_generation_not_match: (Optional) Makes the operation + conditional on whether the source + object's generation does not match + the given value. + + :type if_source_metageneration_match: long + :param if_source_metageneration_match: (Optional) Makes the operation + conditional on whether the source + object's current metageneration + matches the given value. + + :type if_source_metageneration_not_match: long + :param if_source_metageneration_not_match: (Optional) Makes the operation + conditional on whether the source + object's current metageneration + does not match the given value. """ if new_class not in self.STORAGE_CLASSES: raise ValueError("Invalid storage class: %s" % (new_class,)) @@ -2150,9 +2505,30 @@ def update_storage_class(self, new_class, client=None): self._patch_property("storageClass", new_class) # Execute consecutive rewrite operations until operation is done - token, _, _ = self.rewrite(self) + token, _, _ = self.rewrite( + self, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + if_source_generation_match=if_source_generation_match, + if_source_generation_not_match=if_source_generation_not_match, + if_source_metageneration_match=if_source_metageneration_match, + if_source_metageneration_not_match=if_source_metageneration_not_match, + ) while token is not None: - token, _, _ = self.rewrite(self, token=token) + token, _, _ = self.rewrite( + self, + token=token, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + if_source_generation_match=if_source_generation_match, + if_source_generation_not_match=if_source_generation_not_match, + if_source_metageneration_match=if_source_metageneration_match, + if_source_metageneration_not_match=if_source_metageneration_not_match, + ) cache_control = _scalar_property("cacheControl") """HTTP 'Cache-Control' header for this object. @@ -2639,24 +3015,3 @@ def _add_query_parameters(base_url, name_value_pairs): query = parse_qsl(query) query.extend(name_value_pairs) return urlunsplit((scheme, netloc, path, urlencode(query), frag)) - - -def _raise_for_more_than_one_none(**kwargs): - """Raise ``ValueError`` exception if more than one parameter was set. - - :type error: :exc:`ValueError` - :param error: Description of which fields were set - - :raises: :class:`~ValueError` containing the fields that were set - """ - if sum(arg is not None for arg in kwargs.values()) > 1: - escaped_keys = ["'%s'" % name for name in kwargs.keys()] - - keys_but_last = ", ".join(escaped_keys[:-1]) - last_key = escaped_keys[-1] - - msg = "Pass at most one of {keys_but_last} and {last_key}".format( - keys_but_last=keys_but_last, last_key=last_key - ) - - raise ValueError(msg) diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index 8540bef6e..febcfc608 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -32,6 +32,7 @@ from google.cloud.exceptions import NotFound from google.api_core.iam import Policy from google.cloud.storage import _signing +from google.cloud.storage._helpers import _add_generation_match_parameters from google.cloud.storage._helpers import _PropertyMixin from google.cloud.storage._helpers import _scalar_property from google.cloud.storage._helpers import _validate_name @@ -646,15 +647,22 @@ def notification( notification_id=notification_id, ) - def exists(self, client=None, timeout=_DEFAULT_TIMEOUT): + def exists( + self, + client=None, + timeout=_DEFAULT_TIMEOUT, + if_metageneration_match=None, + if_metageneration_not_match=None, + ): """Determines whether or not this bucket exists. If :attr:`user_project` is set, bills the API request to that project. :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: (Optional) The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. + :type timeout: float or tuple :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. @@ -662,6 +670,14 @@ def exists(self, client=None, timeout=_DEFAULT_TIMEOUT): Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + :type if_metageneration_match: long + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. + :rtype: bool :returns: True if the bucket exists in Cloud Storage. """ @@ -673,6 +689,11 @@ def exists(self, client=None, timeout=_DEFAULT_TIMEOUT): if self.user_project is not None: query_params["userProject"] = self.user_project + _add_generation_match_parameters( + query_params, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) try: # We intentionally pass `_target_object=None` since fields=name # would limit the local properties. @@ -762,7 +783,13 @@ def create( timeout=timeout, ) - def patch(self, client=None, timeout=_DEFAULT_TIMEOUT): + def patch( + self, + client=None, + timeout=_DEFAULT_TIMEOUT, + if_metageneration_match=None, + if_metageneration_not_match=None, + ): """Sends all changed properties in a PATCH request. Updates the ``_properties`` with the response from the backend. @@ -771,14 +798,23 @@ def patch(self, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: the client to use. If not passed, falls back to the + :param client: the client to use. If not passed, falls back to the ``client`` stored on the current object. + :type timeout: float or tuple :param timeout: (Optional) The amount of time, in seconds, to wait for the server response. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + + :type if_metageneration_match: long + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. """ # Special case: For buckets, it is possible that labels are being # removed; this requires special handling. @@ -789,7 +825,12 @@ def patch(self, client=None, timeout=_DEFAULT_TIMEOUT): self._properties["labels"][removed_label] = None # Call the superclass method. - return super(Bucket, self).patch(client=client, timeout=timeout) + return super(Bucket, self).patch( + client=client, + timeout=timeout, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) @property def acl(self): @@ -828,6 +869,10 @@ def get_blob( encryption_key=None, generation=None, timeout=_DEFAULT_TIMEOUT, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, **kwargs ): """Get a blob object by name. @@ -865,6 +910,27 @@ def get_blob( Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + :type if_generation_match: long + :param if_generation_match: (Optional) Make the operation conditional on whether + the blob's current generation matches the given value. + Setting to 0 makes the operation succeed only if there + are no live versions of the blob. + + :type if_generation_not_match: long + :param if_generation_not_match: (Optional) Make the operation conditional on whether + the blob's current generation does not match the given + value. If no live blob exists, the precondition fails. + Setting to 0 makes the operation succeed only if there + is a live version of the blob. + + :type if_metageneration_match: long + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. + :param kwargs: Keyword arguments to pass to the :class:`~google.cloud.storage.blob.Blob` constructor. @@ -882,7 +948,14 @@ def get_blob( # NOTE: This will not fail immediately in a batch. However, when # Batch.finish() is called, the resulting `NotFound` will be # raised. - blob.reload(client=client, timeout=timeout) + blob.reload( + client=client, + timeout=timeout, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) except NotFound: return None else: @@ -1097,7 +1170,14 @@ def get_notification(self, notification_id, client=None, timeout=_DEFAULT_TIMEOU notification.reload(client=client, timeout=timeout) return notification - def delete(self, force=False, client=None, timeout=_DEFAULT_TIMEOUT): + def delete( + self, + force=False, + client=None, + timeout=_DEFAULT_TIMEOUT, + if_metageneration_match=None, + if_metageneration_not_match=None, + ): """Delete this bucket. The bucket **must** be empty in order to submit a delete request. If @@ -1105,9 +1185,8 @@ def delete(self, force=False, client=None, timeout=_DEFAULT_TIMEOUT): objects / blobs in the bucket (i.e. try to empty the bucket). If the bucket doesn't exist, this will raise - :class:`google.cloud.exceptions.NotFound`. If the bucket is not empty - (and ``force=False``), will raise - :class:`google.cloud.exceptions.Conflict`. + :class:`google.cloud.exceptions.NotFound`. If the bucket is not empty + (and ``force=False``), will raise :class:`google.cloud.exceptions.Conflict`. If ``force=True`` and the bucket contains more than 256 objects / blobs this will cowardly refuse to delete the objects (or the bucket). This @@ -1121,8 +1200,9 @@ def delete(self, force=False, client=None, timeout=_DEFAULT_TIMEOUT): :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: (Optional) The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. + :type timeout: float or tuple :param timeout: (Optional) The amount of time, in seconds, to wait for the server response on each request. @@ -1130,6 +1210,14 @@ def delete(self, force=False, client=None, timeout=_DEFAULT_TIMEOUT): Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + :type if_metageneration_match: long + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. + :raises: :class:`ValueError` if ``force`` is ``True`` and the bucket contains more than 256 objects / blobs. """ @@ -1139,6 +1227,11 @@ def delete(self, force=False, client=None, timeout=_DEFAULT_TIMEOUT): if self.user_project is not None: query_params["userProject"] = self.user_project + _add_generation_match_parameters( + query_params, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) if force: blobs = list( self.list_blobs( @@ -1173,7 +1266,15 @@ def delete(self, force=False, client=None, timeout=_DEFAULT_TIMEOUT): ) def delete_blob( - self, blob_name, client=None, generation=None, timeout=_DEFAULT_TIMEOUT + self, + blob_name, + client=None, + generation=None, + timeout=_DEFAULT_TIMEOUT, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, ): """Deletes a blob from the current bucket. @@ -1193,7 +1294,7 @@ def delete_blob( :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: (Optional) The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type generation: long @@ -1207,6 +1308,27 @@ def delete_blob( Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + :type if_generation_match: long + :param if_generation_match: (Optional) Make the operation conditional on whether + the blob's current generation matches the given value. + Setting to 0 makes the operation succeed only if there + are no live versions of the blob. + + :type if_generation_not_match: long + :param if_generation_not_match: (Optional) Make the operation conditional on whether + the blob's current generation does not match the given + value. If no live blob exists, the precondition fails. + Setting to 0 makes the operation succeed only if there + is a live version of the blob. + + :type if_metageneration_match: long + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. + :raises: :class:`google.cloud.exceptions.NotFound` (to suppress the exception, call ``delete_blobs``, passing a no-op ``on_error`` callback, e.g.: @@ -1219,13 +1341,21 @@ def delete_blob( client = self._require_client(client) blob = Blob(blob_name, bucket=self, generation=generation) + query_params = copy.deepcopy(blob._query_params) + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) # We intentionally pass `_target_object=None` since a DELETE # request has no response value (whether in a standard request or # in a batch request). client._connection.api_request( method="DELETE", path=blob.path, - query_params=blob._query_params, + query_params=query_params, _target_object=None, timeout=timeout, ) @@ -1283,6 +1413,14 @@ def copy_blob( preserve_acl=True, source_generation=None, timeout=_DEFAULT_TIMEOUT, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, + if_source_generation_match=None, + if_source_generation_not_match=None, + if_source_metageneration_match=None, + if_source_metageneration_not_match=None, ): """Copy the given blob to the given bucket, optionally with a new name. @@ -1300,7 +1438,7 @@ def copy_blob( :type client: :class:`~google.cloud.storage.client.Client` or ``NoneType`` - :param client: (Optional) The client to use. If not passed, falls back + :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. :type preserve_acl: bool @@ -1319,6 +1457,63 @@ def copy_blob( Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + :type if_generation_match: long + :param if_generation_match: (Optional) Makes the operation + conditional on whether the destination + object's current generation matches the + given value. Setting to 0 makes the + operation succeed only if there are no + live versions of the object. + + :type if_generation_not_match: long + :param if_generation_not_match: (Optional) Makes the operation + conditional on whether the + destination object's current + generation does not match the given + value. If no live object exists, + the precondition fails. Setting to + 0 makes the operation succeed only + if there is a live version + of the object. + + :type if_metageneration_match: long + :param if_metageneration_match: (Optional) Makes the operation + conditional on whether the + destination object's current + metageneration matches the given + value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Makes the operation + conditional on whether the + destination object's current + metageneration does not match + the given value. + + :type if_source_generation_match: long + :param if_source_generation_match: (Optional) Makes the operation + conditional on whether the source + object's generation matches the + given value. + + :type if_source_generation_not_match: long + :param if_source_generation_not_match: (Optional) Makes the operation + conditional on whether the source + object's generation does not match + the given value. + + :type if_source_metageneration_match: long + :param if_source_metageneration_match: (Optional) Makes the operation + conditional on whether the source + object's current metageneration + matches the given value. + + :type if_source_metageneration_not_match: long + :param if_source_metageneration_not_match: (Optional) Makes the operation + conditional on whether the source + object's current metageneration + does not match the given value. + :rtype: :class:`google.cloud.storage.blob.Blob` :returns: The new Blob. @@ -1345,6 +1540,18 @@ def copy_blob( if source_generation is not None: query_params["sourceGeneration"] = source_generation + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + if_source_generation_match=if_source_generation_match, + if_source_generation_not_match=if_source_generation_not_match, + if_source_metageneration_match=if_source_metageneration_match, + if_source_metageneration_not_match=if_source_metageneration_not_match, + ) + if new_name is None: new_name = blob.name diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index 4a4bbe733..58c1dcbb8 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -281,7 +281,13 @@ def batch(self): """ return Batch(client=self) - def get_bucket(self, bucket_or_name, timeout=_DEFAULT_TIMEOUT): + def get_bucket( + self, + bucket_or_name, + timeout=_DEFAULT_TIMEOUT, + if_metageneration_match=None, + if_metageneration_not_match=None, + ): """API call: retrieve a bucket via a GET request. See @@ -300,6 +306,14 @@ def get_bucket(self, bucket_or_name, timeout=_DEFAULT_TIMEOUT): Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + if_metageneration_match (Optional[long]): + Make the operation conditional on whether the + blob's current metageneration matches the given value. + + if_metageneration_not_match (Optional[long]): + Make the operation conditional on whether the blob's + current metageneration does not match the given value. + Returns: google.cloud.storage.bucket.Bucket The bucket matching the name provided. @@ -329,11 +343,21 @@ def get_bucket(self, bucket_or_name, timeout=_DEFAULT_TIMEOUT): """ bucket = self._bucket_arg_to_bucket(bucket_or_name) - - bucket.reload(client=self, timeout=timeout) + bucket.reload( + client=self, + timeout=timeout, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) return bucket - def lookup_bucket(self, bucket_name, timeout=_DEFAULT_TIMEOUT): + def lookup_bucket( + self, + bucket_name, + timeout=_DEFAULT_TIMEOUT, + if_metageneration_match=None, + if_metageneration_not_match=None, + ): """Get a bucket by name, returning None if not found. You can use this if you would rather check for a None value @@ -353,11 +377,24 @@ def lookup_bucket(self, bucket_name, timeout=_DEFAULT_TIMEOUT): Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + :type if_metageneration_match: long + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + + :type if_metageneration_not_match: long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. + :rtype: :class:`google.cloud.storage.bucket.Bucket` :returns: The bucket matching the name provided or None if not found. """ try: - return self.get_bucket(bucket_name, timeout=timeout) + return self.get_bucket( + bucket_name, + timeout=timeout, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) except NotFound: return None diff --git a/tests/system/test_system.py b/tests/system/test_system.py index 82f0cb98b..6ca87edb1 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -427,6 +427,66 @@ def test_copy_existing_file_with_user_project(self): for blob in to_delete: retry_429_harder(blob.delete)() + def test_copy_file_with_generation_match(self): + new_bucket_name = "generation-match" + unique_resource_id("-") + created = retry_429_503(Config.CLIENT.create_bucket)( + new_bucket_name, requester_pays=True + ) + self.case_buckets_to_delete.append(new_bucket_name) + self.assertEqual(created.name, new_bucket_name) + + to_delete = [] + blob = storage.Blob("simple", bucket=created) + blob.upload_from_string(b"DEADBEEF") + to_delete.append(blob) + try: + dest_bucket = Config.CLIENT.bucket(new_bucket_name) + + new_blob = dest_bucket.copy_blob( + blob, + dest_bucket, + "simple-copy", + if_source_generation_match=blob.generation, + ) + to_delete.append(new_blob) + + base_contents = blob.download_as_string() + copied_contents = new_blob.download_as_string() + self.assertEqual(base_contents, copied_contents) + finally: + for blob in to_delete: + retry_429_harder(blob.delete)() + + def test_copy_file_with_metageneration_match(self): + new_bucket_name = "generation-match" + unique_resource_id("-") + created = retry_429_503(Config.CLIENT.create_bucket)( + new_bucket_name, requester_pays=True + ) + self.case_buckets_to_delete.append(new_bucket_name) + self.assertEqual(created.name, new_bucket_name) + + to_delete = [] + blob = storage.Blob("simple", bucket=created) + blob.upload_from_string(b"DEADBEEF") + to_delete.append(blob) + try: + dest_bucket = Config.CLIENT.bucket(new_bucket_name) + + new_blob = dest_bucket.copy_blob( + blob, + dest_bucket, + "simple-copy", + if_source_metageneration_match=blob.metageneration, + ) + to_delete.append(new_blob) + + base_contents = blob.download_as_string() + copied_contents = new_blob.download_as_string() + self.assertEqual(base_contents, copied_contents) + finally: + for blob in to_delete: + retry_429_harder(blob.delete)() + @unittest.skipUnless(USER_PROJECT, "USER_PROJECT not set in environment.") def test_bucket_get_blob_with_user_project(self): new_bucket_name = "w-requester-pays" + unique_resource_id("-") @@ -588,6 +648,69 @@ def test_crud_blob_w_user_project(self): blob1.delete() + def test_crud_blob_w_generation_match(self): + WRONG_GENERATION_NUMBER = 6 + WRONG_METAGENERATION_NUMBER = 9 + + bucket = Config.CLIENT.bucket(self.bucket.name) + blob = bucket.blob("SmallFile") + + file_data = self.FILES["simple"] + with open(file_data["path"], mode="rb") as to_read: + file_contents = to_read.read() + + blob.upload_from_filename(file_data["path"]) + gen0 = blob.generation + + # Upload a second generation of the blob + blob.upload_from_string(b"gen1") + gen1 = blob.generation + + blob0 = bucket.blob("SmallFile", generation=gen0) + blob1 = bucket.blob("SmallFile", generation=gen1) + + try: + # Exercise 'objects.get' (metadata) w/ generation match. + with self.assertRaises(google.api_core.exceptions.PreconditionFailed): + blob.exists(if_generation_match=WRONG_GENERATION_NUMBER) + + self.assertTrue(blob.exists(if_generation_match=gen1)) + + with self.assertRaises(google.api_core.exceptions.PreconditionFailed): + blob.reload(if_metageneration_match=WRONG_METAGENERATION_NUMBER) + + blob.reload(if_generation_match=gen1) + + # Exercise 'objects.get' (media) w/ generation match. + self.assertEqual( + blob0.download_as_string(if_generation_match=gen0), file_contents + ) + self.assertEqual( + blob1.download_as_string(if_generation_not_match=gen0), b"gen1" + ) + + # Exercise 'objects.patch' w/ generation match. + blob0.content_language = "en" + blob0.patch(if_generation_match=gen0) + + self.assertEqual(blob0.content_language, "en") + self.assertIsNone(blob1.content_language) + + # Exercise 'objects.update' w/ generation match. + metadata = {"foo": "Foo", "bar": "Bar"} + blob0.metadata = metadata + blob0.update(if_generation_match=gen0) + + self.assertEqual(blob0.metadata, metadata) + self.assertIsNone(blob1.metadata) + finally: + # Exercise 'objects.delete' (metadata) w/ generation match. + with self.assertRaises(google.api_core.exceptions.PreconditionFailed): + blob0.delete(if_metageneration_match=WRONG_METAGENERATION_NUMBER) + + blob0.delete(if_generation_match=gen0) + blob1.delete(if_metageneration_not_match=WRONG_METAGENERATION_NUMBER) + @unittest.skipUnless(USER_PROJECT, "USER_PROJECT not set in environment.") def test_blob_acl_w_user_project(self): with_user_project = Config.CLIENT.bucket( @@ -662,6 +785,34 @@ def test_direct_write_and_read_into_file(self): self.assertEqual(file_contents, stored_contents) + def test_download_w_generation_match(self): + WRONG_GENERATION_NUMBER = 6 + + blob = self.bucket.blob("MyBuffer") + file_contents = b"Hello World" + blob.upload_from_string(file_contents) + self.case_blobs_to_delete.append(blob) + + same_blob = self.bucket.blob("MyBuffer") + same_blob.reload() # Initialize properties. + temp_filename = tempfile.mktemp() + with open(temp_filename, "wb") as file_obj: + with self.assertRaises(google.api_core.exceptions.PreconditionFailed): + same_blob.download_to_file( + file_obj, if_generation_match=WRONG_GENERATION_NUMBER + ) + + same_blob.download_to_file( + file_obj, + if_generation_match=blob.generation, + if_metageneration_match=blob.metageneration, + ) + + with open(temp_filename, "rb") as file_obj: + stored_contents = file_obj.read() + + self.assertEqual(file_contents, stored_contents) + def test_copy_existing_file(self): filename = self.FILES["logo"]["path"] blob = storage.Blob("CloudLogo", bucket=self.bucket) @@ -1410,6 +1561,40 @@ def test_rewrite_rotate_with_user_project(self): finally: retry_429_harder(created.delete)(force=True) + def test_rewrite_with_generation_match(self): + WRONG_GENERATION_NUMBER = 6 + BLOB_NAME = "generation-match" + + file_data = self.FILES["simple"] + new_bucket_name = "rewrite-generation-match" + unique_resource_id("-") + created = retry_429_503(Config.CLIENT.create_bucket)(new_bucket_name) + try: + bucket = Config.CLIENT.bucket(new_bucket_name) + + source = bucket.blob(BLOB_NAME) + source.upload_from_filename(file_data["path"]) + source_data = source.download_as_string() + + dest = bucket.blob(BLOB_NAME) + + with self.assertRaises(google.api_core.exceptions.PreconditionFailed): + token, rewritten, total = dest.rewrite( + source, if_generation_match=WRONG_GENERATION_NUMBER + ) + + token, rewritten, total = dest.rewrite( + source, + if_generation_match=dest.generation, + if_source_generation_match=source.generation, + if_source_metageneration_match=source.metageneration, + ) + self.assertEqual(token, None) + self.assertEqual(rewritten, len(source_data)) + self.assertEqual(total, len(source_data)) + self.assertEqual(dest.download_as_string(), source_data) + finally: + retry_429_harder(created.delete)(force=True) + class TestStorageUpdateStorageClass(TestStorageFiles): def test_update_storage_class_small_file(self): diff --git a/tests/unit/test__helpers.py b/tests/unit/test__helpers.py index 10b71b7bc..7f1f8998e 100644 --- a/tests/unit/test__helpers.py +++ b/tests/unit/test__helpers.py @@ -126,6 +126,42 @@ def test_reload(self): ) self.assertEqual(derived._changes, set()) + def test_reload_with_generation_match(self): + GENERATION_NUMBER = 9 + METAGENERATION_NUMBER = 6 + + connection = _Connection({"foo": "Foo"}) + client = _Client(connection) + derived = self._derivedClass("/path")() + # Make sure changes is not a set instance before calling reload + # (which will clear / replace it with an empty set), checked below. + derived._changes = object() + derived.reload( + client=client, + timeout=42, + if_generation_match=GENERATION_NUMBER, + if_metageneration_match=METAGENERATION_NUMBER, + ) + self.assertEqual(derived._properties, {"foo": "Foo"}) + kw = connection._requested + self.assertEqual(len(kw), 1) + self.assertEqual( + kw[0], + { + "method": "GET", + "path": "/path", + "query_params": { + "projection": "noAcl", + "ifGenerationMatch": GENERATION_NUMBER, + "ifMetagenerationMatch": METAGENERATION_NUMBER, + }, + "headers": {}, + "_target_object": derived, + "timeout": 42, + }, + ) + self.assertEqual(derived._changes, set()) + def test_reload_w_user_project(self): user_project = "user-project-123" connection = _Connection({"foo": "Foo"}) @@ -191,6 +227,46 @@ def test_patch(self): # Make sure changes get reset by patch(). self.assertEqual(derived._changes, set()) + def test_patch_with_metageneration_match(self): + GENERATION_NUMBER = 9 + METAGENERATION_NUMBER = 6 + + connection = _Connection({"foo": "Foo"}) + client = _Client(connection) + derived = self._derivedClass("/path")() + # Make sure changes is non-empty, so we can observe a change. + BAR = object() + BAZ = object() + derived._properties = {"bar": BAR, "baz": BAZ} + derived._changes = set(["bar"]) # Ignore baz. + derived.patch( + client=client, + timeout=42, + if_generation_match=GENERATION_NUMBER, + if_metageneration_match=METAGENERATION_NUMBER, + ) + self.assertEqual(derived._properties, {"foo": "Foo"}) + kw = connection._requested + self.assertEqual(len(kw), 1) + self.assertEqual( + kw[0], + { + "method": "PATCH", + "path": "/path", + "query_params": { + "projection": "full", + "ifGenerationMatch": GENERATION_NUMBER, + "ifMetagenerationMatch": METAGENERATION_NUMBER, + }, + # Since changes does not include `baz`, we don't see it sent. + "data": {"bar": BAR}, + "_target_object": derived, + "timeout": 42, + }, + ) + # Make sure changes get reset by patch(). + self.assertEqual(derived._changes, set()) + def test_patch_w_user_project(self): user_project = "user-project-123" connection = _Connection({"foo": "Foo"}) @@ -241,6 +317,34 @@ def test_update(self): # Make sure changes get reset by patch(). self.assertEqual(derived._changes, set()) + def test_update_with_metageneration_not_match(self): + GENERATION_NUMBER = 6 + + connection = _Connection({"foo": "Foo"}) + client = _Client(connection) + derived = self._derivedClass("/path")() + # Make sure changes is non-empty, so we can observe a change. + BAR = object() + BAZ = object() + derived._properties = {"bar": BAR, "baz": BAZ} + derived._changes = set(["bar"]) # Update sends 'baz' anyway. + derived.update( + client=client, timeout=42, if_metageneration_not_match=GENERATION_NUMBER + ) + self.assertEqual(derived._properties, {"foo": "Foo"}) + kw = connection._requested + self.assertEqual(len(kw), 1) + self.assertEqual(kw[0]["method"], "PUT") + self.assertEqual(kw[0]["path"], "/path") + self.assertEqual( + kw[0]["query_params"], + {"projection": "full", "ifMetagenerationNotMatch": GENERATION_NUMBER}, + ) + self.assertEqual(kw[0]["data"], {"bar": BAR, "baz": BAZ}) + self.assertEqual(kw[0]["timeout"], 42) + # Make sure changes get reset by patch(). + self.assertEqual(derived._changes, set()) + def test_update_w_user_project(self): user_project = "user-project-123" connection = _Connection({"foo": "Foo"}) @@ -343,6 +447,60 @@ def read(self, block_size): self.assertEqual(MD5.hash_obj._blocks, [BYTES_TO_SIGN]) +class Test__add_generation_match_parameters(unittest.TestCase): + def _call_fut(self, params, **match_params): + from google.cloud.storage._helpers import _add_generation_match_parameters + + return _add_generation_match_parameters(params, **match_params) + + def test_add_generation_match_parameters_list(self): + GENERATION_NUMBER = 9 + METAGENERATION_NUMBER = 6 + EXPECTED_PARAMS = [ + ("param1", "value1"), + ("param2", "value2"), + ("ifGenerationMatch", GENERATION_NUMBER), + ("ifMetagenerationMatch", METAGENERATION_NUMBER), + ] + params = [("param1", "value1"), ("param2", "value2")] + self._call_fut( + params, + if_generation_match=GENERATION_NUMBER, + if_metageneration_match=METAGENERATION_NUMBER, + ) + self.assertEqual(params, EXPECTED_PARAMS) + + def test_add_generation_match_parameters_dict(self): + GENERATION_NUMBER = 9 + METAGENERATION_NUMBER = 6 + EXPECTED_PARAMS = { + "param1": "value1", + "param2": "value2", + "ifGenerationMatch": GENERATION_NUMBER, + "ifMetagenerationMatch": METAGENERATION_NUMBER, + } + + params = {"param1": "value1", "param2": "value2"} + self._call_fut( + params, + if_generation_match=GENERATION_NUMBER, + if_metageneration_match=METAGENERATION_NUMBER, + ) + self.assertEqual(params, EXPECTED_PARAMS) + + def test_add_generation_match_parameters_tuple(self): + GENERATION_NUMBER = 9 + METAGENERATION_NUMBER = 6 + + params = (("param1", "value1"), ("param2", "value2")) + with self.assertRaises(ValueError): + self._call_fut( + params, + if_generation_match=GENERATION_NUMBER, + if_metageneration_match=METAGENERATION_NUMBER, + ) + + class _Connection(object): def __init__(self, *responses): self._responses = responses diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index bb1aa11e1..94822b93a 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -707,6 +707,39 @@ def test_exists_hit_w_generation(self): }, ) + def test_exists_w_generation_match(self): + BLOB_NAME = "blob-name" + GENERATION_NUMBER = 123456 + METAGENERATION_NUMBER = 6 + + found_response = ({"status": http_client.OK}, b"") + connection = _Connection(found_response) + client = _Client(connection) + bucket = _Bucket(client) + blob = self._make_one(BLOB_NAME, bucket=bucket) + bucket._blobs[BLOB_NAME] = 1 + self.assertTrue( + blob.exists( + if_generation_match=GENERATION_NUMBER, + if_metageneration_match=METAGENERATION_NUMBER, + ) + ) + self.assertEqual(len(connection._requested), 1) + self.assertEqual( + connection._requested[0], + { + "method": "GET", + "path": "/b/name/o/{}".format(BLOB_NAME), + "query_params": { + "fields": "name", + "ifGenerationMatch": GENERATION_NUMBER, + "ifMetagenerationMatch": METAGENERATION_NUMBER, + }, + "_target_object": None, + "timeout": self._get_default_timeout(), + }, + ) + def test_delete_wo_generation(self): BLOB_NAME = "blob-name" not_found_response = ({"status": http_client.NOT_FOUND}, b"") @@ -718,7 +751,19 @@ def test_delete_wo_generation(self): blob.delete() self.assertFalse(blob.exists()) self.assertEqual( - bucket._deleted, [(BLOB_NAME, None, None, self._get_default_timeout())] + bucket._deleted, + [ + ( + BLOB_NAME, + None, + None, + self._get_default_timeout(), + None, + None, + None, + None, + ) + ], ) def test_delete_w_generation(self): @@ -732,7 +777,25 @@ def test_delete_w_generation(self): bucket._blobs[BLOB_NAME] = 1 blob.delete(timeout=42) self.assertFalse(blob.exists()) - self.assertEqual(bucket._deleted, [(BLOB_NAME, None, GENERATION, 42)]) + self.assertEqual( + bucket._deleted, [(BLOB_NAME, None, GENERATION, 42, None, None, None, None)] + ) + + def test_delete_w_generation_match(self): + BLOB_NAME = "blob-name" + GENERATION = 123456 + not_found_response = ({"status": http_client.NOT_FOUND}, b"") + connection = _Connection(not_found_response) + client = _Client(connection) + bucket = _Bucket(client) + blob = self._make_one(BLOB_NAME, bucket=bucket, generation=GENERATION) + bucket._blobs[BLOB_NAME] = 1 + blob.delete(timeout=42, if_generation_match=GENERATION) + self.assertFalse(blob.exists()) + self.assertEqual( + bucket._deleted, + [(BLOB_NAME, None, GENERATION, 42, GENERATION, None, None, None)], + ) def test__get_transport(self): client = mock.Mock(spec=[u"_credentials", "_http"]) @@ -757,6 +820,24 @@ def test__get_download_url_with_media_link(self): self.assertEqual(download_url, media_link) + def test__get_download_url_with_generation_match(self): + GENERATION_NUMBER = 6 + MEDIA_LINK = "http://test.invalid" + + blob = self._make_one("something.txt", bucket=_Bucket(name="IRRELEVANT")) + # Set the media link on the blob + blob._properties["mediaLink"] = MEDIA_LINK + + client = mock.Mock(_connection=_Connection) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + download_url = blob._get_download_url( + client, if_generation_match=GENERATION_NUMBER + ) + self.assertEqual( + download_url, + "{}?ifGenerationMatch={}".format(MEDIA_LINK, GENERATION_NUMBER), + ) + def test__get_download_url_with_media_link_w_user_project(self): blob_name = "something.txt" user_project = "user-project-123" @@ -1038,6 +1119,28 @@ def test_download_to_file_wo_media_link(self): client._http, file_obj, expected_url, headers, None, None, False ) + def test_download_to_file_w_generation_match(self): + GENERATION_NUMBER = 6 + HEADERS = {"accept-encoding": "gzip"} + EXPECTED_URL = ( + "https://storage.googleapis.com/download/storage/v1/b/" + "name/o/blob-name?alt=media&ifGenerationNotMatch={}".format( + GENERATION_NUMBER + ) + ) + + client = mock.Mock(_connection=_Connection, spec=[u"_http"]) + client._connection.API_BASE_URL = "https://storage.googleapis.com" + blob = self._make_one("blob-name", bucket=_Bucket(client)) + blob._do_download = mock.Mock() + file_obj = io.BytesIO() + + blob.download_to_file(file_obj, if_generation_not_match=GENERATION_NUMBER) + + blob._do_download.assert_called_once_with( + client._http, file_obj, EXPECTED_URL, HEADERS, None, None, False + ) + def _download_to_file_helper(self, use_chunks, raw_download): blob_name = "blob-name" client = mock.Mock(spec=[u"_http"]) @@ -1108,6 +1211,28 @@ def _download_to_filename_helper(self, updated, raw_download): stream = blob._do_download.mock_calls[0].args[1] self.assertEqual(stream.name, temp.name) + def test_download_to_filename_w_generation_match(self): + from google.cloud._testing import _NamedTemporaryFile + + GENERATION_NUMBER = 6 + MEDIA_LINK = "http://example.com/media/" + EXPECTED_LINK = MEDIA_LINK + "?ifGenerationMatch={}".format(GENERATION_NUMBER) + HEADERS = {"accept-encoding": "gzip"} + + client = mock.Mock(spec=["_http"]) + + blob = self._make_one( + "blob-name", bucket=_Bucket(client), properties={"mediaLink": MEDIA_LINK} + ) + blob._do_download = mock.Mock() + + with _NamedTemporaryFile() as temp: + blob.download_to_filename(temp.name, if_generation_match=GENERATION_NUMBER) + + blob._do_download.assert_called_once_with( + client._http, mock.ANY, EXPECTED_LINK, HEADERS, None, None, False + ) + def test_download_to_filename_w_updated_wo_raw(self): updated = "2014-12-06T13:13:50.690Z" self._download_to_filename_helper(updated=updated, raw_download=False) @@ -1201,6 +1326,31 @@ def _download_as_string_helper(self, raw_download): stream = blob._do_download.mock_calls[0].args[1] self.assertIsInstance(stream, io.BytesIO) + def test_download_as_string_w_generation_match(self): + GENERATION_NUMBER = 6 + MEDIA_LINK = "http://example.com/media/" + + client = mock.Mock(spec=["_http"]) + blob = self._make_one( + "blob-name", bucket=_Bucket(client), properties={"mediaLink": MEDIA_LINK} + ) + blob.download_to_file = mock.Mock() + + fetched = blob.download_as_string(if_generation_match=GENERATION_NUMBER) + self.assertEqual(fetched, b"") + + blob.download_to_file.assert_called_once_with( + mock.ANY, + client=None, + start=None, + end=None, + raw_download=False, + if_generation_match=GENERATION_NUMBER, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, + ) + def test_download_as_string_wo_raw(self): self._download_as_string_helper(raw_download=False) @@ -2684,6 +2834,55 @@ def test_rewrite_w_generations(self): self.assertEqual(kw["query_params"], {"sourceGeneration": SOURCE_GENERATION}) self.assertEqual(kw["timeout"], 42) + def test_rewrite_w_generation_match(self): + SOURCE_BLOB = "source" + SOURCE_GENERATION_NUMBER = 42 + DEST_BLOB = "dest" + DEST_BUCKET = "other-bucket" + DEST_GENERATION_NUMBER = 16 + TOKEN = "TOKEN" + RESPONSE = { + "totalBytesRewritten": 33, + "objectSize": 42, + "done": False, + "rewriteToken": TOKEN, + } + response = ({"status": http_client.OK}, RESPONSE) + connection = _Connection(response) + client = _Client(connection) + source_bucket = _Bucket(client=client) + source_blob = self._make_one( + SOURCE_BLOB, bucket=source_bucket, generation=SOURCE_GENERATION_NUMBER + ) + dest_bucket = _Bucket(client=client, name=DEST_BUCKET) + dest_blob = self._make_one( + DEST_BLOB, bucket=dest_bucket, generation=DEST_GENERATION_NUMBER + ) + token, rewritten, size = dest_blob.rewrite( + source_blob, + timeout=42, + if_generation_match=dest_blob.generation, + if_source_generation_match=source_blob.generation, + ) + (kw,) = connection._requested + self.assertEqual(kw["method"], "POST") + self.assertEqual( + kw["path"], + "/b/%s/o/%s/rewriteTo/b/%s/o/%s" + % ( + (source_bucket.name, source_blob.name, dest_bucket.name, dest_blob.name) + ), + ) + self.assertEqual( + kw["query_params"], + { + "ifSourceGenerationMatch": SOURCE_GENERATION_NUMBER, + "ifGenerationMatch": DEST_GENERATION_NUMBER, + "sourceGeneration": SOURCE_GENERATION_NUMBER, + }, + ) + self.assertEqual(kw["timeout"], 42) + def test_rewrite_other_bucket_other_name_no_encryption_partial(self): SOURCE_BLOB = "source" DEST_BLOB = "dest" @@ -2992,6 +3191,45 @@ def test_update_storage_class_w_encryption_key_w_user_project(self): self.assertEqual(headers["X-Goog-Encryption-Key"], BLOB_KEY_B64) self.assertEqual(headers["X-Goog-Encryption-Key-Sha256"], BLOB_KEY_HASH_B64) + def test_update_storage_class_w_generation_match(self): + BLOB_NAME = "blob-name" + STORAGE_CLASS = u"NEARLINE" + GENERATION_NUMBER = 6 + SOURCE_GENERATION_NUMBER = 9 + RESPONSE = { + "totalBytesRewritten": 42, + "objectSize": 42, + "done": True, + "resource": {"storageClass": STORAGE_CLASS}, + } + response = ({"status": http_client.OK}, RESPONSE) + connection = _Connection(response) + client = _Client(connection) + bucket = _Bucket(client=client) + blob = self._make_one(BLOB_NAME, bucket=bucket) + + blob.update_storage_class( + "NEARLINE", + if_generation_match=GENERATION_NUMBER, + if_source_generation_match=SOURCE_GENERATION_NUMBER, + ) + self.assertEqual(blob.storage_class, "NEARLINE") + + kw = connection._requested + self.assertEqual(len(kw), 1) + self.assertEqual(kw[0]["method"], "POST") + PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) + self.assertEqual(kw[0]["path"], PATH) + self.assertEqual( + kw[0]["query_params"], + { + "ifGenerationMatch": GENERATION_NUMBER, + "ifSourceGenerationMatch": SOURCE_GENERATION_NUMBER, + }, + ) + SENT = {"storageClass": STORAGE_CLASS} + self.assertEqual(kw[0]["data"], SENT) + def test_cache_control_getter(self): BLOB_NAME = "blob-name" bucket = _Bucket() @@ -3616,9 +3854,30 @@ def __init__(self, client=None, name="name", user_project=None): self.path = "/b/" + name self.user_project = user_project - def delete_blob(self, blob_name, client=None, generation=None, timeout=None): + def delete_blob( + self, + blob_name, + client=None, + generation=None, + timeout=None, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, + ): del self._blobs[blob_name] - self._deleted.append((blob_name, client, generation, timeout)) + self._deleted.append( + ( + blob_name, + client, + generation, + timeout, + if_generation_match, + if_generation_not_match, + if_metageneration_match, + if_metageneration_not_match, + ) + ) class _Client(object): diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index 7bbcf73df..f270a7ec5 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -588,6 +588,40 @@ def api_request(cls, *args, **kwargs): expected_cw = [((), expected_called_kwargs)] self.assertEqual(_FakeConnection._called_with, expected_cw) + def test_exists_with_metageneration_match(self): + class _FakeConnection(object): + + _called_with = [] + + @classmethod + def api_request(cls, *args, **kwargs): + cls._called_with.append((args, kwargs)) + # exists() does not use the return value + return object() + + BUCKET_NAME = "bucket-name" + METAGENERATION_NUMBER = 6 + + bucket = self._make_one(name=BUCKET_NAME) + client = _Client(_FakeConnection) + self.assertTrue( + bucket.exists( + client=client, timeout=42, if_metageneration_match=METAGENERATION_NUMBER + ) + ) + expected_called_kwargs = { + "method": "GET", + "path": bucket.path, + "query_params": { + "fields": "name", + "ifMetagenerationMatch": METAGENERATION_NUMBER, + }, + "_target_object": None, + "timeout": 42, + } + expected_cw = [((), expected_called_kwargs)] + self.assertEqual(_FakeConnection._called_with, expected_cw) + def test_exists_hit_w_user_project(self): USER_PROJECT = "user-project-123" @@ -688,6 +722,26 @@ def test_get_blob_hit_w_generation(self): self.assertEqual(kw["query_params"], expected_qp) self.assertEqual(kw["timeout"], self._get_default_timeout()) + def test_get_blob_w_generation_match(self): + NAME = "name" + BLOB_NAME = "blob-name" + GENERATION = 1512565576797178 + + connection = _Connection({"name": BLOB_NAME, "generation": GENERATION}) + client = _Client(connection) + bucket = self._make_one(name=NAME) + blob = bucket.get_blob(BLOB_NAME, client=client, if_generation_match=GENERATION) + + self.assertIs(blob.bucket, bucket) + self.assertEqual(blob.name, BLOB_NAME) + self.assertEqual(blob.generation, GENERATION) + (kw,) = connection._requested + expected_qp = {"ifGenerationMatch": GENERATION, "projection": "noAcl"} + self.assertEqual(kw["method"], "GET") + self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) + self.assertEqual(kw["query_params"], expected_qp) + self.assertEqual(kw["timeout"], self._get_default_timeout()) + def test_get_blob_hit_with_kwargs(self): from google.cloud.storage.blob import _get_encryption_headers @@ -937,6 +991,31 @@ def test_delete_force_delete_blobs(self): ] self.assertEqual(connection._deleted_buckets, expected_cw) + def test_delete_with_metageneration_match(self): + NAME = "name" + BLOB_NAME1 = "blob-name1" + BLOB_NAME2 = "blob-name2" + GET_BLOBS_RESP = {"items": [{"name": BLOB_NAME1}, {"name": BLOB_NAME2}]} + DELETE_BLOB1_RESP = DELETE_BLOB2_RESP = {} + METAGENERATION_NUMBER = 6 + + connection = _Connection(GET_BLOBS_RESP, DELETE_BLOB1_RESP, DELETE_BLOB2_RESP) + connection._delete_bucket = True + client = _Client(connection) + bucket = self._make_one(client=client, name=NAME) + result = bucket.delete(if_metageneration_match=METAGENERATION_NUMBER) + self.assertIsNone(result) + expected_cw = [ + { + "method": "DELETE", + "path": bucket.path, + "query_params": {"ifMetagenerationMatch": METAGENERATION_NUMBER}, + "_target_object": None, + "timeout": self._get_default_timeout(), + } + ] + self.assertEqual(connection._deleted_buckets, expected_cw) + def test_delete_force_miss_blobs(self): NAME = "name" BLOB_NAME = "blob-name1" @@ -1019,6 +1098,31 @@ def test_delete_blob_hit_with_generation(self): self.assertEqual(kw["query_params"], {"generation": GENERATION}) self.assertEqual(kw["timeout"], self._get_default_timeout()) + def test_delete_blob_with_generation_match(self): + NAME = "name" + BLOB_NAME = "blob-name" + GENERATION = 6 + METAGENERATION = 9 + + connection = _Connection({}) + client = _Client(connection) + bucket = self._make_one(client=client, name=NAME) + result = bucket.delete_blob( + BLOB_NAME, + if_generation_match=GENERATION, + if_metageneration_match=METAGENERATION, + ) + + self.assertIsNone(result) + (kw,) = connection._requested + self.assertEqual(kw["method"], "DELETE") + self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) + self.assertEqual( + kw["query_params"], + {"ifGenerationMatch": GENERATION, "ifMetagenerationMatch": METAGENERATION}, + ) + self.assertEqual(kw["timeout"], self._get_default_timeout()) + def test_delete_blobs_empty(self): NAME = "name" connection = _Connection() @@ -1139,6 +1243,43 @@ def test_copy_blobs_source_generation(self): self.assertEqual(kw["query_params"], {"sourceGeneration": GENERATION}) self.assertEqual(kw["timeout"], self._get_default_timeout()) + def test_copy_blobs_w_generation_match(self): + SOURCE = "source" + DEST = "dest" + BLOB_NAME = "blob-name" + GENERATION_NUMBER = 6 + SOURCE_GENERATION_NUMBER = 9 + + connection = _Connection({}) + client = _Client(connection) + source = self._make_one(client=client, name=SOURCE) + dest = self._make_one(client=client, name=DEST) + blob = self._make_blob(SOURCE, BLOB_NAME) + + new_blob = source.copy_blob( + blob, + dest, + if_generation_match=GENERATION_NUMBER, + if_source_generation_match=SOURCE_GENERATION_NUMBER, + ) + self.assertIs(new_blob.bucket, dest) + self.assertEqual(new_blob.name, BLOB_NAME) + + (kw,) = connection._requested + COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( + SOURCE, BLOB_NAME, DEST, BLOB_NAME + ) + self.assertEqual(kw["method"], "POST") + self.assertEqual(kw["path"], COPY_PATH) + self.assertEqual( + kw["query_params"], + { + "ifGenerationMatch": GENERATION_NUMBER, + "ifSourceGenerationMatch": SOURCE_GENERATION_NUMBER, + }, + ) + self.assertEqual(kw["timeout"], self._get_default_timeout()) + def test_copy_blobs_preserve_acl(self): from google.cloud.storage.acl import ObjectACL diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index c38d87979..0ce3cad3c 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -436,6 +436,54 @@ def test_get_bucket_with_string_hit(self): timeout=self._get_default_timeout(), ) + def test_get_bucket_with_metageneration_match(self): + from google.cloud.storage.bucket import Bucket + + PROJECT = "PROJECT" + CREDENTIALS = _make_credentials() + METAGENERATION_NUMBER = 6 + client = self._make_one(project=PROJECT, credentials=CREDENTIALS) + + BUCKET_NAME = "bucket-name" + URI1 = "/".join( + [ + client._connection.API_BASE_URL, + "storage", + client._connection.API_VERSION, + "b", + "%s?projection=noAcl&ifMetagenerationMatch=%s" + % (BUCKET_NAME, METAGENERATION_NUMBER), + ] + ) + URI2 = "/".join( + [ + client._connection.API_BASE_URL, + "storage", + client._connection.API_VERSION, + "b", + "%s?ifMetagenerationMatch=%s&projection=noAcl" + % (BUCKET_NAME, METAGENERATION_NUMBER), + ] + ) + data = {"name": BUCKET_NAME} + http = _make_requests_session([_make_json_response(data)]) + client._http_internal = http + + bucket = client.get_bucket( + BUCKET_NAME, if_metageneration_match=METAGENERATION_NUMBER + ) + self.assertIsInstance(bucket, Bucket) + self.assertEqual(bucket.name, BUCKET_NAME) + http.request.assert_called_once_with( + method="GET", + url=mock.ANY, + data=mock.ANY, + headers=mock.ANY, + timeout=self._get_default_timeout(), + ) + _, kwargs = http.request.call_args + self.assertIn(kwargs.get("url"), (URI1, URI2)) + def test_get_bucket_with_object_miss(self): from google.cloud.exceptions import NotFound from google.cloud.storage.bucket import Bucket @@ -566,6 +614,54 @@ def test_lookup_bucket_hit(self): timeout=self._get_default_timeout(), ) + def test_lookup_bucket_with_metageneration_match(self): + from google.cloud.storage.bucket import Bucket + + PROJECT = "PROJECT" + CREDENTIALS = _make_credentials() + METAGENERATION_NUMBER = 6 + client = self._make_one(project=PROJECT, credentials=CREDENTIALS) + + BUCKET_NAME = "bucket-name" + URI1 = "/".join( + [ + client._connection.API_BASE_URL, + "storage", + client._connection.API_VERSION, + "b", + "%s?projection=noAcl&ifMetagenerationMatch=%s" + % (BUCKET_NAME, METAGENERATION_NUMBER), + ] + ) + URI2 = "/".join( + [ + client._connection.API_BASE_URL, + "storage", + client._connection.API_VERSION, + "b", + "%s?ifMetagenerationMatch=%s&projection=noAcl" + % (BUCKET_NAME, METAGENERATION_NUMBER), + ] + ) + data = {"name": BUCKET_NAME} + http = _make_requests_session([_make_json_response(data)]) + client._http_internal = http + + bucket = client.lookup_bucket( + BUCKET_NAME, if_metageneration_match=METAGENERATION_NUMBER + ) + self.assertIsInstance(bucket, Bucket) + self.assertEqual(bucket.name, BUCKET_NAME) + http.request.assert_called_once_with( + method="GET", + url=mock.ANY, + data=mock.ANY, + headers=mock.ANY, + timeout=self._get_default_timeout(), + ) + _, kwargs = http.request.call_args + self.assertIn(kwargs.get("url"), (URI1, URI2)) + def test_create_bucket_w_missing_client_project(self): credentials = _make_credentials() client = self._make_one(project=None, credentials=credentials)