From 5e60991c634bf5f5f3ddd78a6bd192f03b858a15 Mon Sep 17 00:00:00 2001 From: Gurov Ilya Date: Mon, 15 Jun 2020 23:44:09 +0300 Subject: [PATCH] feat: add if*generation*match args into Bucket.delete_blobs() (#130) Towards #127 --- google/cloud/storage/bucket.py | 93 +++++++++++++++++++++++++++++++++- tests/unit/test_bucket.py | 76 +++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 2 deletions(-) diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index 228c0e2aa..2e88f5a84 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -1455,7 +1455,17 @@ def delete_blob( timeout=timeout, ) - def delete_blobs(self, blobs, on_error=None, client=None, timeout=_DEFAULT_TIMEOUT): + def delete_blobs( + self, + blobs, + on_error=None, + client=None, + timeout=_DEFAULT_TIMEOUT, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, + ): """Deletes a list of blobs from the current bucket. Uses :meth:`delete_blob` to delete each individual blob. @@ -1484,15 +1494,74 @@ def delete_blobs(self, blobs, on_error=None, client=None, timeout=_DEFAULT_TIMEO Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + :type if_generation_match: list of long + :param if_generation_match: (Optional) Make the operation conditional on whether + the blob's current generation matches the given value. + Setting to 0 makes the operation succeed only if there + are no live versions of the blob. The list must match + ``blobs`` item-to-item. + + :type if_generation_not_match: list of long + :param if_generation_not_match: (Optional) Make the operation conditional on whether + the blob's current generation does not match the given + value. If no live blob exists, the precondition fails. + Setting to 0 makes the operation succeed only if there + is a live version of the blob. The list must match + ``blobs`` item-to-item. + + :type if_metageneration_match: list of long + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + The list must match ``blobs`` item-to-item. + + :type if_metageneration_not_match: list of long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. + The list must match ``blobs`` item-to-item. + :raises: :class:`~google.cloud.exceptions.NotFound` (if `on_error` is not passed). + + Example: + Delete blobs using generation match preconditions. + + >>> from google.cloud import storage + + >>> client = storage.Client() + >>> bucket = client.bucket("bucket-name") + + >>> blobs = [bucket.blob("blob-name-1"), bucket.blob("blob-name-2")] + >>> if_generation_match = [None] * len(blobs) + >>> if_generation_match[0] = "123" # precondition for "blob-name-1" + + >>> bucket.delete_blobs(blobs, if_generation_match=if_generation_match) """ + _raise_if_len_differs( + len(blobs), + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) + if_generation_match = iter(if_generation_match or []) + if_generation_not_match = iter(if_generation_not_match or []) + if_metageneration_match = iter(if_metageneration_match or []) + if_metageneration_not_match = iter(if_metageneration_not_match or []) + for blob in blobs: try: blob_name = blob if not isinstance(blob_name, six.string_types): blob_name = blob.name - self.delete_blob(blob_name, client=client, timeout=timeout) + self.delete_blob( + blob_name, + client=client, + timeout=timeout, + if_generation_match=next(if_generation_match, None), + if_generation_not_match=next(if_generation_not_match, None), + if_metageneration_match=next(if_metageneration_match, None), + if_metageneration_not_match=next(if_metageneration_not_match, None), + ) except NotFound: if on_error is not None: on_error(blob) @@ -2980,3 +3049,23 @@ def generate_signed_url( headers=headers, query_parameters=query_parameters, ) + + +def _raise_if_len_differs(expected_len, **generation_match_args): + """ + Raise an error if any generation match argument + is set and its len differs from the given value. + + :type expected_len: int + :param expected_len: Expected argument length in case it's set. + + :type generation_match_args: dict + :param generation_match_args: Lists, which length must be checked. + + :raises: :exc:`ValueError` if any argument set, but has an unexpected length. + """ + for name, value in generation_match_args.items(): + if value is not None and len(value) != expected_len: + raise ValueError( + "'{}' length must be the same as 'blobs' length".format(name) + ) diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index 27bd94f1a..3c5f2e68d 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -1146,6 +1146,82 @@ def test_delete_blobs_hit_w_user_project(self): self.assertEqual(kw[0]["query_params"], {"userProject": USER_PROJECT}) self.assertEqual(kw[0]["timeout"], 42) + def test_delete_blobs_w_generation_match(self): + NAME = "name" + BLOB_NAME = "blob-name" + BLOB_NAME2 = "blob-name2" + GENERATION_NUMBER = 6 + GENERATION_NUMBER2 = 9 + + connection = _Connection({}, {}) + client = _Client(connection) + bucket = self._make_one(client=client, name=NAME) + bucket.delete_blobs( + [BLOB_NAME, BLOB_NAME2], + timeout=42, + if_generation_match=[GENERATION_NUMBER, GENERATION_NUMBER2], + ) + kw = connection._requested + self.assertEqual(len(kw), 2) + + self.assertEqual(kw[0]["method"], "DELETE") + self.assertEqual(kw[0]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) + self.assertEqual(kw[0]["timeout"], 42) + self.assertEqual( + kw[0]["query_params"], {"ifGenerationMatch": GENERATION_NUMBER} + ) + self.assertEqual(kw[1]["method"], "DELETE") + self.assertEqual(kw[1]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME2)) + self.assertEqual(kw[1]["timeout"], 42) + self.assertEqual( + kw[1]["query_params"], {"ifGenerationMatch": GENERATION_NUMBER2} + ) + + def test_delete_blobs_w_generation_match_wrong_len(self): + NAME = "name" + BLOB_NAME = "blob-name" + BLOB_NAME2 = "blob-name2" + GENERATION_NUMBER = 6 + + connection = _Connection() + client = _Client(connection) + bucket = self._make_one(client=client, name=NAME) + with self.assertRaises(ValueError): + bucket.delete_blobs( + [BLOB_NAME, BLOB_NAME2], + timeout=42, + if_generation_not_match=[GENERATION_NUMBER], + ) + + def test_delete_blobs_w_generation_match_none(self): + NAME = "name" + BLOB_NAME = "blob-name" + BLOB_NAME2 = "blob-name2" + GENERATION_NUMBER = 6 + GENERATION_NUMBER2 = None + + connection = _Connection({}, {}) + client = _Client(connection) + bucket = self._make_one(client=client, name=NAME) + bucket.delete_blobs( + [BLOB_NAME, BLOB_NAME2], + timeout=42, + if_generation_match=[GENERATION_NUMBER, GENERATION_NUMBER2], + ) + kw = connection._requested + self.assertEqual(len(kw), 2) + + self.assertEqual(kw[0]["method"], "DELETE") + self.assertEqual(kw[0]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) + self.assertEqual(kw[0]["timeout"], 42) + self.assertEqual( + kw[0]["query_params"], {"ifGenerationMatch": GENERATION_NUMBER} + ) + self.assertEqual(kw[1]["method"], "DELETE") + self.assertEqual(kw[1]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME2)) + self.assertEqual(kw[1]["timeout"], 42) + self.assertEqual(kw[1]["query_params"], {}) + def test_delete_blobs_miss_no_on_error(self): from google.cloud.exceptions import NotFound