Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add *generation*match args into Blob.compose() #122

Merged
merged 14 commits into from Jun 9, 2020
Merged
45 changes: 41 additions & 4 deletions google/cloud/storage/blob.py
Expand Up @@ -2006,34 +2006,71 @@ def make_private(self, client=None):
self.acl.all().revoke_read()
self.acl.save(client=client)

def compose(self, sources, client=None, timeout=_DEFAULT_TIMEOUT):
def compose(
self,
sources,
client=None,
timeout=_DEFAULT_TIMEOUT,
if_generation_match=None,
if_metageneration_match=None,
):
"""Concatenate source blobs into this one.

If :attr:`user_project` is set on the bucket, bills the API request
to that project.

:type sources: list of :class:`Blob`
:param sources: blobs whose contents will be composed into this blob.
:param sources: Blobs whose contents will be composed into this blob.

:type client: :class:`~google.cloud.storage.client.Client` or
``NoneType``
:param client: (Optional) The client to use. If not passed, falls back
:param client: (Optional) The client to use. If not passed, falls back
to the ``client`` stored on the blob's bucket.

:type timeout: float or tuple
:param timeout: (Optional) The amount of time, in seconds, to wait
for the server response.

Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.

:type if_generation_match: list of long
IlyaFaer marked this conversation as resolved.
Show resolved Hide resolved
:param if_generation_match: (Optional) Make the operation conditional on whether
the blob's current generation matches the given value.
Setting to 0 makes the operation succeed only if there
are no live versions of the blob. Generation numbers list
should have the same order that ``sources`` have.

:type if_metageneration_match: list of long
:param if_metageneration_match: (Optional) Make the operation conditional on whether the
blob's current metageneration matches the given value.
Metageneration numbers list should have the same order
that ``sources`` have.
"""
client = self._require_client(client)
query_params = {}

if self.user_project is not None:
query_params["userProject"] = self.user_project

source_objects = []
for index, source in enumerate(sources):
source_object = {"name": source.name}

preconditions = {}
if if_generation_match is not None:
preconditions["ifGenerationMatch"] = if_generation_match[index]
IlyaFaer marked this conversation as resolved.
Show resolved Hide resolved

if if_metageneration_match is not None:
preconditions["ifMetagenerationMatch"] = if_metageneration_match[index]

if preconditions:
source_object["objectPreconditions"] = preconditions

source_objects.append(source_object)

request = {
"sourceObjects": [{"name": source.name} for source in sources],
"sourceObjects": source_objects,
"destination": self._properties.copy(),
}
api_response = client._connection.api_request(
Expand Down
21 changes: 21 additions & 0 deletions tests/system/test_system.py
Expand Up @@ -1234,6 +1234,27 @@ def test_compose_replace_existing_blob(self):
composed = original.download_as_string()
self.assertEqual(composed, BEFORE + TO_APPEND)

def test_compose_with_generation_match(self):
IlyaFaer marked this conversation as resolved.
Show resolved Hide resolved
BEFORE = b"AAA\n"
original = self.bucket.blob("original")
original.content_type = "text/plain"
original.upload_from_string(BEFORE)
self.case_blobs_to_delete.append(original)

TO_APPEND = b"BBB\n"
to_append = self.bucket.blob("to_append")
to_append.upload_from_string(TO_APPEND)
self.case_blobs_to_delete.append(to_append)

original.compose(
[original, to_append],
if_generation_match=[original.generation, to_append.generation],
if_metageneration_match=[original.metageneration, to_append.metageneration],
)

composed = original.download_as_string()
self.assertEqual(composed, BEFORE + TO_APPEND)

@unittest.skipUnless(USER_PROJECT, "USER_PROJECT not set in environment.")
def test_compose_with_user_project(self):
new_bucket_name = "compose-user-project" + unique_resource_id("-")
Expand Down
60 changes: 57 additions & 3 deletions tests/unit/test_blob.py
Expand Up @@ -2507,7 +2507,7 @@ def test_make_private(self):
def test_compose_wo_content_type_set(self):
SOURCE_1 = "source-1"
SOURCE_2 = "source-2"
DESTINATION = "destinaton"
DESTINATION = "destination"
RESOURCE = {}
after = ({"status": http_client.OK}, RESOURCE)
connection = _Connection(after)
Expand Down Expand Up @@ -2542,7 +2542,7 @@ def test_compose_wo_content_type_set(self):
def test_compose_minimal_w_user_project(self):
SOURCE_1 = "source-1"
SOURCE_2 = "source-2"
DESTINATION = "destinaton"
DESTINATION = "destination"
RESOURCE = {"etag": "DEADBEEF"}
USER_PROJECT = "user-project-123"
after = ({"status": http_client.OK}, RESOURCE)
Expand Down Expand Up @@ -2578,7 +2578,7 @@ def test_compose_minimal_w_user_project(self):
def test_compose_w_additional_property_changes(self):
SOURCE_1 = "source-1"
SOURCE_2 = "source-2"
DESTINATION = "destinaton"
DESTINATION = "destination"
RESOURCE = {"etag": "DEADBEEF"}
after = ({"status": http_client.OK}, RESOURCE)
connection = _Connection(after)
Expand Down Expand Up @@ -2616,6 +2616,60 @@ def test_compose_w_additional_property_changes(self):
},
)

def test_compose_w_generation_match(self):
SOURCE_1 = "source-1"
SOURCE_2 = "source-2"
DESTINATION = "destination"
RESOURCE = {}
GENERATION_NUMBERS = [6, 9]
METAGENERATION_NUMBERS = [7, 1]

after = ({"status": http_client.OK}, RESOURCE)
connection = _Connection(after)
client = _Client(connection)
bucket = _Bucket(client=client)
source_1 = self._make_one(SOURCE_1, bucket=bucket)
source_2 = self._make_one(SOURCE_2, bucket=bucket)

destination = self._make_one(DESTINATION, bucket=bucket)
destination.compose(
sources=[source_1, source_2],
if_generation_match=GENERATION_NUMBERS,
if_metageneration_match=METAGENERATION_NUMBERS,
)

kw = connection._requested
self.assertEqual(len(kw), 1)
self.assertEqual(
kw[0],
{
"method": "POST",
"path": "/b/name/o/%s/compose" % DESTINATION,
"query_params": {},
"data": {
"sourceObjects": [
{
"name": source_1.name,
"objectPreconditions": {
"ifGenerationMatch": GENERATION_NUMBERS[0],
"ifMetagenerationMatch": METAGENERATION_NUMBERS[0],
},
},
{
"name": source_2.name,
"objectPreconditions": {
"ifGenerationMatch": GENERATION_NUMBERS[1],
"ifMetagenerationMatch": METAGENERATION_NUMBERS[1],
},
},
],
"destination": {},
},
"_target_object": destination,
"timeout": self._get_default_timeout(),
},
)

def test_rewrite_response_without_resource(self):
SOURCE_BLOB = "source"
DEST_BLOB = "dest"
Expand Down