Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: rename 'Blob.download_as_{string,bytes}', add 'Blob.download_as_text' #182

Merged
merged 10 commits into from Aug 10, 2020
104 changes: 98 additions & 6 deletions google/cloud/storage/blob.py
Expand Up @@ -1085,10 +1085,6 @@ def download_as_bytes(
):
"""Download the contents of this blob as a bytes object.

.. note::
The method only supports `python3`, for python2 it returns data
as a string.

If :attr:`user_project` is set on the bucket, bills the API request
to that project.

Expand Down Expand Up @@ -1165,6 +1161,85 @@ def download_as_string(
if_generation_not_match=None,
if_metageneration_match=None,
if_metageneration_not_match=None,
timeout=_DEFAULT_TIMEOUT,
):
"""Download the contents of this blob as a bytes object.
tseaver marked this conversation as resolved.
Show resolved Hide resolved

If :attr:`user_project` is set on the bucket, bills the API request
to that project.

:type client: :class:`~google.cloud.storage.client.Client` or
``NoneType``
:param client: (Optional) The client to use. If not passed, falls back
to the ``client`` stored on the blob's bucket.

:type start: int
:param start: (Optional) The first byte in a range to be downloaded.

:type end: int
:param end: (Optional) The last byte in a range to be downloaded.

:type raw_download: bool
:param raw_download:
(Optional) If true, download the object without any expansion.

:type if_generation_match: long
:param if_generation_match: (Optional) Make the operation conditional on whether
the blob's current generation matches the given value.
Setting to 0 makes the operation succeed only if there
are no live versions of the blob.

:type if_generation_not_match: long
:param if_generation_not_match: (Optional) Make the operation conditional on whether
the blob's current generation does not match the given
value. If no live blob exists, the precondition fails.
Setting to 0 makes the operation succeed only if there
is a live version of the blob.

:param if_metageneration_match: (Optional) Make the operation conditional on whether the
blob's current metageneration matches the given value.

:type if_metageneration_not_match: long
:param if_metageneration_not_match: (Optional) Make the operation conditional on whether the
blob's current metageneration does not match the given value.

:type timeout: float or tuple
:param timeout:
(Optional) The number of seconds the transport should wait for the
server response. Depending on the retry strategy, a request may be
repeated several times using the same timeout each time.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.

:rtype: bytes
:returns: The data stored in this blob.

:raises: :class:`google.cloud.exceptions.NotFound`
"""
tseaver marked this conversation as resolved.
Show resolved Hide resolved
return self.download_as_bytes(
client=client,
start=start,
end=end,
raw_download=raw_download,
if_generation_match=if_generation_match,
if_generation_not_match=if_generation_not_match,
if_metageneration_match=if_metageneration_match,
if_metageneration_not_match=if_metageneration_not_match,
timeout=timeout,
)

def download_as_text(
self,
client=None,
start=None,
end=None,
raw_download=False,
encoding="utf-8",
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
if_metageneration_not_match=None,
timeout=_DEFAULT_TIMEOUT,
):
"""Download the contents of this blob as a string.

Expand All @@ -1186,6 +1261,11 @@ def download_as_string(
:param raw_download:
(Optional) If true, download the object without any expansion.

:type encoding: str
:param encoding: (Optional) The data of the blob will be decoded by
encoding method. Defaults to UTF-8. Apply only
if the value of ``blob.content_encoding`` is None.

:type if_generation_match: long
:param if_generation_match: (Optional) Make the operation conditional on whether
the blob's current generation matches the given value.
Expand All @@ -1206,7 +1286,15 @@ def download_as_string(
:param if_metageneration_not_match: (Optional) Make the operation conditional on whether the
blob's current metageneration does not match the given value.

:rtype: str
:type timeout: float or tuple
:param timeout:
(Optional) The number of seconds the transport should wait for the
server response. Depending on the retry strategy, a request may be
repeated several times using the same timeout each time.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.

:rtype: text
:returns: The data stored in this blob.

:raises: :class:`google.cloud.exceptions.NotFound`
Expand All @@ -1220,11 +1308,15 @@ def download_as_string(
if_generation_not_match=if_generation_not_match,
if_metageneration_match=if_metageneration_match,
if_metageneration_not_match=if_metageneration_not_match,
timeout=timeout,
)
if six.PY2:
tseaver marked this conversation as resolved.
Show resolved Hide resolved
return data
tseaver marked this conversation as resolved.
Show resolved Hide resolved
else:
return data.decode()
if self.content_encoding:
return data.decode(self.content_encoding)
else:
return data.decode(encoding)

def _get_content_type(self, content_type, filename=None):
"""Determine the content type from the current object.
Expand Down
2 changes: 1 addition & 1 deletion tests/perf/benchwrapper.py
Expand Up @@ -35,7 +35,7 @@ def Write(self, request, context):
def Read(self, request, context):
bucket = client.bucket(request.bucketName)
blob = storage.Blob(request.objectName, bucket)
blob.download_as_bytes()
blob.download_as_string()
return storage_pb2.EmptyResponse()


Expand Down
4 changes: 2 additions & 2 deletions tests/system/test_system.py
Expand Up @@ -845,13 +845,13 @@ def test_download_blob_w_uri(self):

self.assertEqual(file_contents, stored_contents)

def test_download_blob_as_string(self):
def test_download_blob_as_text(self):
blob = self.bucket.blob("MyBuffer")
file_contents = "Hello World"
blob.upload_from_string(file_contents)
self.case_blobs_to_delete.append(blob)

stored_contents = blob.download_as_string()
stored_contents = blob.download_as_text()
self.assertEqual(file_contents, stored_contents)

def test_upload_gzip_encoded_download_raw(self):
Expand Down
78 changes: 65 additions & 13 deletions tests/unit/test_blob.py
Expand Up @@ -1417,7 +1417,7 @@ def test_download_to_filename_w_key(self):
stream = blob._do_download.mock_calls[0].args[1]
self.assertEqual(stream.name, temp.name)

def _download_as_bytes_helper(self, raw_download):
def _download_as_bytes_helper(self, raw_download, timeout=None):
blob_name = "blob-name"
client = mock.Mock(spec=["_http"])
bucket = _Bucket(client)
Expand All @@ -1426,12 +1426,24 @@ def _download_as_bytes_helper(self, raw_download):
blob = self._make_one(blob_name, bucket=bucket, properties=properties)
blob._do_download = mock.Mock()

fetched = blob.download_as_bytes(raw_download=raw_download)
if timeout is None:
expected_timeout = self._get_default_timeout()
fetched = blob.download_as_bytes(raw_download=raw_download)
else:
expected_timeout = timeout
fetched = blob.download_as_bytes(raw_download=raw_download, timeout=timeout)
self.assertEqual(fetched, b"")

headers = {"accept-encoding": "gzip"}
blob._do_download.assert_called_once_with(
client._http, mock.ANY, media_link, headers, None, None, raw_download
client._http,
mock.ANY,
media_link,
headers,
None,
None,
raw_download,
timeout=expected_timeout,
)
stream = blob._do_download.mock_calls[0].args[1]
self.assertIsInstance(stream, io.BytesIO)
Expand Down Expand Up @@ -1459,6 +1471,7 @@ def test_download_as_bytes_w_generation_match(self):
if_generation_not_match=None,
if_metageneration_match=None,
if_metageneration_not_match=None,
timeout=self._get_default_timeout(),
)

def test_download_as_bytes_wo_raw(self):
Expand All @@ -1467,16 +1480,27 @@ def test_download_as_bytes_wo_raw(self):
def test_download_as_bytes_w_raw(self):
self._download_as_bytes_helper(raw_download=True)

def _download_as_string_helper(self, raw_download):
def test_download_as_byte_w_custom_timeout(self):
self._download_as_bytes_helper(raw_download=False, timeout=9.58)

def _download_as_text_helper(self, raw_download, encoding=None, timeout=None):
blob_name = "blob-name"
client = mock.Mock(spec=["_http"])
bucket = _Bucket(client)
media_link = "http://example.com/media/"
properties = {"mediaLink": media_link}
if encoding:
properties["contentEncoding"] = encoding
blob = self._make_one(blob_name, bucket=bucket, properties=properties)
blob._do_download = mock.Mock()

fetched = blob.download_as_string(raw_download=raw_download)
if timeout is None:
expected_timeout = self._get_default_timeout()
fetched = blob.download_as_text(raw_download=raw_download)
else:
expected_timeout = timeout
fetched = blob.download_as_text(raw_download=raw_download, timeout=timeout)

self.assertEqual(fetched, "")

headers = {"accept-encoding": "gzip"}
Expand All @@ -1493,7 +1517,7 @@ def _download_as_string_helper(self, raw_download):
stream = blob._do_download.mock_calls[0].args[1]
self.assertIsInstance(stream, io.BytesIO)

def test_download_as_string_w_generation_match(self):
def test_download_as_text_w_generation_match(self):
GENERATION_NUMBER = 6
MEDIA_LINK = "http://example.com/media/"

Expand All @@ -1503,7 +1527,7 @@ def test_download_as_string_w_generation_match(self):
)
blob.download_to_file = mock.Mock()

fetched = blob.download_as_string(if_generation_match=GENERATION_NUMBER)
fetched = blob.download_as_text(if_generation_match=GENERATION_NUMBER)
self.assertEqual(fetched, "")

blob.download_to_file.assert_called_once_with(
Expand All @@ -1519,14 +1543,42 @@ def test_download_as_string_w_generation_match(self):
timeout=self._get_default_timeout(),
)

def test_download_as_string_wo_raw(self):
self._download_as_string_helper(raw_download=False)
def test_download_as_text_wo_raw(self):
self._download_as_text_helper(raw_download=False)

def test_download_as_text_w_raw(self):
self._download_as_text_helper(raw_download=True)

def test_download_as_text_w_custom_timeout(self):
self._download_as_text_helper(raw_download=False, timeout=9.58)

def test_download_as_string_w_raw(self):
self._download_as_string_helper(raw_download=True)
def test_download_as_text_w_encoding(self):
self._download_as_text_helper(raw_download=False, encoding="utf-8")

def test_download_as_string_w_custom_timeout(self):
self._download_as_string_helper(raw_download=False, timeout=9.58)
def test_download_as_string(self):
MEDIA_LINK = "http://example.com/media/"

client = mock.Mock(spec=["_http"])
blob = self._make_one(
"blob-name", bucket=_Bucket(client), properties={"mediaLink": MEDIA_LINK}
)
blob.download_to_file = mock.Mock()

fetched = blob.download_as_string()
self.assertEqual(fetched, b"")

blob.download_to_file.assert_called_once_with(
mock.ANY,
client=None,
start=None,
end=None,
raw_download=False,
if_generation_match=None,
if_generation_not_match=None,
if_metageneration_match=None,
if_metageneration_not_match=None,
timeout=self._get_default_timeout(),
)

def test__get_content_type_explicit(self):
blob = self._make_one(u"blob-name", bucket=None)
Expand Down
You are viewing a condensed version of this merge commit. You can view the full changes here.