feat: media operation retries can be configured using the same interface as with non-media operation

google/cloud/storage/client.py

@@ -1259,6 +1259,7 @@ def list_blobs(
             max_results=max_results,
             extra_params=extra_params,
             page_start=_blobs_page_start,
+            timeout=timeout,
         )
         iterator.bucket = bucket
         iterator.prefixes = set()

tests/unit/test__helpers.py

@@ -632,17 +632,6 @@ def test_none(self):
         self.assertEqual(retry_strategy.max_retries, 0)
 
 
-class _Connection(object):
-    def __init__(self, *responses):
-        self._responses = responses
-        self._requested = []
-
-    def api_request(self, **kw):
-        self._requested.append(kw)
-        response, self._responses = self._responses[0], self._responses[1:]
-        return response
-
-
 class _MD5Hash(object):
     def __init__(self, digest_val):
         self.digest_val = digest_val

tests/unit/test_blob.py

@@ -1022,6 +1022,99 @@ def _mock_requests_response(status_code, headers, content=b""):
         response.request = requests.Request("POST", "http://example.com").prepare()
         return response
 
+    def test__extract_headers_from_download_gzipped(self):
+        blob_name = "blob-name"
+        client = mock.Mock(spec=["_http"])
+        bucket = _Bucket(client)
+        blob = self._make_one(blob_name, bucket=bucket)
+
+        response = self._mock_requests_response(
+            http_client.OK,
+            headers={
+                "Content-Type": "application/json",
+                "Content-Language": "ko-kr",
+                "Cache-Control": "max-age=1337;public",
+                "Content-Encoding": "gzip",
+                "X-Goog-Storage-Class": "STANDARD",
+                "X-Goog-Hash": "crc32c=4gcgLQ==,md5=CS9tHYTtyFntzj7B9nkkJQ==",
+            },
+            # { "x": 5 } gzipped
+            content=b"\x1f\x8b\x08\x00\xcfo\x17_\x02\xff\xabVP\xaaP\xb2R0U\xa8\x05\x00\xa1\xcaQ\x93\n\x00\x00\x00",
+        )
+        blob._extract_headers_from_download(response)
+
+        self.assertEqual(blob.content_type, "application/json")
+        self.assertEqual(blob.content_language, "ko-kr")
+        self.assertEqual(blob.content_encoding, "gzip")
+        self.assertEqual(blob.cache_control, "max-age=1337;public")
+        self.assertEqual(blob.storage_class, "STANDARD")
+        self.assertEqual(blob.md5_hash, "CS9tHYTtyFntzj7B9nkkJQ==")
+        self.assertEqual(blob.crc32c, "4gcgLQ==")
+
+    def test__extract_headers_from_download_empty(self):
+        blob_name = "blob-name"
+        client = mock.Mock(spec=["_http"])
+        bucket = _Bucket(client)
+        blob = self._make_one(blob_name, bucket=bucket)
+
+        response = self._mock_requests_response(
+            http_client.OK,
+            headers={
+                "Content-Type": "application/octet-stream",
+                "Content-Language": "en-US",
+                "Cache-Control": "max-age=1337;public",
+                "Content-Encoding": "gzip",
+                "X-Goog-Storage-Class": "STANDARD",
+                "X-Goog-Hash": "crc32c=4/c+LQ==,md5=CS9tHYTt/+ntzj7B9nkkJQ==",
+            },
+            content=b"",
+        )
+        blob._extract_headers_from_download(response)
+        self.assertEqual(blob.content_type, "application/octet-stream")
+        self.assertEqual(blob.content_language, "en-US")
+        self.assertEqual(blob.md5_hash, "CS9tHYTt/+ntzj7B9nkkJQ==")
+        self.assertEqual(blob.crc32c, "4/c+LQ==")
+
+    def test__extract_headers_from_download_w_hash_response_header_none(self):
+        blob_name = "blob-name"
+        md5_hash = "CS9tHYTtyFntzj7B9nkkJQ=="
+        crc32c = "4gcgLQ=="
+        client = mock.Mock(spec=["_http"])
+        bucket = _Bucket(client)
+        properties = {
+            "md5Hash": md5_hash,
+            "crc32c": crc32c,
+        }
+        blob = self._make_one(blob_name, bucket=bucket, properties=properties)
+
+        response = self._mock_requests_response(
+            http_client.OK,
+            headers={"X-Goog-Hash": ""},
+            # { "x": 5 } gzipped
+            content=b"\x1f\x8b\x08\x00\xcfo\x17_\x02\xff\xabVP\xaaP\xb2R0U\xa8\x05\x00\xa1\xcaQ\x93\n\x00\x00\x00",
+        )
+        blob._extract_headers_from_download(response)
+
+        self.assertEqual(blob.md5_hash, md5_hash)
+        self.assertEqual(blob.crc32c, crc32c)
+
+    def test__extract_headers_from_download_w_response_headers_not_match(self):
+        blob_name = "blob-name"
+        client = mock.Mock(spec=["_http"])
+        bucket = _Bucket(client)
+        blob = self._make_one(blob_name, bucket=bucket)
+
+        response = self._mock_requests_response(
+            http_client.OK,
+            headers={"X-Goog-Hash": "bogus=4gcgLQ==,"},
+            # { "x": 5 } gzipped
+            content=b"",
+        )
+        blob._extract_headers_from_download(response)
+
+        self.assertIsNone(blob.md5_hash)
+        self.assertIsNone(blob.crc32c)
+
     def _do_download_helper_wo_chunks(
         self, w_range, raw_download, timeout=None, **extra_kwargs
     ):
@@ -1342,9 +1435,8 @@ def _download_to_file_helper(
             blob.download_to_file(file_obj, **extra_kwargs)
 
         expected_retry = extra_kwargs.get("retry", DEFAULT_RETRY)
-        headers = {"accept-encoding": "gzip"}
-        blob._do_download.assert_called_once_with(
-            client._http,
+        client.download_blob_to_file.assert_called_once_with(
+            blob,
             file_obj,
             start=None,
             end=None,
@@ -1421,9 +1513,8 @@ def _download_to_filename_helper(
 
         expected_retry = extra_kwargs.get("retry", DEFAULT_RETRY)
 
-        headers = {"accept-encoding": "gzip"}
-        blob._do_download.assert_called_once_with(
-            client._http,
+        client.download_blob_to_file.assert_called_once_with(
+            blob,
             mock.ANY,
             start=None,
             end=None,
@@ -1439,33 +1530,6 @@ def _download_to_filename_helper(
         stream = client.download_blob_to_file.mock_calls[0].args[1]
         self.assertEqual(stream.name, temp.name)
 
-    def test_download_to_filename_w_updated_wo_raw(self):
-        updated = "2014-12-06T13:13:50.690Z"
-        self._download_to_filename_helper(updated=updated, raw_download=False)
-
-        client = self._make_client()
-
-        blob = self._make_one(
-            "blob-name", bucket=_Bucket(client), properties={"mediaLink": MEDIA_LINK}
-        )
-        blob._do_download = mock.Mock()
-
-        with _NamedTemporaryFile() as temp:
-            blob.download_to_filename(temp.name, if_generation_match=GENERATION_NUMBER)
-
-        blob._do_download.assert_called_once_with(
-            client._http,
-            mock.ANY,
-            EXPECTED_LINK,
-            HEADERS,
-            None,
-            None,
-            False,
-            timeout=self._get_default_timeout(),
-            checksum="md5",
-            retry=DEFAULT_RETRY,
-        )
-
     def test_download_to_filename_w_updated_wo_raw(self):
         updated = "2014-12-06T13:13:50.690Z"
         self._download_to_filename_helper(updated=updated, raw_download=False)
@@ -1514,6 +1578,7 @@ def test_download_to_filename_w_generation_match(self):
             raw_download=False,
             timeout=expected_timeout,
             checksum="md5",
+            retry=DEFAULT_RETRY,
         )
         stream = client.download_blob_to_file.mock_calls[0].args[1]
         self.assertEqual(stream.name, temp.name)
@@ -1558,42 +1623,6 @@ def test_download_to_filename_corrupted(self):
         stream = client.download_blob_to_file.mock_calls[0].args[1]
         self.assertEqual(stream.name, filename)
 
-    def test_download_to_filename_w_key(self):
-        from google.cloud._testing import _NamedTemporaryFile
-        from google.cloud.storage.blob import _get_encryption_headers
-
-        blob_name = "blob-name"
-        # Create a fake client/bucket and use them in the Blob() constructor.
-        client = self._make_client()
-        bucket = _Bucket(client)
-        media_link = "http://example.com/media/"
-        properties = {"mediaLink": media_link}
-        key = b"aa426195405adee2c8081bb9e7e74b19"
-        blob = self._make_one(
-            blob_name, bucket=bucket, properties=properties, encryption_key=key
-        )
-        blob._do_download = mock.Mock()
-
-        with _NamedTemporaryFile() as temp:
-            blob.download_to_filename(temp.name)
-
-        headers = {"accept-encoding": "gzip"}
-        headers.update(_get_encryption_headers(key))
-        blob._do_download.assert_called_once_with(
-            client._http,
-            mock.ANY,
-            media_link,
-            headers,
-            None,
-            None,
-            False,
-            timeout=self._get_default_timeout(),
-            checksum="md5",
-            retry=DEFAULT_RETRY,
-        )
-        stream = blob._do_download.mock_calls[0].args[1]
-        self.assertEqual(stream.name, temp.name)
-
     def _download_as_bytes_helper(self, raw_download, timeout=None, **extra_kwargs):
         blob_name = "blob-name"
         client = self._make_client()
@@ -1612,9 +1641,8 @@ def _download_as_bytes_helper(self, raw_download, timeout=None, **extra_kwargs):
 
         expected_retry = extra_kwargs.get("retry", DEFAULT_RETRY)
 
-        headers = {"accept-encoding": "gzip"}
-        blob._do_download.assert_called_once_with(
-            client._http,
+        client.download_blob_to_file.assert_called_once_with(
+            blob,
             mock.ANY,
             start=None,
             end=None,
@@ -1630,12 +1658,6 @@ def _download_as_bytes_helper(self, raw_download, timeout=None, **extra_kwargs):
         stream = client.download_blob_to_file.mock_calls[0].args[1]
         self.assertIsInstance(stream, io.BytesIO)
 
-    def test_download_as_bytes_wo_raw(self):
-        self._download_as_bytes_helper(raw_download=False)
-
-    def test_download_as_bytes_w_raw(self):
-        self._download_as_bytes_helper(raw_download=True)
-
     def test_download_as_bytes_w_custom_timeout(self):
         self._download_as_bytes_helper(raw_download=False, timeout=9.58)