From cc853af6bf8e44e5b16e8cdfb3a275629ffb1f27 Mon Sep 17 00:00:00 2001 From: HemangChothani <50404902+HemangChothani@users.noreply.github.com> Date: Wed, 11 Mar 2020 10:50:55 +0530 Subject: [PATCH] feat(storage): Add cname support for V4 signature (#72) * feat(storage): add cname support for v4 signature * docs(storage): comment changes * feat(storage): address comment * feat(storage): doc fix * feat(storage): nit addressed * feat(storage): add conformance tests * feat(storage): nit Co-authored-by: Frank Natividad --- google/cloud/storage/_signing.py | 2 +- google/cloud/storage/blob.py | 40 +++++++++++++++++++- google/cloud/storage/bucket.py | 39 ++++++++++++++++++- tests/unit/test__signing.py | 31 ++++++++++++--- tests/unit/test_blob.py | 22 ++++++++++- tests/unit/test_bucket.py | 22 ++++++++++- tests/unit/url_signer_v4_test_data.json | 50 +++++++++++++++++++++++++ 7 files changed, 195 insertions(+), 11 deletions(-) diff --git a/google/cloud/storage/_signing.py b/google/cloud/storage/_signing.py index 8151786e9..9c9ad9242 100644 --- a/google/cloud/storage/_signing.py +++ b/google/cloud/storage/_signing.py @@ -553,7 +553,7 @@ def generate_signed_url_v4( header_names = [key.lower() for key in headers] if "host" not in header_names: - headers["Host"] = "storage.googleapis.com" + headers["Host"] = six.moves.urllib.parse.urlparse(api_access_endpoint).netloc if method.upper() == "RESUMABLE": method = "POST" diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index d29e1e1a8..2ba2da1c3 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -362,6 +362,8 @@ def generate_signed_url( service_account_email=None, access_token=None, virtual_hosted_style=False, + bucket_bound_hostname=None, + scheme="http", ): """Generates a signed URL for this blob. @@ -380,6 +382,21 @@ def generate_signed_url( amount of time, you can use this method to generate a URL that is only valid within a certain time period. + If ``bucket_bound_hostname`` is set as an argument of :attr:`api_access_endpoint`, + ``https`` works only if using a ``CDN``. + + Example: + Generates a signed URL for this blob using bucket_bound_hostname and scheme. + + >>> from google.cloud import storage + >>> client = storage.Client() + >>> bucket = client.get_bucket('my-bucket-name') + >>> blob = client.get_blob('my-blob-name') + >>> url = blob.generate_signed_url(expiration='url-expiration-time', bucket_bound_hostname='mydomain.tld', + >>> version='v4') + >>> url = blob.generate_signed_url(expiration='url-expiration-time', bucket_bound_hostname='mydomain.tld', + >>> version='v4',scheme='https') # If using ``CDN`` + This is particularly useful if you don't want publicly accessible blobs, but don't want to require users to explicitly log in. @@ -460,6 +477,18 @@ def generate_signed_url( (Optional) If true, then construct the URL relative the bucket's virtual hostname, e.g., '.storage.googleapis.com'. + :type bucket_bound_hostname: str + :param bucket_bound_hostname: + (Optional) If pass, then construct the URL relative to the bucket-bound hostname. + Value cane be a bare or with scheme, e.g., 'example.com' or 'http://example.com'. + See: https://cloud.google.com/storage/docs/request-endpoints#cname + + :type scheme: str + :param scheme: + (Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use + this value as the scheme. ``https`` will work only when using a CDN. + Defaults to ``"http"``. + :raises: :exc:`ValueError` when version is invalid. :raises: :exc:`TypeError` when expiration is not a valid type. :raises: :exc:`AttributeError` if credentials is not an instance @@ -480,12 +509,21 @@ def generate_signed_url( api_access_endpoint = "https://{bucket_name}.storage.googleapis.com".format( bucket_name=self.bucket.name ) - resource = "/{quoted_name}".format(quoted_name=quoted_name) + elif bucket_bound_hostname: + if ":" in bucket_bound_hostname: + api_access_endpoint = bucket_bound_hostname + else: + api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format( + scheme=scheme, bucket_bound_hostname=bucket_bound_hostname + ) else: resource = "/{bucket_name}/{quoted_name}".format( bucket_name=self.bucket.name, quoted_name=quoted_name ) + if virtual_hosted_style or bucket_bound_hostname: + resource = "/{quoted_name}".format(quoted_name=quoted_name) + if credentials is None: client = self._require_client(client) credentials = client._credentials diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index 2ac7c097f..4a96313d4 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -2395,6 +2395,8 @@ def generate_signed_url( credentials=None, version=None, virtual_hosted_style=False, + bucket_bound_hostname=None, + scheme="http", ): """Generates a signed URL for this bucket. @@ -2413,6 +2415,20 @@ def generate_signed_url( amount of time, you can use this method to generate a URL that is only valid within a certain time period. + If ``bucket_bound_hostname`` is set as an argument of :attr:`api_access_endpoint`, + ``https`` works only if using a ``CDN``. + + Example: + Generates a signed URL for this bucket using bucket_bound_hostname and scheme. + + >>> from google.cloud import storage + >>> client = storage.Client() + >>> bucket = client.get_bucket('my-bucket-name') + >>> url = bucket.generate_signed_url(expiration='url-expiration-time', bucket_bound_hostname='mydomain.tld', + >>> version='v4') + >>> url = bucket.generate_signed_url(expiration='url-expiration-time', bucket_bound_hostname='mydomain.tld', + >>> version='v4',scheme='https') # If using ``CDN`` + This is particularly useful if you don't want publicly accessible buckets, but don't want to require users to explicitly log in. @@ -2462,6 +2478,18 @@ def generate_signed_url( (Optional) If true, then construct the URL relative the bucket's virtual hostname, e.g., '.storage.googleapis.com'. + :type bucket_bound_hostname: str + :param bucket_bound_hostname: + (Optional) If pass, then construct the URL relative to the bucket-bound hostname. + Value cane be a bare or with scheme, e.g., 'example.com' or 'http://example.com'. + See: https://cloud.google.com/storage/docs/request-endpoints#cname + + :type scheme: str + :param scheme: + (Optional) If ``bucket_bound_hostname`` is passed as a bare hostname, use + this value as the scheme. ``https`` will work only when using a CDN. + Defaults to ``"http"``. + :raises: :exc:`ValueError` when version is invalid. :raises: :exc:`TypeError` when expiration is not a valid type. :raises: :exc:`AttributeError` if credentials is not an instance @@ -2480,10 +2508,19 @@ def generate_signed_url( api_access_endpoint = "https://{bucket_name}.storage.googleapis.com".format( bucket_name=self.name ) - resource = "/" + elif bucket_bound_hostname: + if ":" in bucket_bound_hostname: + api_access_endpoint = bucket_bound_hostname + else: + api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format( + scheme=scheme, bucket_bound_hostname=bucket_bound_hostname + ) else: resource = "/{bucket_name}".format(bucket_name=self.name) + if virtual_hosted_style or bucket_bound_hostname: + resource = "/" + if credentials is None: client = self._require_client(client) credentials = client._credentials diff --git a/tests/unit/test__signing.py b/tests/unit/test__signing.py index 47ed2bdf0..c3b911f1d 100644 --- a/tests/unit/test__signing.py +++ b/tests/unit/test__signing.py @@ -778,13 +778,15 @@ def dummy_service_account(): return _DUMMY_SERVICE_ACCOUNT -def _run_conformance_test(resource, test_data): +def _run_conformance_test( + resource, test_data, api_access_endpoint="https://storage.googleapis.com" +): credentials = dummy_service_account() - url = Test_generate_signed_url_v4._call_fut( credentials, resource, expiration=test_data["expiration"], + api_access_endpoint=api_access_endpoint, method=test_data["method"], _request_timestamp=test_data["timestamp"], headers=test_data.get("headers"), @@ -802,14 +804,31 @@ def test_conformance_client(test_data): @pytest.mark.parametrize("test_data", _BUCKET_TESTS) def test_conformance_bucket(test_data): - resource = "/{}".format(test_data["bucket"]) - _run_conformance_test(resource, test_data) + if "urlStyle" in test_data and test_data["urlStyle"] == "BUCKET_BOUND_HOSTNAME": + api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format( + scheme=test_data["scheme"], + bucket_bound_hostname=test_data["bucketBoundHostname"], + ) + resource = "/" + _run_conformance_test(resource, test_data, api_access_endpoint) + else: + resource = "/{}".format(test_data["bucket"]) + _run_conformance_test(resource, test_data) @pytest.mark.parametrize("test_data", _BLOB_TESTS) def test_conformance_blob(test_data): - resource = "/{}/{}".format(test_data["bucket"], test_data["object"]) - _run_conformance_test(resource, test_data) + if "urlStyle" in test_data and test_data["urlStyle"] == "BUCKET_BOUND_HOSTNAME": + api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format( + scheme=test_data["scheme"], + bucket_bound_hostname=test_data["bucketBoundHostname"], + ) + resource = "/{}".format(test_data["object"]) + _run_conformance_test(resource, test_data, api_access_endpoint) + else: + + resource = "/{}/{}".format(test_data["bucket"], test_data["object"]) + _run_conformance_test(resource, test_data) def _make_credentials(signer_email=None): diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index 856aa712e..f656e6441 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -400,6 +400,8 @@ def _generate_signed_url_helper( access_token=None, service_account_email=None, virtual_hosted_style=False, + bucket_bound_hostname=None, + scheme="http", ): from six.moves.urllib import parse from google.cloud._helpers import UTC @@ -444,6 +446,7 @@ def _generate_signed_url_helper( access_token=access_token, service_account_email=service_account_email, virtual_hosted_style=virtual_hosted_style, + bucket_bound_hostname=bucket_bound_hostname, ) self.assertEqual(signed_uri, signer.return_value) @@ -460,11 +463,20 @@ def _generate_signed_url_helper( expected_api_access_endpoint = "https://{}.storage.googleapis.com".format( bucket.name ) - expected_resource = "/{}".format(quoted_name) + elif bucket_bound_hostname: + if ":" in bucket_bound_hostname: + expected_api_access_endpoint = bucket_bound_hostname + else: + expected_api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format( + scheme=scheme, bucket_bound_hostname=bucket_bound_hostname + ) else: expected_api_access_endpoint = api_access_endpoint expected_resource = "/{}/{}".format(bucket.name, quoted_name) + if virtual_hosted_style or bucket_bound_hostname: + expected_resource = "/{}".format(quoted_name) + if encryption_key is not None: expected_headers = headers or {} if effective_version == "v2": @@ -619,6 +631,14 @@ def test_generate_signed_url_v4_w_csek_and_headers(self): def test_generate_signed_url_v4_w_virtual_hostname(self): self._generate_signed_url_v4_helper(virtual_hosted_style=True) + def test_generate_signed_url_v4_w_bucket_bound_hostname_w_scheme(self): + self._generate_signed_url_v4_helper( + bucket_bound_hostname="http://cdn.example.com" + ) + + def test_generate_signed_url_v4_w_bucket_bound_hostname_w_bare_hostname(self): + self._generate_signed_url_v4_helper(bucket_bound_hostname="cdn.example.com") + def test_generate_signed_url_v4_w_credentials(self): credentials = object() self._generate_signed_url_v4_helper(credentials=credentials) diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index 1024cbba7..365e1f0e1 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -2779,6 +2779,8 @@ def _generate_signed_url_helper( credentials=None, expiration=None, virtual_hosted_style=False, + bucket_bound_hostname=None, + scheme="http", ): from six.moves.urllib import parse from google.cloud._helpers import UTC @@ -2814,6 +2816,7 @@ def _generate_signed_url_helper( query_parameters=query_parameters, version=version, virtual_hosted_style=virtual_hosted_style, + bucket_bound_hostname=bucket_bound_hostname, ) self.assertEqual(signed_uri, signer.return_value) @@ -2827,11 +2830,20 @@ def _generate_signed_url_helper( expected_api_access_endpoint = "https://{}.storage.googleapis.com".format( bucket_name ) - expected_resource = "/" + elif bucket_bound_hostname: + if ":" in bucket_bound_hostname: + expected_api_access_endpoint = bucket_bound_hostname + else: + expected_api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format( + scheme=scheme, bucket_bound_hostname=bucket_bound_hostname + ) else: expected_api_access_endpoint = api_access_endpoint expected_resource = "/{}".format(parse.quote(bucket_name)) + if virtual_hosted_style or bucket_bound_hostname: + expected_resource = "/" + expected_kwargs = { "resource": expected_resource, "expiration": expiration, @@ -2967,6 +2979,14 @@ def test_generate_signed_url_v4_w_credentials(self): def test_generate_signed_url_v4_w_virtual_hostname(self): self._generate_signed_url_v4_helper(virtual_hosted_style=True) + def test_generate_signed_url_v4_w_bucket_bound_hostname_w_scheme(self): + self._generate_signed_url_v4_helper( + bucket_bound_hostname="http://cdn.example.com" + ) + + def test_generate_signed_url_v4_w_bucket_bound_hostname_w_bare_hostname(self): + self._generate_signed_url_v4_helper(bucket_bound_hostname="cdn.example.com") + class _Connection(object): _delete_bucket = False diff --git a/tests/unit/url_signer_v4_test_data.json b/tests/unit/url_signer_v4_test_data.json index 807f6cf49..a43fc5cd5 100644 --- a/tests/unit/url_signer_v4_test_data.json +++ b/tests/unit/url_signer_v4_test_data.json @@ -118,5 +118,55 @@ "expiration": 10, "timestamp": "20190201T090000Z", "expectedUrl": "https://storage.googleapis.com/test-bucket?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=6dbe94f8e52b2b8a9a476b1c857efa474e09944e2b52b925800316e094a7169d8dbe0df9c0ac08dabb22ac7e827470ceccd65f5a3eadba2a4fb9beebfe37f0d9bb1e552b851fa31a25045bdf019e507f5feb44f061551ef1aeb18dcec0e38ba2e2f77d560a46eaace9c56ed9aa642281301a9d848b0eb30749e34bc7f73a3d596240533466ff9b5f289cd0d4c845c7d96b82a35a5abd0c3aff83e4440ee6873e796087f43545544dc8c01afe1d79c726696b6f555371e491980e7ec145cca0803cf562c38f3fa1d724242f5dea25aac91d74ec9ddd739ff65523627763eaef25cd1f95ad985aaf0079b7c74eb5bcb2870a9b137a7b2c8e41fbe838c95872f75b" + }, + + { + "description": "HTTP Bucket Bound Hostname Support", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "http://mydomain.tld/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=7115a77f8c7ed1a8b74bca8b520904fca7f3bab90d69ea052687a94efd9b3a4e2a7fb7135d40e295e0a21958194c55da7e106227957c22ed6edc9d8b3d2a8133bc8af84fc9695dda8081d53f0db5ea9f28e5bfc225d78f873e9f571fd287bb7a95330e726aebd8eb4623cdb0b1a7ceb210b2ce1351b6be0191c2ad7b38f7ceb6c5ce2f98dbfb5a5a649050585e46e97f72f1f5407de657a56e34a3fdc80cdaa0598bd47f3e8af5ff22d0916b19b106890bff8c3f6587f1d3b076b16cd0ba0508607a672be33b9c75d537e15258450b43d22a21c4d528090acbb8e5bae7b31fc394e61394106ef1d6a8ed43074ab05bcec65674cd8113fb3de388da4d97e62f56", + "scheme": "http", + "urlStyle": "BUCKET_BOUND_HOSTNAME", + "bucketBoundHostname": "mydomain.tld" + }, + + { + "description": "HTTPS Bucket Bound Hostname Support", + "bucket": "test-bucket", + "object": "test-object", + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://mydomain.tld/test-object?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=7115a77f8c7ed1a8b74bca8b520904fca7f3bab90d69ea052687a94efd9b3a4e2a7fb7135d40e295e0a21958194c55da7e106227957c22ed6edc9d8b3d2a8133bc8af84fc9695dda8081d53f0db5ea9f28e5bfc225d78f873e9f571fd287bb7a95330e726aebd8eb4623cdb0b1a7ceb210b2ce1351b6be0191c2ad7b38f7ceb6c5ce2f98dbfb5a5a649050585e46e97f72f1f5407de657a56e34a3fdc80cdaa0598bd47f3e8af5ff22d0916b19b106890bff8c3f6587f1d3b076b16cd0ba0508607a672be33b9c75d537e15258450b43d22a21c4d528090acbb8e5bae7b31fc394e61394106ef1d6a8ed43074ab05bcec65674cd8113fb3de388da4d97e62f56", + "scheme": "https", + "urlStyle": "BUCKET_BOUND_HOSTNAME", + "bucketBoundHostname": "mydomain.tld" + }, + + { + "description": "HTTP Bucket Bound Hostname Support", + "bucket": "test-bucket", + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "http://mydomain.tld/?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=7a629a5632f16dba78961250b17c1f0d2ac0d2a28dbd7cbf79088fd6cd0b7f3ec66285cdeccca024f7b8134376f5cdcf0d60f399c6df1f19fcf5cf3be9d7f905d72cb6c0b5600f83dd6a7c8df607510c0e12e36216530a7b832eab87920363c5368a7e610d44005c73324f6ca4b435e8687672f46cc1342419ec4a5264549cb4b77bdc73f4f461edf39fbdd8fda99db440b077e906ef48d2c6b854c11ded58096f293d664650c123c6ec2a0379affd05bf5696ba11d3474623e039d5e05d3dc331b86ff4f7afb9262cf9750ff5944e661e70cc443b28f7e150796dde831d70e205c7e848c19b8281510f1d195e5819176e4868713266d0e0db7a3354857187cf", + "scheme": "http", + "urlStyle": "BUCKET_BOUND_HOSTNAME", + "bucketBoundHostname": "mydomain.tld" + }, + + { + "description": "HTTPS Bucket Bound Hostname Support", + "bucket": "test-bucket", + "method": "GET", + "expiration": 10, + "timestamp": "20190201T090000Z", + "expectedUrl": "https://mydomain.tld/?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=test-iam-credentials%40dummy-project-id.iam.gserviceaccount.com%2F20190201%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20190201T090000Z&X-Goog-Expires=10&X-Goog-SignedHeaders=host&X-Goog-Signature=7a629a5632f16dba78961250b17c1f0d2ac0d2a28dbd7cbf79088fd6cd0b7f3ec66285cdeccca024f7b8134376f5cdcf0d60f399c6df1f19fcf5cf3be9d7f905d72cb6c0b5600f83dd6a7c8df607510c0e12e36216530a7b832eab87920363c5368a7e610d44005c73324f6ca4b435e8687672f46cc1342419ec4a5264549cb4b77bdc73f4f461edf39fbdd8fda99db440b077e906ef48d2c6b854c11ded58096f293d664650c123c6ec2a0379affd05bf5696ba11d3474623e039d5e05d3dc331b86ff4f7afb9262cf9750ff5944e661e70cc443b28f7e150796dde831d70e205c7e848c19b8281510f1d195e5819176e4868713266d0e0db7a3354857187cf", + "scheme": "https", + "urlStyle": "BUCKET_BOUND_HOSTNAME", + "bucketBoundHostname": "mydomain.tld" } ]