From 7a94acb50e75fe0a51688e0f968bca3fa9bd9082 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Wed, 3 Feb 2021 16:57:08 -0700 Subject: [PATCH] feat: support self-signed jwt in requests and urllib3 transports (#679) --- google/auth/transport/requests.py | 17 +++++++ google/auth/transport/urllib3.py | 17 +++++++ system_tests/noxfile.py | 17 +++++++ system_tests/system_tests_sync/test_grpc.py | 3 +- .../system_tests_sync/test_requests.py | 40 +++++++++++++++++ .../system_tests_sync/test_urllib3.py | 44 +++++++++++++++++++ tests/transport/test_requests.py | 20 +++++++++ tests/transport/test_urllib3.py | 20 +++++++++ 8 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 system_tests/system_tests_sync/test_requests.py create mode 100644 system_tests/system_tests_sync/test_urllib3.py diff --git a/google/auth/transport/requests.py b/google/auth/transport/requests.py index 9a2f3afc7..ef973fce4 100644 --- a/google/auth/transport/requests.py +++ b/google/auth/transport/requests.py @@ -45,6 +45,7 @@ from google.auth import exceptions from google.auth import transport import google.auth.transport._mtls_helper +from google.oauth2 import service_account _LOGGER = logging.getLogger(__name__) @@ -313,6 +314,9 @@ def my_cert_callback(): refreshing credentials. If not passed, an instance of :class:`~google.auth.transport.requests.Request` is created. + default_host (Optional[str]): A host like "pubsub.googleapis.com". + This is used when a self-signed JWT is created from service + account credentials. """ def __init__( @@ -322,6 +326,7 @@ def __init__( max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS, refresh_timeout=None, auth_request=None, + default_host=None, ): super(AuthorizedSession, self).__init__() self.credentials = credentials @@ -329,6 +334,7 @@ def __init__( self._max_refresh_attempts = max_refresh_attempts self._refresh_timeout = refresh_timeout self._is_mtls = False + self._default_host = default_host if auth_request is None: auth_request_session = requests.Session() @@ -347,6 +353,17 @@ def __init__( # credentials.refresh). self._auth_request = auth_request + # https://google.aip.dev/auth/4111 + # Attempt to use self-signed JWTs when a service account is used. + # A default host must be explicitly provided. + if ( + isinstance(self.credentials, service_account.Credentials) + and self._default_host + ): + self.credentials._create_self_signed_jwt( + "https://{}/".format(self._default_host) + ) + def configure_mtls_channel(self, client_cert_callback=None): """Configure the client certificate and key for SSL connection. diff --git a/google/auth/transport/urllib3.py b/google/auth/transport/urllib3.py index 209fc51bc..aadd116e8 100644 --- a/google/auth/transport/urllib3.py +++ b/google/auth/transport/urllib3.py @@ -49,6 +49,7 @@ from google.auth import environment_vars from google.auth import exceptions from google.auth import transport +from google.oauth2 import service_account _LOGGER = logging.getLogger(__name__) @@ -262,6 +263,9 @@ def my_cert_callback(): retried. max_refresh_attempts (int): The maximum number of times to attempt to refresh the credentials and retry the request. + default_host (Optional[str]): A host like "pubsub.googleapis.com". + This is used when a self-signed JWT is created from service + account credentials. """ def __init__( @@ -270,6 +274,7 @@ def __init__( http=None, refresh_status_codes=transport.DEFAULT_REFRESH_STATUS_CODES, max_refresh_attempts=transport.DEFAULT_MAX_REFRESH_ATTEMPTS, + default_host=None, ): if http is None: self.http = _make_default_http() @@ -281,10 +286,22 @@ def __init__( self.credentials = credentials self._refresh_status_codes = refresh_status_codes self._max_refresh_attempts = max_refresh_attempts + self._default_host = default_host # Request instance used by internal methods (for example, # credentials.refresh). self._request = Request(self.http) + # https://google.aip.dev/auth/4111 + # Attempt to use self-signed JWTs when a service account is used. + # A default host must be explicitly provided. + if ( + isinstance(self.credentials, service_account.Credentials) + and self._default_host + ): + self.credentials._create_self_signed_jwt( + "https://{}/".format(self._default_host) + ) + super(AuthorizedHttp, self).__init__() def configure_mtls_channel(self, client_cert_callback=None): diff --git a/system_tests/noxfile.py b/system_tests/noxfile.py index 5d0014bc8..4ba7cc3ef 100644 --- a/system_tests/noxfile.py +++ b/system_tests/noxfile.py @@ -293,6 +293,22 @@ def grpc(session): session.run("pytest", "system_tests_sync/test_grpc.py") +@nox.session(python=PYTHON_VERSIONS_SYNC) +def requests(session): + session.install(LIBRARY_DIR) + session.install(*TEST_DEPENDENCIES_SYNC) + session.env[EXPLICIT_CREDENTIALS_ENV] = SERVICE_ACCOUNT_FILE + session.run("pytest", "system_tests_sync/test_requests.py") + + +@nox.session(python=PYTHON_VERSIONS_SYNC) +def urllib3(session): + session.install(LIBRARY_DIR) + session.install(*TEST_DEPENDENCIES_SYNC) + session.env[EXPLICIT_CREDENTIALS_ENV] = SERVICE_ACCOUNT_FILE + session.run("pytest", "system_tests_sync/test_urllib3.py") + + @nox.session(python=PYTHON_VERSIONS_SYNC) def mtls_http(session): session.install(LIBRARY_DIR) @@ -300,6 +316,7 @@ def mtls_http(session): session.env[EXPLICIT_CREDENTIALS_ENV] = SERVICE_ACCOUNT_FILE session.run("pytest", "system_tests_sync/test_mtls_http.py") + #ASYNC SYSTEM TESTS @nox.session(python=PYTHON_VERSIONS_ASYNC) diff --git a/system_tests/system_tests_sync/test_grpc.py b/system_tests/system_tests_sync/test_grpc.py index da2eb71fb..7f548ec0e 100644 --- a/system_tests/system_tests_sync/test_grpc.py +++ b/system_tests/system_tests_sync/test_grpc.py @@ -57,8 +57,9 @@ def test_grpc_request_with_regular_credentials_and_self_signed_jwt(http_request) list_topics_iter = client.list_topics(project="projects/{}".format(project_id)) list(list_topics_iter) - # Check that self-signed JWT was created + # Check that self-signed JWT was created and is being used assert credentials._jwt_credentials is not None + assert credentials._jwt_credentials.token == credentials.token def test_grpc_request_with_jwt_credentials(): diff --git a/system_tests/system_tests_sync/test_requests.py b/system_tests/system_tests_sync/test_requests.py new file mode 100644 index 000000000..3ac9179b5 --- /dev/null +++ b/system_tests/system_tests_sync/test_requests.py @@ -0,0 +1,40 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import google.auth +import google.auth.credentials +import google.auth.transport.requests +from google.oauth2 import service_account + + +def test_authorized_session_with_service_account_and_self_signed_jwt(): + credentials, project_id = google.auth.default() + + credentials = credentials.with_scopes( + scopes=[], + default_scopes=["https://www.googleapis.com/auth/pubsub"], + ) + + session = google.auth.transport.requests.AuthorizedSession( + credentials=credentials, default_host="pubsub.googleapis.com" + ) + + # List Pub/Sub Topics through the REST API + # https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/list + response = session.get("https://pubsub.googleapis.com/v1/projects/{}/topics".format(project_id)) + response.raise_for_status() + + # Check that self-signed JWT was created and is being used + assert credentials._jwt_credentials is not None + assert credentials._jwt_credentials.token == credentials.token diff --git a/system_tests/system_tests_sync/test_urllib3.py b/system_tests/system_tests_sync/test_urllib3.py new file mode 100644 index 000000000..1932e1913 --- /dev/null +++ b/system_tests/system_tests_sync/test_urllib3.py @@ -0,0 +1,44 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import google.auth +import google.auth.credentials +import google.auth.transport.requests +from google.oauth2 import service_account + + +def test_authorized_session_with_service_account_and_self_signed_jwt(): + credentials, project_id = google.auth.default() + + credentials = credentials.with_scopes( + scopes=[], + default_scopes=["https://www.googleapis.com/auth/pubsub"], + ) + + http = google.auth.transport.urllib3.AuthorizedHttp( + credentials=credentials, default_host="pubsub.googleapis.com" + ) + + # List Pub/Sub Topics through the REST API + # https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/list + response = http.urlopen( + method="GET", + url="https://pubsub.googleapis.com/v1/projects/{}/topics".format(project_id) + ) + + assert response.status == 200 + + # Check that self-signed JWT was created and is being used + assert credentials._jwt_credentials is not None + assert credentials._jwt_credentials.token == credentials.token diff --git a/tests/transport/test_requests.py b/tests/transport/test_requests.py index d56c2be55..3fdd17c3e 100644 --- a/tests/transport/test_requests.py +++ b/tests/transport/test_requests.py @@ -30,6 +30,7 @@ import google.auth.credentials import google.auth.transport._mtls_helper import google.auth.transport.requests +from google.oauth2 import service_account from tests.transport import compliance @@ -372,6 +373,25 @@ def test_request_timeout_w_refresh_timeout_timeout_error(self, frozen_time): "GET", self.TEST_URL, timeout=60, max_allowed_time=2.9 ) + def test_authorized_session_without_default_host(self): + credentials = mock.create_autospec(service_account.Credentials) + + authed_session = google.auth.transport.requests.AuthorizedSession(credentials) + + authed_session.credentials._create_self_signed_jwt.assert_not_called() + + def test_authorized_session_with_default_host(self): + default_host = "pubsub.googleapis.com" + credentials = mock.create_autospec(service_account.Credentials) + + authed_session = google.auth.transport.requests.AuthorizedSession( + credentials, default_host=default_host + ) + + authed_session.credentials._create_self_signed_jwt.assert_called_once_with( + "https://{}/".format(default_host) + ) + def test_configure_mtls_channel_with_callback(self): mock_callback = mock.Mock() mock_callback.return_value = ( diff --git a/tests/transport/test_urllib3.py b/tests/transport/test_urllib3.py index 29561f6d6..7c0693476 100644 --- a/tests/transport/test_urllib3.py +++ b/tests/transport/test_urllib3.py @@ -26,6 +26,7 @@ import google.auth.credentials import google.auth.transport._mtls_helper import google.auth.transport.urllib3 +from google.oauth2 import service_account from tests.transport import compliance @@ -158,6 +159,25 @@ def test_urlopen_refresh(self): ("GET", self.TEST_URL, None, {"authorization": "token1"}, {}), ] + def test_urlopen_no_default_host(self): + credentials = mock.create_autospec(service_account.Credentials) + + authed_http = google.auth.transport.urllib3.AuthorizedHttp(credentials) + + authed_http.credentials._create_self_signed_jwt.assert_not_called() + + def test_urlopen_with_default_host(self): + default_host = "pubsub.googleapis.com" + credentials = mock.create_autospec(service_account.Credentials) + + authed_http = google.auth.transport.urllib3.AuthorizedHttp( + credentials, default_host=default_host + ) + + authed_http.credentials._create_self_signed_jwt.assert_called_once_with( + "https://{}/".format(default_host) + ) + def test_proxies(self): http = mock.create_autospec(urllib3.PoolManager) authed_http = google.auth.transport.urllib3.AuthorizedHttp(None, http=http)