Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
refactor: consolidate BigQuery client creation and set user-agent (#100)
* refactor: consolidate BigQuery client creation and set user-agent * add tests for default project logic * use google-auth >=1.2.0 for AnonymousCredentials support * bump minimum google-cloud-bigquery version
- Loading branch information
Showing
6 changed files
with
219 additions
and
63 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
# Copyright 2021 The PyBigQuery Authors | ||
# | ||
# Use of this source code is governed by an MIT-style | ||
# license that can be found in the LICENSE file or at | ||
# https://opensource.org/licenses/MIT. | ||
|
||
from google.api_core import client_info | ||
import google.auth | ||
from google.cloud import bigquery | ||
from google.oauth2 import service_account | ||
import sqlalchemy | ||
|
||
|
||
USER_AGENT_TEMPLATE = "sqlalchemy/{}" | ||
SCOPES = ( | ||
"https://www.googleapis.com/auth/bigquery", | ||
"https://www.googleapis.com/auth/cloud-platform", | ||
"https://www.googleapis.com/auth/drive", | ||
) | ||
|
||
|
||
def google_client_info(): | ||
user_agent = USER_AGENT_TEMPLATE.format(sqlalchemy.__version__) | ||
return client_info.ClientInfo(user_agent=user_agent) | ||
|
||
|
||
def create_bigquery_client( | ||
credentials_info=None, | ||
credentials_path=None, | ||
default_query_job_config=None, | ||
location=None, | ||
project_id=None, | ||
): | ||
default_project = None | ||
|
||
if credentials_path: | ||
credentials = service_account.Credentials.from_service_account_file( | ||
credentials_path | ||
) | ||
credentials = credentials.with_scopes(SCOPES) | ||
default_project = credentials.project | ||
elif credentials_info: | ||
credentials = service_account.Credentials.from_service_account_info( | ||
credentials_info | ||
) | ||
credentials = credentials.with_scopes(SCOPES) | ||
default_project = credentials.project | ||
else: | ||
credentials, default_project = google.auth.default(scopes=SCOPES) | ||
|
||
if project_id is None: | ||
project_id = default_project | ||
|
||
return bigquery.Client( | ||
client_info=google_client_info(), | ||
project=project_id, | ||
credentials=credentials, | ||
location=location, | ||
default_query_job_config=default_query_job_config, | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,138 @@ | ||
# Copyright 2021 The PyBigQuery Authors | ||
# | ||
# Use of this source code is governed by an MIT-style | ||
# license that can be found in the LICENSE file or at | ||
# https://opensource.org/licenses/MIT. | ||
|
||
from unittest import mock | ||
|
||
import google.auth | ||
import google.auth.credentials | ||
from google.oauth2 import service_account | ||
import pytest | ||
|
||
|
||
class AnonymousCredentialsWithProject(google.auth.credentials.AnonymousCredentials): | ||
"""Fake credentials to trick isinstance""" | ||
|
||
def __init__(self, project): | ||
super().__init__() | ||
self.project = project | ||
|
||
def with_scopes(self, scopes): | ||
return self | ||
|
||
|
||
@pytest.fixture(scope="session") | ||
def module_under_test(): | ||
from pybigquery import _helpers | ||
|
||
return _helpers | ||
|
||
|
||
def test_create_bigquery_client_with_credentials_path(monkeypatch, module_under_test): | ||
mock_service_account = mock.create_autospec(service_account.Credentials) | ||
mock_service_account.from_service_account_file.return_value = AnonymousCredentialsWithProject( | ||
"service-account-project" | ||
) | ||
monkeypatch.setattr(service_account, "Credentials", mock_service_account) | ||
|
||
bqclient = module_under_test.create_bigquery_client( | ||
credentials_path="path/to/key.json", | ||
) | ||
|
||
assert bqclient.project == "service-account-project" | ||
|
||
|
||
def test_create_bigquery_client_with_credentials_path_respects_project( | ||
monkeypatch, module_under_test | ||
): | ||
"""Test that project_id is used, even when there is a default project. | ||
https://github.com/googleapis/python-bigquery-sqlalchemy/issues/48 | ||
""" | ||
mock_service_account = mock.create_autospec(service_account.Credentials) | ||
mock_service_account.from_service_account_file.return_value = AnonymousCredentialsWithProject( | ||
"service-account-project" | ||
) | ||
monkeypatch.setattr(service_account, "Credentials", mock_service_account) | ||
|
||
bqclient = module_under_test.create_bigquery_client( | ||
credentials_path="path/to/key.json", project_id="connection-url-project", | ||
) | ||
|
||
assert bqclient.project == "connection-url-project" | ||
|
||
|
||
def test_create_bigquery_client_with_credentials_info(monkeypatch, module_under_test): | ||
mock_service_account = mock.create_autospec(service_account.Credentials) | ||
mock_service_account.from_service_account_info.return_value = AnonymousCredentialsWithProject( | ||
"service-account-project" | ||
) | ||
monkeypatch.setattr(service_account, "Credentials", mock_service_account) | ||
|
||
bqclient = module_under_test.create_bigquery_client( | ||
credentials_info={ | ||
"type": "service_account", | ||
"project_id": "service-account-project", | ||
}, | ||
) | ||
|
||
assert bqclient.project == "service-account-project" | ||
|
||
|
||
def test_create_bigquery_client_with_credentials_info_respects_project( | ||
monkeypatch, module_under_test | ||
): | ||
"""Test that project_id is used, even when there is a default project. | ||
https://github.com/googleapis/python-bigquery-sqlalchemy/issues/48 | ||
""" | ||
mock_service_account = mock.create_autospec(service_account.Credentials) | ||
mock_service_account.from_service_account_info.return_value = AnonymousCredentialsWithProject( | ||
"service-account-project" | ||
) | ||
monkeypatch.setattr(service_account, "Credentials", mock_service_account) | ||
|
||
bqclient = module_under_test.create_bigquery_client( | ||
credentials_info={ | ||
"type": "service_account", | ||
"project_id": "service-account-project", | ||
}, | ||
project_id="connection-url-project", | ||
) | ||
|
||
assert bqclient.project == "connection-url-project" | ||
|
||
|
||
def test_create_bigquery_client_with_default_credentials( | ||
monkeypatch, module_under_test | ||
): | ||
def mock_default_credentials(*args, **kwargs): | ||
return (google.auth.credentials.AnonymousCredentials(), "default-project") | ||
|
||
monkeypatch.setattr(google.auth, "default", mock_default_credentials) | ||
|
||
bqclient = module_under_test.create_bigquery_client() | ||
|
||
assert bqclient.project == "default-project" | ||
|
||
|
||
def test_create_bigquery_client_with_default_credentials_respects_project( | ||
monkeypatch, module_under_test | ||
): | ||
"""Test that project_id is used, even when there is a default project. | ||
https://github.com/googleapis/python-bigquery-sqlalchemy/issues/48 | ||
""" | ||
|
||
def mock_default_credentials(*args, **kwargs): | ||
return (google.auth.credentials.AnonymousCredentials(), "default-project") | ||
|
||
monkeypatch.setattr(google.auth, "default", mock_default_credentials) | ||
|
||
bqclient = module_under_test.create_bigquery_client( | ||
project_id="connection-url-project", | ||
) | ||
|
||
assert bqclient.project == "connection-url-project" |