From ea018c9f4a1f9c360dbe9f08650250ea8c505f29 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Thu, 1 Jul 2021 14:15:32 -0500 Subject: [PATCH] docs: add sample to include run notification (#173) * docs: add sample to include run notification * blacken samples * fix sample lint --- owlbot.py | 6 ++ samples/snippets/conftest.py | 55 +++++++++++++++++-- samples/snippets/manage_transfer_configs.py | 4 +- .../snippets/manage_transfer_configs_test.py | 4 +- samples/snippets/noxfile.py | 39 +++++++------ samples/snippets/requirements-test.txt | 1 + samples/snippets/run_notification.py | 44 +++++++++++++++ samples/snippets/run_notification_test.py | 26 +++++++++ 8 files changed, 151 insertions(+), 28 deletions(-) create mode 100644 samples/snippets/run_notification.py create mode 100644 samples/snippets/run_notification_test.py diff --git a/owlbot.py b/owlbot.py index 6a5c03b1..08dd3376 100644 --- a/owlbot.py +++ b/owlbot.py @@ -14,12 +14,16 @@ """This script is used to synthesize generated parts of this library.""" +import pathlib + import synthtool as s from synthtool import gcp from synthtool.languages import python +REPO_ROOT = pathlib.Path(__file__).parent.absolute() + common = gcp.CommonTemplates() # ---------------------------------------------------------------------------- @@ -53,3 +57,5 @@ python.py_samples(skip_readmes=True) s.shell.run(["nox", "-s", "blacken"], hide_output=False) +for noxfile in REPO_ROOT.glob("samples/**/noxfile.py"): + s.shell.run(["nox", "-s", "blacken"], cwd=noxfile.parent, hide_output=False) diff --git a/samples/snippets/conftest.py b/samples/snippets/conftest.py index 998d5ea7..f708ff48 100644 --- a/samples/snippets/conftest.py +++ b/samples/snippets/conftest.py @@ -14,6 +14,7 @@ import datetime import os +import random import uuid from google.api_core import client_options @@ -21,9 +22,40 @@ import google.auth from google.cloud import bigquery from google.cloud import bigquery_datatransfer +from google.cloud import pubsub_v1 import pytest +RESOURCE_PREFIX = "python_bigquery_datatransfer_samples_snippets" +RESOURCE_DATE_FORMAT = "%Y%m%d%H%M%S" +RESOURCE_DATE_LENGTH = 4 + 2 + 2 + 2 + 2 + 2 + + +def resource_prefix() -> str: + timestamp = datetime.datetime.utcnow().strftime(RESOURCE_DATE_FORMAT) + random_string = hex(random.randrange(1000000))[2:] + return f"{RESOURCE_PREFIX}_{timestamp}_{random_string}" + + +def resource_name_to_date(resource_name: str): + start_date = len(RESOURCE_PREFIX) + 1 + date_string = resource_name[start_date : start_date + RESOURCE_DATE_LENGTH] + parsed_date = datetime.datetime.strptime(date_string, RESOURCE_DATE_FORMAT) + return parsed_date + + +@pytest.fixture(scope="session", autouse=True) +def cleanup_pubsub_topics(pubsub_client: pubsub_v1.PublisherClient, project_id): + yesterday = datetime.datetime.utcnow() - datetime.timedelta(days=1) + for topic in pubsub_client.list_topics(project=f"projects/{project_id}"): + topic_id = topic.name.split("/")[-1] + if ( + topic_id.startswith(RESOURCE_PREFIX) + and resource_name_to_date(topic_id) < yesterday + ): + pubsub_client.delete_topic(topic=topic.name) + + def temp_suffix(): now = datetime.datetime.now() return f"{now.strftime('%Y%m%d%H%M%S')}_{uuid.uuid4().hex[:8]}" @@ -35,6 +67,21 @@ def bigquery_client(default_credentials): return bigquery.Client(credentials=credentials, project=project_id) +@pytest.fixture(scope="session") +def pubsub_client(default_credentials): + credentials, _ = default_credentials + return pubsub_v1.PublisherClient(credentials=credentials) + + +@pytest.fixture(scope="session") +def pubsub_topic(pubsub_client: pubsub_v1.PublisherClient, project_id): + topic_id = resource_prefix() + topic_path = pubsub_v1.PublisherClient.topic_path(project_id, topic_id) + pubsub_client.create_topic(name=topic_path) + yield topic_path + pubsub_client.delete_topic(topic=topic_path) + + @pytest.fixture(scope="session") def dataset_id(bigquery_client, project_id): dataset_id = f"bqdts_{temp_suffix()}" @@ -56,10 +103,10 @@ def project_id(): @pytest.fixture(scope="session") def service_account_name(default_credentials): credentials, _ = default_credentials - # Note: this property is not available when running with user account - # credentials, but only service account credentials are used in our test - # infrastructure. - return credentials.service_account_email + # The service_account_email attribute is not available when running with + # user account credentials, but should be available when running from our + # continuous integration tests. + return getattr(credentials, "service_account_email", None) @pytest.fixture(scope="session") diff --git a/samples/snippets/manage_transfer_configs.py b/samples/snippets/manage_transfer_configs.py index 6b4abd78..5f775f10 100644 --- a/samples/snippets/manage_transfer_configs.py +++ b/samples/snippets/manage_transfer_configs.py @@ -135,9 +135,7 @@ def schedule_backfill(override_values={}): ) response = transfer_client.schedule_transfer_runs( - parent=transfer_config_name, - start_time=start_time, - end_time=end_time, + parent=transfer_config_name, start_time=start_time, end_time=end_time, ) print("Started transfer runs:") diff --git a/samples/snippets/manage_transfer_configs_test.py b/samples/snippets/manage_transfer_configs_test.py index de31c713..52d16dc2 100644 --- a/samples/snippets/manage_transfer_configs_test.py +++ b/samples/snippets/manage_transfer_configs_test.py @@ -52,9 +52,7 @@ def test_update_credentials_with_service_account( def test_schedule_backfill(capsys, transfer_config_name): runs = manage_transfer_configs.schedule_backfill( - { - "transfer_config_name": transfer_config_name, - } + {"transfer_config_name": transfer_config_name} ) out, _ = capsys.readouterr() assert "Started transfer runs:" in out diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 5ff9e1db..160fe728 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -38,17 +38,15 @@ TEST_CONFIG = { # You can opt out from the test for specific Python versions. - 'ignored_versions': ["2.7"], - + "ignored_versions": ["2.7"], # Old samples are opted out of enforcing Python type hints # All new samples should feature them - 'enforce_type_hints': False, - + "enforce_type_hints": False, # An envvar key for determining the project id to use. Change it # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. - 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', # If you need to use a specific version of pip, # change pip_version_override to the string representation @@ -56,13 +54,13 @@ "pip_version_override": None, # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - 'envs': {}, + "envs": {}, } try: # Ensure we can import noxfile_config in the project's directory. - sys.path.append('.') + sys.path.append(".") from noxfile_config import TEST_CONFIG_OVERRIDE except ImportError as e: print("No user noxfile_config found: detail: {}".format(e)) @@ -77,12 +75,12 @@ def get_pytest_env_vars() -> Dict[str, str]: ret = {} # Override the GCLOUD_PROJECT and the alias. - env_key = TEST_CONFIG['gcloud_project_env'] + env_key = TEST_CONFIG["gcloud_project_env"] # This should error out if not set. - ret['GOOGLE_CLOUD_PROJECT'] = os.environ[env_key] + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] # Apply user supplied envs. - ret.update(TEST_CONFIG['envs']) + ret.update(TEST_CONFIG["envs"]) return ret @@ -91,7 +89,7 @@ def get_pytest_env_vars() -> Dict[str, str]: ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8", "3.9"] # Any default versions that should be ignored. -IGNORED_VERSIONS = TEST_CONFIG['ignored_versions'] +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) @@ -140,7 +138,7 @@ def _determine_local_import_names(start_dir: str) -> List[str]: @nox.session def lint(session: nox.sessions.Session) -> None: - if not TEST_CONFIG['enforce_type_hints']: + if not TEST_CONFIG["enforce_type_hints"]: session.install("flake8", "flake8-import-order") else: session.install("flake8", "flake8-import-order", "flake8-annotations") @@ -149,9 +147,11 @@ def lint(session: nox.sessions.Session) -> None: args = FLAKE8_COMMON_ARGS + [ "--application-import-names", ",".join(local_names), - "." + ".", ] session.run("flake8", *args) + + # # Black # @@ -164,6 +164,7 @@ def blacken(session: nox.sessions.Session) -> None: session.run("black", *python_files) + # # Sample Tests # @@ -172,7 +173,9 @@ def blacken(session: nox.sessions.Session) -> None: PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] -def _session_tests(session: nox.sessions.Session, post_install: Callable = None) -> None: +def _session_tests( + session: nox.sessions.Session, post_install: Callable = None +) -> None: if TEST_CONFIG["pip_version_override"]: pip_version = TEST_CONFIG["pip_version_override"] session.install(f"pip=={pip_version}") @@ -202,7 +205,7 @@ def _session_tests(session: nox.sessions.Session, post_install: Callable = None) # on travis where slow and flaky tests are excluded. # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html success_codes=[0, 5], - env=get_pytest_env_vars() + env=get_pytest_env_vars(), ) @@ -212,9 +215,9 @@ def py(session: nox.sessions.Session) -> None: if session.python in TESTED_VERSIONS: _session_tests(session) else: - session.skip("SKIPPED: {} tests are disabled for this sample.".format( - session.python - )) + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) # diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index e4d022b0..f0a3f547 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,3 +1,4 @@ google-cloud-bigquery==2.20.0 +google-cloud-pubsub==2.6.0 pytest==6.2.4 mock==4.0.3 diff --git a/samples/snippets/run_notification.py b/samples/snippets/run_notification.py new file mode 100644 index 00000000..44f1bf12 --- /dev/null +++ b/samples/snippets/run_notification.py @@ -0,0 +1,44 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def run_notification(transfer_config_name, pubsub_topic): + orig_transfer_config_name = transfer_config_name + orig_pubsub_topic = pubsub_topic + # [START bigquerydatatransfer_run_notification] + transfer_config_name = "projects/1234/locations/us/transferConfigs/abcd" + pubsub_topic = "projects/PROJECT-ID/topics/TOPIC-ID" + # [END bigquerydatatransfer_run_notification] + transfer_config_name = orig_transfer_config_name + pubsub_topic = orig_pubsub_topic + + # [START bigquerydatatransfer_run_notification] + from google.cloud import bigquery_datatransfer + from google.protobuf import field_mask_pb2 + + transfer_client = bigquery_datatransfer.DataTransferServiceClient() + + transfer_config = bigquery_datatransfer.TransferConfig(name=transfer_config_name) + transfer_config.notification_pubsub_topic = pubsub_topic + update_mask = field_mask_pb2.FieldMask(paths=["notification_pubsub_topic"]) + + transfer_config = transfer_client.update_transfer_config( + {"transfer_config": transfer_config, "update_mask": update_mask} + ) + + print(f"Updated config: '{transfer_config.name}'") + print(f"Notification Pub/Sub topic: '{transfer_config.notification_pubsub_topic}'") + # [END bigquerydatatransfer_run_notification] + # Return the config name for testing purposes, so that it can be deleted. + return transfer_config diff --git a/samples/snippets/run_notification_test.py b/samples/snippets/run_notification_test.py new file mode 100644 index 00000000..4c41e689 --- /dev/null +++ b/samples/snippets/run_notification_test.py @@ -0,0 +1,26 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import run_notification + + +def test_run_notification(capsys, transfer_config_name, pubsub_topic): + run_notification.run_notification( + transfer_config_name=transfer_config_name, pubsub_topic=pubsub_topic, + ) + out, _ = capsys.readouterr() + assert "Updated config:" in out + assert transfer_config_name in out + assert "Notification Pub/Sub topic:" in out + assert pubsub_topic in out