chore: migrate to owl bot

.github/.OwlBot.lock.yaml

@@ -0,0 +1,4 @@
+docker:
+  digest: sha256:457583330eec64daa02aeb7a72a04d33e7be2428f646671ce4045dcbc0191b1e
+  image: gcr.io/repo-automation-bots/owlbot-python:latest  
+

.github/.OwlBot.yaml

@@ -0,0 +1,26 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker:
+  image: gcr.io/repo-automation-bots/owlbot-python:latest
+
+deep-remove-regex:
+  - /owl-bot-staging
+
+deep-copy-regex:
+  - source: /google/pubsub/(v.*)/.*-py/(.*)
+    dest: /owl-bot-staging/$1/$2
+
+begin-after-commit-hash: 40278112d2922ec917140dcb5cc6d5ef2923aeb2
+

.pre-commit-config.yaml

@@ -26,6 +26,6 @@ repos:
     hooks:
     - id: black
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.0
+    rev: 3.9.1
     hooks:
     - id: flake8

CONTRIBUTING.rst

@@ -160,21 +160,7 @@ Running System Tests
   auth settings and change some configuration in your project to
   run all the tests.
 
-- System tests will be run against an actual project and
-  so you'll need to provide some environment variables to facilitate
-  authentication to your project:
-
-  - ``GOOGLE_APPLICATION_CREDENTIALS``: The path to a JSON key file;
-    Such a file can be downloaded directly from the developer's console by clicking
-    "Generate new JSON key". See private key
-    `docs <https://cloud.google.com/storage/docs/authentication#generating-a-private-key>`__
-    for more details.
-
-- Once you have downloaded your json keys, set the environment variable 
-  ``GOOGLE_APPLICATION_CREDENTIALS`` to the absolute path of the json file::
-
-   $ export GOOGLE_APPLICATION_CREDENTIALS="/Users/<your_username>/path/to/app_credentials.json"
-
+- System tests will be run against an actual project. You should use local credentials from gcloud when possible. See `Best practices for application authentication <https://cloud.google.com/docs/authentication/best-practices-applications#local_development_and_testing_with_the>`__. Some tests require a service account. For those tests see `Authenticating as a service account <https://cloud.google.com/docs/authentication/production>`__.
 
 *************
 Test Coverage

noxfile.py

@@ -62,16 +62,9 @@ def lint(session):
     session.run("flake8", "google", "tests")
 
 
-@nox.session(python="3.6")
+@nox.session(python=DEFAULT_PYTHON_VERSION)
 def blacken(session):
-    """Run black.
-
-    Format code to uniform standard.
-
-    This currently uses Python 3.6 due to the automated Kokoro run of synthtool.
-    That run uses an image that doesn't have 3.6 installed. Before updating this
-    check the state of the `gcp_ubuntu_config` we use for that Kokoro run.
-    """
+    """Run black. Format code to uniform standard."""
     session.install(BLACK_VERSION)
     session.run(
         "black", *BLACK_PATHS,
@@ -131,9 +124,6 @@ def system(session):
     # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true.
     if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false":
         session.skip("RUN_SYSTEM_TESTS is set to false, skipping")
-    # Sanity check: Only run tests if the environment variable is set.
-    if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""):
-        session.skip("Credentials must be set via environment variable")
     # Install pyopenssl for mTLS testing.
     if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true":
         session.install("pyopenssl")

owlbot.py

@@ -0,0 +1,267 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This script is used to synthesize generated parts of this library."""
+
+import re
+import textwrap
+
+import synthtool as s
+from synthtool import gcp
+from synthtool.languages import python
+
+common = gcp.CommonTemplates()
+
+default_version = "v1"
+
+for library in s.get_staging_dirs(default_version):    
+    # DEFAULT SCOPES and SERVICE_ADDRESS are being used. so let's force them in.
+    s.replace(
+        library / f"google/pubsub_{library.name}/services/*er/*client.py",
+        r"DEFAULT_ENDPOINT = 'pubsub\.googleapis\.com'",
+        """
+    # The scopes needed to make gRPC calls to all of the methods defined in
+    # this service
+    _DEFAULT_SCOPES = (
+        'https://www.googleapis.com/auth/cloud-platform',
+        'https://www.googleapis.com/auth/pubsub',
+    )
+
+    SERVICE_ADDRESS = "pubsub.googleapis.com:443"
+    \"""The default address of the service.\"""
+
+    \g<0>""",
+    )
+
+    # Modify GRPC options in transports.
+    count = s.replace(
+        [
+            library / f"google/pubsub_{library.name}/services/*/transports/grpc*", 
+            library / f"tests/unit/gapic/pubsub_{library.name}/*"
+        ],
+        "options=\[.*?\]",
+        """options=[
+                    ("grpc.max_send_message_length", -1),
+                    ("grpc.max_receive_message_length", -1),
+                    ("grpc.keepalive_time_ms", 30000),
+                ]""",
+        flags=re.MULTILINE | re.DOTALL,
+    )
+
+    if count < 15:
+        raise Exception("Expected replacements for gRPC channel options not made.")
+
+    # If the emulator is used, force an insecure gRPC channel to avoid SSL errors.
+    clients_to_patch = [
+        library / f"google/pubsub_{library.name}/services/publisher/client.py",
+        library / f"google/pubsub_{library.name}/services/subscriber/client.py",
+    ]
+    err_msg = "Expected replacements for gRPC channel to use with the emulator not made."
+
+    count = s.replace(
+        clients_to_patch,
+        r"import os",
+        "import functools\n\g<0>"
+    )
+
+    if count < len(clients_to_patch):
+        raise Exception(err_msg)
+
+    count = s.replace(
+        clients_to_patch,
+        f"from google\.pubsub_{library.name}\.types import pubsub",
+        "\g<0>\n\nimport grpc"
+    )
+
+    if count < len(clients_to_patch):
+        raise Exception(err_msg)
+
+    count = s.replace(
+        clients_to_patch,
+        r"Transport = type\(self\)\.get_transport_class\(transport\)",
+        """\g<0>
+
+            emulator_host = os.environ.get("PUBSUB_EMULATOR_HOST")
+            if emulator_host:
+                if issubclass(Transport, type(self)._transport_registry["grpc"]):
+                    channel = grpc.insecure_channel(target=emulator_host)
+                else:
+                    channel = grpc.aio.insecure_channel(target=emulator_host)
+                Transport = functools.partial(Transport, channel=channel)
+
+    """,
+    )
+
+    if count < len(clients_to_patch):
+        raise Exception(err_msg)
+
+    # Monkey patch the streaming_pull() GAPIC method to disable pre-fetching stream
+    # results.
+    s.replace(
+        library / f"google/pubsub_{library.name}/services/subscriber/client.py",
+        (
+            r"# Wrap the RPC method.*\n"
+            r"\s+# and friendly error.*\n"
+            r"\s+rpc = self\._transport\._wrapped_methods\[self\._transport\.streaming_pull\]"
+        ),
+        """
+        # Wrappers in api-core should not automatically pre-fetch the first
+        # stream result, as this breaks the stream when re-opening it.
+        # https://github.com/googleapis/python-pubsub/issues/93#issuecomment-630762257
+        self._transport.streaming_pull._prefetch_first_result_ = False
+
+        \g<0>""",
+    )
+
+    # Emit deprecation warning if return_immediately flag is set with synchronous pull.
+    s.replace(
+        library / f"google/pubsub_{library.name}/services/subscriber/*client.py",
+        r"import pkg_resources",
+        "import warnings\n\g<0>",
+    )
+
+    count = s.replace(
+        library / f"google/pubsub_{library.name}/services/subscriber/*client.py",
+        r"""
+    ([^\n\S]+(?:async\ )?def\ pull\(.*?->\ pubsub\.PullResponse:.*?)
+    ((?P<indent>[^\n\S]+)\#\ Wrap\ the\ RPC\ method)
+    """,
+        textwrap.dedent(
+        """
+    \g<1>
+    \g<indent>if request.return_immediately:
+    \g<indent>    warnings.warn(
+    \g<indent>        "The return_immediately flag is deprecated and should be set to False.",
+    \g<indent>        category=DeprecationWarning,
+    \g<indent>    )
+
+    \g<2>"""),
+        flags=re.MULTILINE | re.DOTALL | re.VERBOSE,
+    )
+
+    if count != 2:
+        raise Exception("Too many or too few replacements in pull() methods.")
+
+    # Silence deprecation warnings in pull() method flattened parameter tests.
+    s.replace(
+        library / f"tests/unit/gapic/pubsub_{library.name}/test_subscriber.py",
+        "import mock",
+        "\g<0>\nimport warnings",
+    )
+
+    count = s.replace(
+        library / f"tests/unit/gapic/pubsub_{library.name}/test_subscriber.py",
+        textwrap.dedent(
+        r"""
+        ([^\n\S]+# Call the method with a truthy value for each flattened field,
+        [^\n\S]+# using the keyword arguments to the method\.)
+        \s+(client\.pull\(.*?\))"""
+        ),
+        """\n\g<1>
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", category=DeprecationWarning)
+            \g<2>""",
+        flags = re.MULTILINE | re.DOTALL,
+    )
+
+    if count < 1:
+        raise Exception("Catch warnings replacement failed.")
+
+    count = s.replace(
+        library / f"tests/unit/gapic/pubsub_{library.name}/test_subscriber.py",
+        textwrap.dedent(
+            r"""
+        ([^\n\S]+# Call the method with a truthy value for each flattened field,
+        [^\n\S]+# using the keyword arguments to the method\.)
+        \s+response = (await client\.pull\(.*?\))"""
+        ),
+        """\n\g<1>
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", category=DeprecationWarning)
+            \g<2>""",
+        flags = re.MULTILINE | re.DOTALL,
+    )
+
+    if count < 1:
+        raise Exception("Catch warnings replacement failed.")
+
+    # Make sure that client library version is present in user agent header.
+    s.replace(
+        [
+            library / f"google/pubsub_{library.name}/services/publisher/async_client.py",
+            library / f"google/pubsub_{library.name}/services/publisher/client.py",
+            library / f"google/pubsub_{library.name}/services/publisher/transports/base.py",
+            library / f"google/pubsub_{library.name}/services/schema_service/async_client.py",
+            library / f"google/pubsub_{library.name}/services/schema_service/client.py",
+            library / f"google/pubsub_{library.name}/services/schema_service/transports/base.py",
+            library / f"google/pubsub_{library.name}/services/subscriber/async_client.py",
+            library / f"google/pubsub_{library.name}/services/subscriber/client.py",
+            library / f"google/pubsub_{library.name}/services/subscriber/transports/base.py",
+        ],
+        r"""gapic_version=(pkg_resources\.get_distribution\(\s+)['"]google-pubsub['"]""",
+        "client_library_version=\g<1>'google-cloud-pubsub'",
+    )
+
+    # Docstrings of *_iam_policy() methods are formatted poorly and must be fixed
+    # in order to avoid docstring format warnings in docs.
+    s.replace(library / f"google/pubsub_{library.name}/services/*er/client.py", r"(\s+)Args:", "\n\g<1>Args:")
+    s.replace(
+        library / f"google/pubsub_{library.name}/services/*er/client.py",
+        r"(\s+)\*\*JSON Example\*\*\s+::",
+        "\n\g<1>**JSON Example**::\n",
+    )
+
+    s.replace(
+        library / f"google/pubsub_{library.name}/services/*er/client.py",
+        r"(\s+)\*\*YAML Example\*\*\s+::",
+        "\n\g<1>**YAML Example**::\n",
+    )
+
+    s.replace(
+        library / f"google/pubsub_{library.name}/services/*er/client.py",
+        r"(\s+)For a description of IAM and its features, see",
+        "\n\g<0>",
+    )
+
+    s.move(
+        library,
+        excludes=[
+            "docs/**/*",
+            "nox.py",
+            "README.rst",
+            "setup.py",
+            f"google/cloud/pubsub_{library.name}/__init__.py",
+            f"google/cloud/pubsub_{library.name}/types.py",
+        ],
+    )
+
+s.remove_staging_dirs()
+
+# ----------------------------------------------------------------------------
+# Add templated files
+# ----------------------------------------------------------------------------
+templated_files = gcp.CommonTemplates().py_library(
+    microgenerator=True,
+    samples=True,
+    cov_level=100,
+    system_test_external_dependencies=["psutil"],
+)
+s.move(templated_files, excludes=[".coveragerc"])
+
+# ----------------------------------------------------------------------------
+# Samples templates
+# ----------------------------------------------------------------------------
+python.py_samples()
+
+s.shell.run(["nox", "-s", "blacken"], hide_output=False)