ci: run samples under Python 3.9 / 3.10 (#478)

* ci: run samples under Python 3.9 / 3.10

Refresh each sample's noxfile via:

----------------------------- %< -----------------------------
$ for noxfile in samples/*/noxfile.py; do
    echo "Refreshing $noxfile";
    wget -O $noxfile https://github.com/GoogleCloudPlatform/python-docs-samples/raw/main/noxfile-template.py
    echo "Blackening samples for $noxfile"
    nox -f $noxfile -s blacken
done
----------------------------- %< -----------------------------

Closes #477.

* fix: disable install-from-sorce for beam sample

Per #203.

* fix: skip beam sample for Python 3.10

Beam-related wheels are not yet available.

* fix: also refresh noxfiles for 'samples/snippets'

* ci: don't enforce type hints on old samples

* resolve issue where samples templates are not updated

* 🦉 Updates from OwlBot

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* resolve mypy error Name __path__ already defined

* add workaroud from PR #203

Co-authored-by: Anthonios Partheniou <partheniou@google.com>
Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

google/__init__.py

@@ -1,10 +1,6 @@
-from typing import List
-
 try:
     import pkg_resources
 
     pkg_resources.declare_namespace(__name__)
 except ImportError:
-    import pkgutil
-
-    __path__: List[str] = pkgutil.extend_path(__path__, __name__)
+    pass

google/cloud/__init__.py

@@ -1,10 +1,6 @@
-from typing import List
-
 try:
     import pkg_resources
 
     pkg_resources.declare_namespace(__name__)
 except ImportError:
-    import pkgutil
-
-    __path__: List[str] = pkgutil.extend_path(__path__, __name__)
+    pass

owlbot.py

@@ -217,20 +217,15 @@ def lint_setup_py(session):
 # Samples templates
 # ----------------------------------------------------------------------------
 
-sample_files = common.py_samples(samples=True)
-for path in sample_files:
-    s.move(path)
-
-# Note: python-docs-samples is not yet using 'main':
-#s.replace(
-#    "samples/**/*.md",
-#    r"python-docs-samples/blob/master/",
-#    "python-docs-samples/blob/main/",
-#)
+python.py_samples(skip_readmes=True)
+
 s.replace(
-    "samples/**/*.md",
-    r"google-cloud-python/blob/master/",
-    "google-cloud-python/blob/main/",
-)
+    "samples/beam/noxfile.py",
+    """INSTALL_LIBRARY_FROM_SOURCE \= os.environ.get\("INSTALL_LIBRARY_FROM_SOURCE", False\) in \(
+    "True",
+    "true",
+\)""",
+    """# todo(kolea2): temporary workaround to install pinned dep version
+INSTALL_LIBRARY_FROM_SOURCE = False""")
 
 s.shell.run(["nox", "-s", "blacken"], hide_output=False)

samples/beam/hello_world_write.py

@@ -23,42 +23,44 @@ class BigtableOptions(PipelineOptions):
     @classmethod
     def _add_argparse_args(cls, parser):
         parser.add_argument(
-            '--bigtable-project',
-            help='The Bigtable project ID, this can be different than your '
-                 'Dataflow project',
-            default='bigtable-project')
+            "--bigtable-project",
+            help="The Bigtable project ID, this can be different than your "
+            "Dataflow project",
+            default="bigtable-project",
+        )
         parser.add_argument(
-            '--bigtable-instance',
-            help='The Bigtable instance ID',
-            default='bigtable-instance')
+            "--bigtable-instance",
+            help="The Bigtable instance ID",
+            default="bigtable-instance",
+        )
         parser.add_argument(
-            '--bigtable-table',
-            help='The Bigtable table ID in the instance.',
-            default='bigtable-table')
+            "--bigtable-table",
+            help="The Bigtable table ID in the instance.",
+            default="bigtable-table",
+        )
 
 
 class CreateRowFn(beam.DoFn):
     def process(self, key):
         direct_row = row.DirectRow(row_key=key)
         direct_row.set_cell(
-            "stats_summary",
-            b"os_build",
-            b"android",
-            datetime.datetime.now())
+            "stats_summary", b"os_build", b"android", datetime.datetime.now()
+        )
         return [direct_row]
 
 
 def run(argv=None):
     """Build and run the pipeline."""
     options = BigtableOptions(argv)
     with beam.Pipeline(options=options) as p:
-        p | beam.Create(["phone#4c410523#20190501",
-                         "phone#4c410523#20190502"]) | beam.ParDo(
-            CreateRowFn()) | WriteToBigTable(
+        p | beam.Create(
+            ["phone#4c410523#20190501", "phone#4c410523#20190502"]
+        ) | beam.ParDo(CreateRowFn()) | WriteToBigTable(
             project_id=options.bigtable_project,
             instance_id=options.bigtable_instance,
-            table_id=options.bigtable_table)
+            table_id=options.bigtable_table,
+        )
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     run()

samples/beam/hello_world_write_test.py

@@ -19,9 +19,9 @@
 
 import hello_world_write
 
-PROJECT = os.environ['GOOGLE_CLOUD_PROJECT']
-BIGTABLE_INSTANCE = os.environ['BIGTABLE_INSTANCE']
-TABLE_ID_PREFIX = 'mobile-time-series-{}'
+PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"]
+BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"]
+TABLE_ID_PREFIX = "mobile-time-series-{}"
 
 
 @pytest.fixture(scope="module", autouse=True)
@@ -34,17 +34,20 @@ def table_id():
     if table.exists():
         table.delete()
 
-    table.create(column_families={'stats_summary': None})
+    table.create(column_families={"stats_summary": None})
     yield table_id
 
     table.delete()
 
 
 def test_hello_world_write(table_id):
-    hello_world_write.run([
-        '--bigtable-project=%s' % PROJECT,
-        '--bigtable-instance=%s' % BIGTABLE_INSTANCE,
-        '--bigtable-table=%s' % table_id])
+    hello_world_write.run(
+        [
+            "--bigtable-project=%s" % PROJECT,
+            "--bigtable-instance=%s" % BIGTABLE_INSTANCE,
+            "--bigtable-table=%s" % table_id,
+        ]
+    )
 
     client = bigtable.Client(project=PROJECT, admin=True)
     instance = client.instance(BIGTABLE_INSTANCE)

samples/beam/noxfile.py

@@ -17,6 +17,7 @@
 import os
 from pathlib import Path
 import sys
+from typing import Callable, Dict, List, Optional
 
 import nox
 
@@ -27,8 +28,9 @@
 # WARNING - WARNING - WARNING - WARNING - WARNING
 # WARNING - WARNING - WARNING - WARNING - WARNING
 
-# Copy `noxfile_config.py` to your directory and modify it instead.
+BLACK_VERSION = "black==19.10b0"
 
+# Copy `noxfile_config.py` to your directory and modify it instead.
 
 # `TEST_CONFIG` dict is a configuration hook that allows users to
 # modify the test configurations. The values here should be in sync
@@ -37,24 +39,29 @@
 
 TEST_CONFIG = {
     # You can opt out from the test for specific Python versions.
-    'ignored_versions': ["2.7"],
-
+    "ignored_versions": [],
+    # Old samples are opted out of enforcing Python type hints
+    # All new samples should feature them
+    "enforce_type_hints": False,
     # An envvar key for determining the project id to use. Change it
     # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a
     # build specific Cloud project. You can also use your own string
     # to use your own Cloud project.
-    'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT',
+    "gcloud_project_env": "GOOGLE_CLOUD_PROJECT",
     # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT',
-
+    # If you need to use a specific version of pip,
+    # change pip_version_override to the string representation
+    # of the version number, for example, "20.2.4"
+    "pip_version_override": None,
     # A dictionary you want to inject into your test. Don't put any
     # secrets here. These values will override predefined values.
-    'envs': {},
+    "envs": {},
 }
 
 
 try:
     # Ensure we can import noxfile_config in the project's directory.
-    sys.path.append('.')
+    sys.path.append(".")
     from noxfile_config import TEST_CONFIG_OVERRIDE
 except ImportError as e:
     print("No user noxfile_config found: detail: {}".format(e))
@@ -64,37 +71,41 @@
 TEST_CONFIG.update(TEST_CONFIG_OVERRIDE)
 
 
-def get_pytest_env_vars():
+def get_pytest_env_vars() -> Dict[str, str]:
     """Returns a dict for pytest invocation."""
     ret = {}
 
     # Override the GCLOUD_PROJECT and the alias.
-    env_key = TEST_CONFIG['gcloud_project_env']
+    env_key = TEST_CONFIG["gcloud_project_env"]
     # This should error out if not set.
-    ret['GOOGLE_CLOUD_PROJECT'] = os.environ[env_key]
+    ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key]
 
     # Apply user supplied envs.
-    ret.update(TEST_CONFIG['envs'])
+    ret.update(TEST_CONFIG["envs"])
     return ret
 
 
 # DO NOT EDIT - automatically generated.
-# All versions used to tested samples.
-ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"]
+# All versions used to test samples.
+ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"]
 
 # Any default versions that should be ignored.
-IGNORED_VERSIONS = TEST_CONFIG['ignored_versions']
+IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"]
 
 TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS])
 
 # todo(kolea2): temporary workaround to install pinned dep version
 INSTALL_LIBRARY_FROM_SOURCE = False
+
+# Error if a python version is missing
+nox.options.error_on_missing_interpreters = True
+
 #
 # Style Checks
 #
 
 
-def _determine_local_import_names(start_dir):
+def _determine_local_import_names(start_dir: str) -> List[str]:
     """Determines all import names that should be considered "local".
 
     This is used when running the linter to insure that import order is
@@ -132,18 +143,34 @@ def _determine_local_import_names(start_dir):
 
 
 @nox.session
-def lint(session):
-    session.install("flake8", "flake8-import-order")
+def lint(session: nox.sessions.Session) -> None:
+    if not TEST_CONFIG["enforce_type_hints"]:
+        session.install("flake8", "flake8-import-order")
+    else:
+        session.install("flake8", "flake8-import-order", "flake8-annotations")
 
     local_names = _determine_local_import_names(".")
     args = FLAKE8_COMMON_ARGS + [
         "--application-import-names",
         ",".join(local_names),
-        "."
+        ".",
     ]
     session.run("flake8", *args)
 
 
+#
+# Black
+#
+
+
+@nox.session
+def blacken(session: nox.sessions.Session) -> None:
+    session.install(BLACK_VERSION)
+    python_files = [path for path in os.listdir(".") if path.endswith(".py")]
+
+    session.run("black", *python_files)
+
+
 #
 # Sample Tests
 #
@@ -152,13 +179,24 @@ def lint(session):
 PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"]
 
 
-def _session_tests(session, post_install=None):
+def _session_tests(
+    session: nox.sessions.Session, post_install: Callable = None
+) -> None:
+    if TEST_CONFIG["pip_version_override"]:
+        pip_version = TEST_CONFIG["pip_version_override"]
+        session.install(f"pip=={pip_version}")
     """Runs py.test for a particular project."""
     if os.path.exists("requirements.txt"):
-        session.install("-r", "requirements.txt")
+        if os.path.exists("constraints.txt"):
+            session.install("-r", "requirements.txt", "-c", "constraints.txt")
+        else:
+            session.install("-r", "requirements.txt")
 
     if os.path.exists("requirements-test.txt"):
-        session.install("-r", "requirements-test.txt")
+        if os.path.exists("constraints-test.txt"):
+            session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt")
+        else:
+            session.install("-r", "requirements-test.txt")
 
     if INSTALL_LIBRARY_FROM_SOURCE:
         session.install("-e", _get_repo_root())
@@ -173,27 +211,27 @@ def _session_tests(session, post_install=None):
         # on travis where slow and flaky tests are excluded.
         # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html
         success_codes=[0, 5],
-        env=get_pytest_env_vars()
+        env=get_pytest_env_vars(),
     )
 
 
 @nox.session(python=ALL_VERSIONS)
-def py(session):
+def py(session: nox.sessions.Session) -> None:
     """Runs py.test for a sample using the specified version of Python."""
     if session.python in TESTED_VERSIONS:
         _session_tests(session)
     else:
-        session.skip("SKIPPED: {} tests are disabled for this sample.".format(
-            session.python
-        ))
+        session.skip(
+            "SKIPPED: {} tests are disabled for this sample.".format(session.python)
+        )
 
 
 #
 # Readmegen
 #
 
 
-def _get_repo_root():
+def _get_repo_root() -> Optional[str]:
     """ Returns the root folder of the project. """
     # Get root of this repository. Assume we don't have directories nested deeper than 10 items.
     p = Path(os.getcwd())
@@ -202,6 +240,11 @@ def _get_repo_root():
             break
         if Path(p / ".git").exists():
             return str(p)
+        # .git is not available in repos cloned via Cloud Build
+        # setup.py is always in the library's root, so use that instead
+        # https://github.com/googleapis/synthtool/issues/792
+        if Path(p / "setup.py").exists():
+            return str(p)
         p = p.parent
     raise Exception("Unable to detect repository root.")
 
@@ -211,7 +254,7 @@ def _get_repo_root():
 
 @nox.session
 @nox.parametrize("path", GENERATED_READMES)
-def readmegen(session, path):
+def readmegen(session: nox.sessions.Session, path: str) -> None:
     """(Re-)generates the readme for a sample."""
     session.install("jinja2", "pyyaml")
     dir_ = os.path.dirname(path)