chore(CI): add snippet bot support (via synth) (#134)

Closes #125

.kokoro/populate-secrets.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Copyright 2020 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+
+function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;}
+function msg { println "$*" >&2 ;}
+function println { printf '%s\n' "$(now) $*" ;}
+
+
+# Populates requested secrets set in SECRET_MANAGER_KEYS from service account:
+# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com
+SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager"
+msg "Creating folder on disk for secrets: ${SECRET_LOCATION}"
+mkdir -p ${SECRET_LOCATION}
+for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g")
+do
+  msg "Retrieving secret ${key}"
+  docker run --entrypoint=gcloud \
+    --volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \
+    gcr.io/google.com/cloudsdktool/cloud-sdk \
+    secrets versions access latest \
+    --project cloud-devrel-kokoro-resources \
+    --secret ${key} > \
+    "${SECRET_LOCATION}/${key}"
+  if [[ $? == 0 ]]; then
+    msg "Secret written to ${SECRET_LOCATION}/${key}"
+  else
+    msg "Error retrieving secret ${key}"
+  fi
+done

.kokoro/release/common.cfg

@@ -23,42 +23,18 @@ env_vars: {
     value: "github/python-bigtable/.kokoro/release.sh"
 }
 
-# Fetch the token needed for reporting release status to GitHub
-before_action {
-  fetch_keystore {
-    keystore_resource {
-      keystore_config_id: 73713
-      keyname: "yoshi-automation-github-key"
-    }
-  }
-}
-
-# Fetch PyPI password
-before_action {
-  fetch_keystore {
-    keystore_resource {
-      keystore_config_id: 73713
-      keyname: "google_cloud_pypi_password"
-    }
-  }
-}
-
-# Fetch magictoken to use with Magic Github Proxy 
-before_action {
-  fetch_keystore {
-    keystore_resource {
-      keystore_config_id: 73713
-      keyname: "releasetool-magictoken"
-    }
-  }
+# Fetch PyPI password	
+before_action {	
+  fetch_keystore {	
+    keystore_resource {	
+      keystore_config_id: 73713	
+      keyname: "google_cloud_pypi_password"	
+    }	
+  }	
 }
 
-# Fetch api key to use with Magic Github Proxy 
-before_action {
-  fetch_keystore {
-    keystore_resource {
-      keystore_config_id: 73713
-      keyname: "magic-github-proxy-api-key"
-    }
-  }
-}
+# Tokens needed to report release status back to GitHub
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem"
+}

.kokoro/trampoline.sh

@@ -15,9 +15,14 @@
 
 set -eo pipefail
 
-python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py"  || ret_code=$?
+# Always run the cleanup script, regardless of the success of bouncing into
+# the container.
+function cleanup() {
+    chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh
+    ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh
+    echo "cleanup";
+}
+trap cleanup EXIT
 
-chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh
-${KOKORO_GFILE_DIR}/trampoline_cleanup.sh || true
-
-exit ${ret_code}
+$(dirname $0)/populate-secrets.sh # Secret Manager secrets.
+python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py"

docs/conf.py

@@ -29,7 +29,7 @@
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
-needs_sphinx = "1.6.3"
+needs_sphinx = "1.5.5"
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
@@ -39,6 +39,7 @@
     "sphinx.ext.autosummary",
     "sphinx.ext.intersphinx",
     "sphinx.ext.coverage",
+    "sphinx.ext.doctest",
     "sphinx.ext.napoleon",
     "sphinx.ext.todo",
     "sphinx.ext.viewcode",

samples/hello/README.md

@@ -30,6 +30,7 @@ To run this sample:
         python main.py
 
 
+
 <pre>usage: main.py [-h] [--table TABLE] project_id instance_id<br>Demonstrates how to connect to Cloud Bigtable and run some basic operations.<br>Prerequisites: - Create a Cloud Bigtable cluster.<br>https://cloud.google.com/bigtable/docs/creating-cluster - Set your Google<br>Application Default Credentials.<br>https://developers.google.com/identity/protocols/application-default-<br>credentials<br><br><br>positional arguments:<br>&nbsp; project_id &nbsp; &nbsp; Your Cloud Platform project ID.<br>&nbsp; instance_id &nbsp; &nbsp;ID of the Cloud Bigtable instance to connect to.<br><br><br>optional arguments:<br>&nbsp; -h, --help &nbsp; &nbsp; show this help message and exit<br>&nbsp; --table TABLE &nbsp;Table to create and destroy. (default: Hello-Bigtable)</pre>
 
 ## Additional Information

samples/hello_happybase/README.md

@@ -30,6 +30,7 @@ To run this sample:
         python main.py
 
 
+
 <pre>usage: main.py [-h] [--table TABLE] project_id instance_id<br>Demonstrates how to connect to Cloud Bigtable and run some basic operations.<br>Prerequisites: - Create a Cloud Bigtable cluster.<br>https://cloud.google.com/bigtable/docs/creating-cluster - Set your Google<br>Application Default Credentials.<br>https://developers.google.com/identity/protocols/application-default-<br>credentials<br><br><br>positional arguments:<br>&nbsp; project_id &nbsp; &nbsp; Your Cloud Platform project ID.<br>&nbsp; instance_id &nbsp; &nbsp;ID of the Cloud Bigtable instance to connect to.<br><br><br>optional arguments:<br>&nbsp; -h, --help &nbsp; &nbsp; show this help message and exit<br>&nbsp; --table TABLE &nbsp;Table to create and destroy. (default: Hello-Bigtable)</pre>
 
 ## Additional Information

samples/instanceadmin/README.md

@@ -30,6 +30,7 @@ To run this sample:
         python instanceadmin.py
 
 
+
 <pre>usage: instanceadmin.py [-h] [run] [dev-instance] [del-instance] [add-cluster] [del-cluster] project_id instance_id cluster_id<br>Demonstrates how to connect to Cloud Bigtable and run some basic operations.<br>Prerequisites: - Create a Cloud Bigtable cluster.<br>https://cloud.google.com/bigtable/docs/creating-cluster - Set your Google<br>Application Default Credentials.<br>https://developers.google.com/identity/protocols/application-default-<br>credentials<br><br><br>positional arguments:<br>&nbsp; project_id &nbsp; &nbsp; Your Cloud Platform project ID.<br>&nbsp; instance_id &nbsp; &nbsp;ID of the Cloud Bigtable instance to connect to.<br><br><br>optional arguments:<br>&nbsp; -h, --help &nbsp; &nbsp; show this help message and exit<br>&nbsp; --table TABLE &nbsp;Table to create and destroy. (default: Hello-Bigtable)</pre>
 
 ## Additional Information

samples/metricscaler/README.md

@@ -30,6 +30,7 @@ To run this sample:
         python metricscaler.py
 
 
+
 <pre>usage: metricscaler.py [-h] [--high_cpu_threshold HIGH_CPU_THRESHOLD] [--low_cpu_threshold LOW_CPU_THRESHOLD] [--short_sleep SHORT_SLEEP] [--long_sleep LONG_SLEEP] bigtable_instance bigtable_cluster<br>usage: metricscaler.py [-h] [--high_cpu_threshold HIGH_CPU_THRESHOLD] <br>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;[--low_cpu_threshold LOW_CPU_THRESHOLD] <br>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;[--short_sleep SHORT_SLEEP] [--long_sleep LONG_SLEEP] <br>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;bigtable_instance bigtable_cluster <br><br> <br>Scales Cloud Bigtable clusters based on CPU usage. <br><br> <br>positional arguments: <br>&nbsp; bigtable_instance &nbsp; &nbsp; ID of the Cloud Bigtable instance to connect to. <br>&nbsp; bigtable_cluster &nbsp; &nbsp; &nbsp;ID of the Cloud Bigtable cluster to connect to. <br><br> <br>optional arguments: <br>&nbsp; -h, --help &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;show this help message and exit <br>&nbsp; --high_cpu_threshold HIGH_CPU_THRESHOLD <br>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; If Cloud Bigtable CPU usage is above this threshold, <br>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; scale up <br>&nbsp; --low_cpu_threshold LOW_CPU_THRESHOLD <br>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; If Cloud Bigtable CPU usage is below this threshold, <br>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; scale down <br>&nbsp; --short_sleep SHORT_SLEEP <br>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; How long to sleep in seconds between checking metrics <br>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; after no scale operation <br>&nbsp; --long_sleep LONG_SLEEP <br>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; How long to sleep in seconds between checking metrics <br>&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; after a scaling operation</pre>
 
 ## Additional Information

samples/quickstart/README.md

@@ -30,6 +30,7 @@ To run this sample:
         python main.py
 
 
+
 <pre>usage: main.py [-h] [--table TABLE] project_id instance_id <br><br> <br>positional arguments: <br>&nbsp; project_id &nbsp; &nbsp; Your Cloud Platform project ID. <br>&nbsp; instance_id &nbsp; &nbsp;ID of the Cloud Bigtable instance to connect to. <br><br> <br>optional arguments: <br>&nbsp; -h, --help &nbsp; &nbsp; show this help message and exit <br>&nbsp; --table TABLE &nbsp;Existing table used in the quickstart. (default: my-table)</pre>
 
 ## Additional Information

samples/quickstart_happybase/README.md

@@ -30,6 +30,7 @@ To run this sample:
         python main.py
 
 
+
 <pre>usage: main.py [-h] [--table TABLE] project_id instance_id<br>usage: main.py [-h] [--table TABLE] project_id instance_id <br><br> <br>positional arguments: <br>&nbsp; project_id &nbsp; &nbsp; Your Cloud Platform project ID. <br>&nbsp; instance_id &nbsp; &nbsp;ID of the Cloud Bigtable instance to connect to. <br><br> <br>optional arguments: <br>&nbsp; -h, --help &nbsp; &nbsp; show this help message and exit <br>&nbsp; --table TABLE &nbsp;Existing table used in the quickstart. (default: my-table)</code
 
 ## Additional Information

samples/snippets/README.md

@@ -13,6 +13,7 @@ This folder contains snippets for Python Cloud Bigtable.
 
 
 
+
 ## Additional Information
 
 You can read the documentation for more details on API usage and use GitHub

samples/tableadmin/README.md

@@ -30,6 +30,7 @@ To run this sample:
         python tableadmin.py
 
 
+
 <pre>usage: tableadmin.py [-h] [run] [delete] [--table TABLE] project_id instance_id <br><br> <br>Demonstrates how to connect to Cloud Bigtable and run some basic operations. <br>Prerequisites: - Create a Cloud Bigtable cluster. <br>https://cloud.google.com/bigtable/docs/creating-cluster - Set your Google <br>Application Default Credentials. <br>https://developers.google.com/identity/protocols/application-default- <br>credentials <br><br> <br>positional arguments: <br>&nbsp; project_id &nbsp; &nbsp; Your Cloud Platform project ID. <br>&nbsp; instance_id &nbsp; &nbsp;ID of the Cloud Bigtable instance to connect to. <br><br> <br>optional arguments: <br>&nbsp; -h, --help &nbsp; &nbsp; show this help message and exit <br>&nbsp; --table TABLE &nbsp;Table to create and destroy. (default: Hello-Bigtable)</pre>
 
 ## Additional Information

scripts/decrypt-secrets.sh

@@ -20,14 +20,27 @@ ROOT=$( dirname "$DIR" )
 # Work from the project root.
 cd $ROOT
 
+# Prevent it from overriding files.
+# We recommend that sample authors use their own service account files and cloud project.
+# In that case, they are supposed to prepare these files by themselves.
+if [[ -f "testing/test-env.sh" ]] || \
+       [[ -f "testing/service-account.json" ]] || \
+       [[ -f "testing/client-secrets.json" ]]; then
+    echo "One or more target files exist, aborting."
+    exit 1
+fi
+
 # Use SECRET_MANAGER_PROJECT if set, fallback to cloud-devrel-kokoro-resources.
 PROJECT_ID="${SECRET_MANAGER_PROJECT:-cloud-devrel-kokoro-resources}"
 
 gcloud secrets versions access latest --secret="python-docs-samples-test-env" \
+       --project="${PROJECT_ID}" \
        > testing/test-env.sh
 gcloud secrets versions access latest \
        --secret="python-docs-samples-service-account" \
+       --project="${PROJECT_ID}" \
        > testing/service-account.json
 gcloud secrets versions access latest \
        --secret="python-docs-samples-client-secrets" \
-       > testing/client-secrets.json
+       --project="${PROJECT_ID}" \
+       > testing/client-secrets.json

synth.metadata

@@ -3,30 +3,30 @@
     {
       "git": {
         "name": ".",
-        "remote": "https://github.com/googleapis/python-bigtable.git",
-        "sha": "8f764343e01d50ad880363f5a4e5630122cbdb25"
+        "remote": "git@github.com:googleapis/python-bigtable",
+        "sha": "474b9f3408efa102533da5b9066cba0cd1ff3a9a"
       }
     },
     {
       "git": {
         "name": "googleapis",
         "remote": "https://github.com/googleapis/googleapis.git",
-        "sha": "0dc0a6c0f1a9f979bc0690f0caa5fbafa3000c2c",
-        "internalRef": "327026955"
+        "sha": "8d73f9486fc193a150f6c907dfb9f49431aff3ff",
+        "internalRef": "332497859"
       }
     },
     {
       "git": {
         "name": "synthtool",
         "remote": "https://github.com/googleapis/synthtool.git",
-        "sha": "2e85c10b5153defd9d654c34b57e7e9263361959"
+        "sha": "9d216d21544b5c9f2c85a6380ffcf20b67e1e459"
       }
     },
     {
       "git": {
         "name": "synthtool",
         "remote": "https://github.com/googleapis/synthtool.git",
-        "sha": "2e85c10b5153defd9d654c34b57e7e9263361959"
+        "sha": "9d216d21544b5c9f2c85a6380ffcf20b67e1e459"
       }
     }
   ],
@@ -49,123 +49,5 @@
         "generator": "bazel"
       }
     }
-  ],
-  "generatedFiles": [
-    ".coveragerc",
-    ".flake8",
-    ".github/CONTRIBUTING.md",
-    ".github/ISSUE_TEMPLATE/bug_report.md",
-    ".github/ISSUE_TEMPLATE/feature_request.md",
-    ".github/ISSUE_TEMPLATE/support_request.md",
-    ".github/PULL_REQUEST_TEMPLATE.md",
-    ".github/release-please.yml",
-    ".gitignore",
-    ".kokoro/build.sh",
-    ".kokoro/continuous/common.cfg",
-    ".kokoro/continuous/continuous.cfg",
-    ".kokoro/docker/docs/Dockerfile",
-    ".kokoro/docker/docs/fetch_gpg_keys.sh",
-    ".kokoro/docs/common.cfg",
-    ".kokoro/docs/docs-presubmit.cfg",
-    ".kokoro/docs/docs.cfg",
-    ".kokoro/presubmit/common.cfg",
-    ".kokoro/presubmit/presubmit.cfg",
-    ".kokoro/publish-docs.sh",
-    ".kokoro/release.sh",
-    ".kokoro/release/common.cfg",
-    ".kokoro/release/release.cfg",
-    ".kokoro/samples/lint/common.cfg",
-    ".kokoro/samples/lint/continuous.cfg",
-    ".kokoro/samples/lint/periodic.cfg",
-    ".kokoro/samples/lint/presubmit.cfg",
-    ".kokoro/samples/python3.6/common.cfg",
-    ".kokoro/samples/python3.6/continuous.cfg",
-    ".kokoro/samples/python3.6/periodic.cfg",
-    ".kokoro/samples/python3.6/presubmit.cfg",
-    ".kokoro/samples/python3.7/common.cfg",
-    ".kokoro/samples/python3.7/continuous.cfg",
-    ".kokoro/samples/python3.7/periodic.cfg",
-    ".kokoro/samples/python3.7/presubmit.cfg",
-    ".kokoro/samples/python3.8/common.cfg",
-    ".kokoro/samples/python3.8/continuous.cfg",
-    ".kokoro/samples/python3.8/periodic.cfg",
-    ".kokoro/samples/python3.8/presubmit.cfg",
-    ".kokoro/test-samples.sh",
-    ".kokoro/trampoline.sh",
-    ".kokoro/trampoline_v2.sh",
-    ".trampolinerc",
-    "CODE_OF_CONDUCT.md",
-    "CONTRIBUTING.rst",
-    "LICENSE",
-    "MANIFEST.in",
-    "docs/_static/custom.css",
-    "docs/_templates/layout.html",
-    "docs/conf.py",
-    "docs/multiprocessing.rst",
-    "google/cloud/bigtable_admin_v2/__init__.py",
-    "google/cloud/bigtable_admin_v2/gapic/__init__.py",
-    "google/cloud/bigtable_admin_v2/gapic/bigtable_instance_admin_client.py",
-    "google/cloud/bigtable_admin_v2/gapic/bigtable_instance_admin_client_config.py",
-    "google/cloud/bigtable_admin_v2/gapic/bigtable_table_admin_client.py",
-    "google/cloud/bigtable_admin_v2/gapic/bigtable_table_admin_client_config.py",
-    "google/cloud/bigtable_admin_v2/gapic/enums.py",
-    "google/cloud/bigtable_admin_v2/gapic/transports/__init__.py",
-    "google/cloud/bigtable_admin_v2/gapic/transports/bigtable_instance_admin_grpc_transport.py",
-    "google/cloud/bigtable_admin_v2/gapic/transports/bigtable_table_admin_grpc_transport.py",
-    "google/cloud/bigtable_admin_v2/proto/__init__.py",
-    "google/cloud/bigtable_admin_v2/proto/bigtable_instance_admin.proto",
-    "google/cloud/bigtable_admin_v2/proto/bigtable_instance_admin_pb2.py",
-    "google/cloud/bigtable_admin_v2/proto/bigtable_instance_admin_pb2_grpc.py",
-    "google/cloud/bigtable_admin_v2/proto/bigtable_table_admin.proto",
-    "google/cloud/bigtable_admin_v2/proto/bigtable_table_admin_pb2.py",
-    "google/cloud/bigtable_admin_v2/proto/bigtable_table_admin_pb2_grpc.py",
-    "google/cloud/bigtable_admin_v2/proto/common.proto",
-    "google/cloud/bigtable_admin_v2/proto/common_pb2.py",
-    "google/cloud/bigtable_admin_v2/proto/common_pb2_grpc.py",
-    "google/cloud/bigtable_admin_v2/proto/instance.proto",
-    "google/cloud/bigtable_admin_v2/proto/instance_pb2.py",
-    "google/cloud/bigtable_admin_v2/proto/instance_pb2_grpc.py",
-    "google/cloud/bigtable_admin_v2/proto/table.proto",
-    "google/cloud/bigtable_admin_v2/proto/table_pb2.py",
-    "google/cloud/bigtable_admin_v2/proto/table_pb2_grpc.py",
-    "google/cloud/bigtable_admin_v2/types.py",
-    "google/cloud/bigtable_v2/__init__.py",
-    "google/cloud/bigtable_v2/gapic/__init__.py",
-    "google/cloud/bigtable_v2/gapic/bigtable_client.py",
-    "google/cloud/bigtable_v2/gapic/bigtable_client_config.py",
-    "google/cloud/bigtable_v2/gapic/transports/__init__.py",
-    "google/cloud/bigtable_v2/gapic/transports/bigtable_grpc_transport.py",
-    "google/cloud/bigtable_v2/proto/__init__.py",
-    "google/cloud/bigtable_v2/proto/bigtable.proto",
-    "google/cloud/bigtable_v2/proto/bigtable_pb2.py",
-    "google/cloud/bigtable_v2/proto/bigtable_pb2_grpc.py",
-    "google/cloud/bigtable_v2/proto/data.proto",
-    "google/cloud/bigtable_v2/proto/data_pb2.py",
-    "google/cloud/bigtable_v2/proto/data_pb2_grpc.py",
-    "google/cloud/bigtable_v2/types.py",
-    "renovate.json",
-    "samples/AUTHORING_GUIDE.md",
-    "samples/CONTRIBUTING.md",
-    "samples/README.md",
-    "samples/hello/README.md",
-    "samples/hello_happybase/README.md",
-    "samples/instanceadmin/README.md",
-    "samples/metricscaler/README.md",
-    "samples/quickstart/README.md",
-    "samples/quickstart_happybase/README.md",
-    "samples/snippets/README.md",
-    "samples/tableadmin/README.md",
-    "scripts/decrypt-secrets.sh",
-    "scripts/readme-gen/readme_gen.py",
-    "scripts/readme-gen/templates/README.tmpl.rst",
-    "scripts/readme-gen/templates/auth.tmpl.rst",
-    "scripts/readme-gen/templates/auth_api_key.tmpl.rst",
-    "scripts/readme-gen/templates/install_deps.tmpl.rst",
-    "scripts/readme-gen/templates/install_portaudio.tmpl.rst",
-    "setup.cfg",
-    "testing/.gitignore",
-    "tests/unit/gapic/v2/test_bigtable_client_v2.py",
-    "tests/unit/gapic/v2/test_bigtable_instance_admin_client_v2.py",
-    "tests/unit/gapic/v2/test_bigtable_table_admin_client_v2.py"
   ]
 }