From 11bc9c2b47a3185bd96150ade5cead34d6408b65 Mon Sep 17 00:00:00 2001
From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com>
Date: Thu, 22 Jul 2021 00:46:35 -0600
Subject: [PATCH] chore: fix release build (#659)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* chore: fix release build

Fix release build by migrating to secret manager secrets and use templated kokoro configs for docs/ and release/

* fix: fix config names

* chore: add populate secrets script

* docs: fix license

* chore: preserve original year

* chore: revert years in manually committed files

* 🦉 Updates from OwlBot

See https://github.com/googleapis/repo-automation-bots/blob/master/packages/owl-bot/README.md

* chore: update lockfile

* chore: add .kokoro/docker directory

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
---
 .github/.OwlBot.lock.yaml             |  9 +++++
 .github/.OwlBot.yaml                  |  8 +++++
 .kokoro/build.sh                      | 14 ++------
 .kokoro/docker/docs/Dockerfile        | 49 +++------------------------
 .kokoro/docker/docs/fetch_gpg_keys.sh | 18 +++-------
 .kokoro/docs/common.cfg               |  2 +-
 .kokoro/populate-secrets.sh           | 35 +++++++++++++++++++
 .kokoro/publish-docs.sh               | 14 ++------
 .kokoro/release.sh                    | 20 +++--------
 .kokoro/release/common.cfg            | 42 +++--------------------
 .kokoro/release/release.cfg           |  2 +-
 .kokoro/trampoline.sh                 | 31 ++++++++---------
 .kokoro/trampoline_v2.sh              | 14 ++------
 .trampolinerc                         |  2 +-
 owlbot.py                             | 42 +++++++++++++++++++++++
 15 files changed, 139 insertions(+), 163 deletions(-)
 create mode 100644 .github/.OwlBot.lock.yaml
 create mode 100644 .github/.OwlBot.yaml
 create mode 100755 .kokoro/populate-secrets.sh
 create mode 100644 owlbot.py

diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml
new file mode 100644
index 0000000000..9ce8396fe0
--- /dev/null
+++ b/.github/.OwlBot.lock.yaml
@@ -0,0 +1,9 @@
+# Copyright 2021 Google LLC
+#
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+
+docker:
+  image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest
+  digest: sha256:82d046d572a11b84d64d4b9af93ad55a1b6a4854917504ba7557acdfce907dde
diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml
new file mode 100644
index 0000000000..1bfc664928
--- /dev/null
+++ b/.github/.OwlBot.yaml
@@ -0,0 +1,8 @@
+# Copyright 2021 Google LLC
+#
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+
+docker:
+  image: gcr.io/repo-automation-bots/owlbot-python:latest
diff --git a/.kokoro/build.sh b/.kokoro/build.sh
index 4ac9dd8b76..eea1d84933 100755
--- a/.kokoro/build.sh
+++ b/.kokoro/build.sh
@@ -1,17 +1,9 @@
 #!/bin/bash
 # Copyright 2020 Google LLC
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
 
 set -eo pipefail
 
diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile
index 412b0b56a9..92ade8f99f 100644
--- a/.kokoro/docker/docs/Dockerfile
+++ b/.kokoro/docker/docs/Dockerfile
@@ -1,16 +1,8 @@
 # Copyright 2020 Google LLC
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
 
 from ubuntu:20.04
 
@@ -40,6 +32,7 @@ RUN apt-get update \
     libssl-dev \
     libsqlite3-dev \
     portaudio19-dev \
+    python3-distutils \
     redis-server \
     software-properties-common \
     ssh \
@@ -59,40 +52,8 @@ RUN apt-get update \
   && rm -rf /var/lib/apt/lists/* \
   && rm -f /var/cache/apt/archives/*.deb
 
-
-COPY fetch_gpg_keys.sh /tmp
-# Install the desired versions of Python.
-RUN set -ex \
-    && export GNUPGHOME="$(mktemp -d)" \
-    && echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \
-    && /tmp/fetch_gpg_keys.sh \
-    && for PYTHON_VERSION in 3.7.8 3.8.5; do \
-        wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
-        && wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
-        && gpg --batch --verify python-${PYTHON_VERSION}.tar.xz.asc python-${PYTHON_VERSION}.tar.xz \
-        && rm -r python-${PYTHON_VERSION}.tar.xz.asc \
-        && mkdir -p /usr/src/python-${PYTHON_VERSION} \
-        && tar -xJC /usr/src/python-${PYTHON_VERSION} --strip-components=1 -f python-${PYTHON_VERSION}.tar.xz \
-        && rm python-${PYTHON_VERSION}.tar.xz \
-        && cd /usr/src/python-${PYTHON_VERSION} \
-        && ./configure \
-            --enable-shared \
-            # This works only on Python 2.7 and throws a warning on every other
-            # version, but seems otherwise harmless.
-            --enable-unicode=ucs4 \
-            --with-system-ffi \
-            --without-ensurepip \
-        && make -j$(nproc) \
-        && make install \
-        && ldconfig \
-  ; done \
-  && rm -rf "${GNUPGHOME}" \
-  && rm -rf /usr/src/python* \
-  && rm -rf ~/.cache/
-
 RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \
-  && python3.7 /tmp/get-pip.py \
   && python3.8 /tmp/get-pip.py \
   && rm /tmp/get-pip.py
 
-CMD ["python3.7"]
+CMD ["python3.8"]
diff --git a/.kokoro/docker/docs/fetch_gpg_keys.sh b/.kokoro/docker/docs/fetch_gpg_keys.sh
index d653dd868e..c26695ddf0 100755
--- a/.kokoro/docker/docs/fetch_gpg_keys.sh
+++ b/.kokoro/docker/docs/fetch_gpg_keys.sh
@@ -1,21 +1,13 @@
 #!/bin/bash
 # Copyright 2020 Google LLC
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
 
 # A script to fetch gpg keys with retry.
 # Avoid jinja parsing the file.
-# 
+#
 
 function retry {
     if [[ "${#}" -le 1 ]]; then
@@ -42,4 +34,4 @@ retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
 retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
       E3FF2839C048B25C084DEBE9B26995E310250568
 
-# 
+#
diff --git a/.kokoro/docs/common.cfg b/.kokoro/docs/common.cfg
index 2003193838..cdf8e2a4ef 100644
--- a/.kokoro/docs/common.cfg
+++ b/.kokoro/docs/common.cfg
@@ -62,4 +62,4 @@ before_action {
       keyname: "docuploader_service_account"
     }
   }
-}
+}
\ No newline at end of file
diff --git a/.kokoro/populate-secrets.sh b/.kokoro/populate-secrets.sh
new file mode 100755
index 0000000000..7b511c3257
--- /dev/null
+++ b/.kokoro/populate-secrets.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright 2020 Google LLC
+#
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+
+set -eo pipefail
+
+function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;}
+function msg { println "$*" >&2 ;}
+function println { printf '%s\n' "$(now) $*" ;}
+
+
+# Populates requested secrets set in SECRET_MANAGER_KEYS from service account:
+# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com
+SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager"
+msg "Creating folder on disk for secrets: ${SECRET_LOCATION}"
+mkdir -p ${SECRET_LOCATION}
+for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g")
+do
+  msg "Retrieving secret ${key}"
+  docker run --entrypoint=gcloud \
+    --volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \
+    gcr.io/google.com/cloudsdktool/cloud-sdk \
+    secrets versions access latest \
+    --project cloud-devrel-kokoro-resources \
+    --secret ${key} > \
+    "${SECRET_LOCATION}/${key}"
+  if [[ $? == 0 ]]; then
+    msg "Secret written to ${SECRET_LOCATION}/${key}"
+  else
+    msg "Error retrieving secret ${key}"
+  fi
+done
diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh
index 8acb14e802..a80ea7c0a0 100755
--- a/.kokoro/publish-docs.sh
+++ b/.kokoro/publish-docs.sh
@@ -1,17 +1,9 @@
 #!/bin/bash
 # Copyright 2020 Google LLC
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
 
 set -eo pipefail
 
diff --git a/.kokoro/release.sh b/.kokoro/release.sh
index 9f60d5f251..57e2f28ce1 100755
--- a/.kokoro/release.sh
+++ b/.kokoro/release.sh
@@ -1,19 +1,9 @@
 #!/bin/bash
 # Copyright 2020 Google LLC
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#!/bin/bash
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
 
 set -eo pipefail
 
@@ -28,7 +18,7 @@ python3 -m pip install --upgrade twine wheel setuptools
 export PYTHONUNBUFFERED=1
 
 # Move into the package, build the distribution and upload.
-TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google_cloud_pypi_password")
+TWINE_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secret_manager/google-cloud-pypi-token")
 cd github/python-spanner-django
 python3 setup.py sdist bdist_wheel
-twine upload --username gcloudpypi --password "${TWINE_PASSWORD}" dist/*
+twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/*
diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg
index 31533b977a..2cf743579d 100644
--- a/.kokoro/release/common.cfg
+++ b/.kokoro/release/common.cfg
@@ -23,42 +23,8 @@ env_vars: {
     value: "github/python-spanner-django/.kokoro/release.sh"
 }
 
-# Fetch the token needed for reporting release status to GitHub
-before_action {
-  fetch_keystore {
-    keystore_resource {
-      keystore_config_id: 73713
-      keyname: "yoshi-automation-github-key"
-    }
-  }
-}
-
-# Fetch PyPI password
-before_action {
-  fetch_keystore {
-    keystore_resource {
-      keystore_config_id: 73713
-      keyname: "google_cloud_pypi_password"
-    }
-  }
-}
-
-# Fetch magictoken to use with Magic Github Proxy 
-before_action {
-  fetch_keystore {
-    keystore_resource {
-      keystore_config_id: 73713
-      keyname: "releasetool-magictoken"
-    }
-  }
-}
-
-# Fetch api key to use with Magic Github Proxy 
-before_action {
-  fetch_keystore {
-    keystore_resource {
-      keystore_config_id: 73713
-      keyname: "magic-github-proxy-api-key"
-    }
-  }
+# Tokens needed to report release status back to GitHub
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,google-cloud-pypi-token"
 }
diff --git a/.kokoro/release/release.cfg b/.kokoro/release/release.cfg
index 18a4c35325..8f43917d92 100644
--- a/.kokoro/release/release.cfg
+++ b/.kokoro/release/release.cfg
@@ -1 +1 @@
-# Format: //devtools/kokoro/config/proto/build.proto
+# Format: //devtools/kokoro/config/proto/build.proto
\ No newline at end of file
diff --git a/.kokoro/trampoline.sh b/.kokoro/trampoline.sh
index 8f0237f322..d42a40acbb 100755
--- a/.kokoro/trampoline.sh
+++ b/.kokoro/trampoline.sh
@@ -1,23 +1,20 @@
 #!/bin/bash
-# Copyright 2020 Google LLC.
+# Copyright 2017 Google LLC
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
 
 set -eo pipefail
 
-python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py"  || ret_code=$?
-
-chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh
-${KOKORO_GFILE_DIR}/trampoline_cleanup.sh || true
+# Always run the cleanup script, regardless of the success of bouncing into
+# the container.
+function cleanup() {
+    chmod +x ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh
+    ${KOKORO_GFILE_DIR}/trampoline_cleanup.sh
+    echo "cleanup";
+}
+trap cleanup EXIT
 
-exit ${ret_code}
+$(dirname $0)/populate-secrets.sh # Secret Manager secrets.
+python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py"
\ No newline at end of file
diff --git a/.kokoro/trampoline_v2.sh b/.kokoro/trampoline_v2.sh
index 4af6cdc26d..591faf816e 100755
--- a/.kokoro/trampoline_v2.sh
+++ b/.kokoro/trampoline_v2.sh
@@ -1,17 +1,9 @@
 #!/usr/bin/env bash
 # Copyright 2020 Google LLC
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
 
 # trampoline_v2.sh
 #
diff --git a/.trampolinerc b/.trampolinerc
index 28037bbb86..383b6ec89f 100644
--- a/.trampolinerc
+++ b/.trampolinerc
@@ -1,4 +1,4 @@
-# Copyright 2021 Google LLC
+# Copyright 2020 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/owlbot.py b/owlbot.py
new file mode 100644
index 0000000000..d0087b4c0b
--- /dev/null
+++ b/owlbot.py
@@ -0,0 +1,42 @@
+# Copyright 2021 Google LLC
+#
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd
+
+"""This script is used to synthesize generated parts of this library."""
+import re
+
+import synthtool as s
+import synthtool.gcp as gcp
+
+# ----------------------------------------------------------------------------
+# Add templated files
+# ----------------------------------------------------------------------------
+templated_files = gcp.CommonTemplates().py_library(microgenerator=True)
+
+# Just move templates for building docs and releases
+# Presubmit and continuous are configured differently
+s.move(templated_files / ".trampolinerc")
+s.move(templated_files / ".kokoro" / "docker")
+s.move(templated_files / ".kokoro" / "docs")
+s.move(templated_files / ".kokoro" / "release.sh")
+s.move(templated_files / ".kokoro" / "trampoline_v2.sh")
+s.move(templated_files / ".kokoro" / "trampoline.sh")
+s.move(templated_files / ".kokoro" / "populate-secrets.sh")
+s.move(templated_files / ".kokoro" / "release")
+
+# Replace the Apache Licenses in the `.kokoro` directory
+# with the BSD license expected in this repository
+s.replace(
+    ".kokoro/**/*",
+    "# Copyright.*(\d{4}).*# limitations under the License\.",
+    """# Copyright \g<1> Google LLC
+#
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file or at
+# https://developers.google.com/open-source/licenses/bsd""",
+    flags=re.DOTALL
+)
+
+s.shell.run(["nox", "-s", "blacken"], hide_output=False)