docs: add docs build kokoro config for django spanner (#644)

* docs: added trampoline v2 file and docs config for kokoro * fix: change library_type to INTEGRATION * docs: add kokoro docker build for docs * fix:repo metadata * fix: updated PROJECT_ROOT for django spanner docs build using kokoro * fix: remove sudo apt update because sudo is not allowed * fix: add build_nox.sh as original build.sh file is used by kokoro workers to run parallelized tests * refactor: renamed build_nox.sh to build-nox.sh * refactor: change build-nox to nox.sh * refactor: change build-nox to nox.sh * refactor: change kokoro build to run different test for docs and different test for workers * refactor: change kokoro build to run different test for docs and different test for workers * refactor: add log msg for debugging * refactor: remove log msg added for debugging * fix: correct docs build command * refactor: remove unused nox file * feat: add .trampolinerc file * fix: correct repo_metadata release_level to beta
googleapis · Jun 17, 2021 · ecf241a · ecf241a
1 parent fe4a062
commit ecf241a
Show file tree

Hide file tree

Showing 10 changed files with 890 additions and 21 deletions.
diff --git a/.kokoro/build.sh b/.kokoro/build.sh
@@ -15,7 +15,11 @@
 
 set -eo pipefail
 
-cd github/python-spanner-django
+if [[ -z "${PROJECT_ROOT:-}" ]]; then
+    PROJECT_ROOT="github/python-spanner-django"
+fi
+
+cd "${PROJECT_ROOT}"
 
 # Disable buffering, so that the logs stream through.
 export PYTHONUNBUFFERED=1
@@ -27,26 +31,38 @@ export GOOGLE_CLOUD_PROJECT=$(cat "${KOKORO_GFILE_DIR}/project-id.json")
 
 export RUNNING_SPANNER_BACKEND_TESTS=1
 
-pip3 install .
-export DJANGO_TESTS_DIR="django_tests_dir"
-mkdir -p $DJANGO_TESTS_DIR && git clone --depth 1 --single-branch --branch "spanner/stable/2.2.x" https://github.com/c24t/django.git $DJANGO_TESTS_DIR/django
-
-# Install dependencies for Django tests.
-sudo apt-get update
-apt-get install -y libffi-dev libjpeg-dev zlib1g-dev libmemcached-dev
-cd $DJANGO_TESTS_DIR/django && pip3 install -e . && pip3 install -r tests/requirements/py3.txt; cd ../../
-
-# Hardcode the max number of workers since Spanner has a very low
-# QPS for administrative RPCs of 5QPS (averaged every 100 seconds)
-if [[ $KOKORO_JOB_NAME == *"continuous"* ]]
+if [[ $KOKORO_JOB_NAME == *"docs"* ]]
 then
-    # Disable continuous build as it creates too many Spanner instances
-    # ("Quota exceeded for quota metric 'Instance create requests' and
-    # limit 'Instance create requests per minute' of service
-    # 'spanner.googleapis.com').
-    export DJANGO_WORKER_COUNT=0
+    echo "Running docs generation."
+    # Remove old nox
+    python3 -m pip uninstall --yes --quiet nox-automation
+
+    # Install nox
+    python3 -m pip install --upgrade --quiet nox
+    # Generate docs.
+    python3 -m nox -s docs docfx
 else
-    export DJANGO_WORKER_COUNT=5
-fi
+    pip3 install .
+    export DJANGO_TESTS_DIR="django_tests_dir"
+    mkdir -p $DJANGO_TESTS_DIR && git clone --depth 1 --single-branch --branch "spanner/stable/2.2.x" https://github.com/c24t/django.git $DJANGO_TESTS_DIR/django
+
+    # Install dependencies for Django tests.
+    sudo apt-get update
+    apt-get install -y libffi-dev libjpeg-dev zlib1g-dev libmemcached-dev
+    cd $DJANGO_TESTS_DIR/django && pip3 install -e . && pip3 install -r tests/requirements/py3.txt; cd ../../
 
-python3 ./run_testing_worker.py
+    # Hardcode the max number of workers since Spanner has a very low
+    # QPS for administrative RPCs of 5QPS (averaged every 100 seconds)
+    if [[ $KOKORO_JOB_NAME == *"continuous"* ]]
+    then
+        # Disable continuous build as it creates too many Spanner instances
+        # ("Quota exceeded for quota metric 'Instance create requests' and
+        # limit 'Instance create requests per minute' of service
+        # 'spanner.googleapis.com').
+        export DJANGO_WORKER_COUNT=0
+    else
+        export DJANGO_WORKER_COUNT=5
+    fi
+
+    python3 ./run_testing_worker.py
+fi
diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile
@@ -0,0 +1,98 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ubuntu:20.04
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Ensure local Python is preferred over distribution Python.
+ENV PATH /usr/local/bin:$PATH
+
+# Install dependencies.
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends \
+    apt-transport-https \
+    build-essential \
+    ca-certificates \
+    curl \
+    dirmngr \
+    git \
+    gpg-agent \
+    graphviz \
+    libbz2-dev \
+    libdb5.3-dev \
+    libexpat1-dev \
+    libffi-dev \
+    liblzma-dev \
+    libreadline-dev \
+    libsnappy-dev \
+    libssl-dev \
+    libsqlite3-dev \
+    portaudio19-dev \
+    redis-server \
+    software-properties-common \
+    ssh \
+    sudo \
+    tcl \
+    tcl-dev \
+    tk \
+    tk-dev \
+    uuid-dev \
+    wget \
+    zlib1g-dev \
+  && add-apt-repository universe \
+  && apt-get update \
+  && apt-get -y install jq \
+  && apt-get clean autoclean \
+  && apt-get autoremove -y \
+  && rm -rf /var/lib/apt/lists/* \
+  && rm -f /var/cache/apt/archives/*.deb
+
+
+COPY fetch_gpg_keys.sh /tmp
+# Install the desired versions of Python.
+RUN set -ex \
+    && export GNUPGHOME="$(mktemp -d)" \
+    && echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \
+    && /tmp/fetch_gpg_keys.sh \
+    && for PYTHON_VERSION in 3.7.8 3.8.5; do \
+        wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
+        && wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
+        && gpg --batch --verify python-${PYTHON_VERSION}.tar.xz.asc python-${PYTHON_VERSION}.tar.xz \
+        && rm -r python-${PYTHON_VERSION}.tar.xz.asc \
+        && mkdir -p /usr/src/python-${PYTHON_VERSION} \
+        && tar -xJC /usr/src/python-${PYTHON_VERSION} --strip-components=1 -f python-${PYTHON_VERSION}.tar.xz \
+        && rm python-${PYTHON_VERSION}.tar.xz \
+        && cd /usr/src/python-${PYTHON_VERSION} \
+        && ./configure \
+            --enable-shared \
+            # This works only on Python 2.7 and throws a warning on every other
+            # version, but seems otherwise harmless.
+            --enable-unicode=ucs4 \
+            --with-system-ffi \
+            --without-ensurepip \
+        && make -j$(nproc) \
+        && make install \
+        && ldconfig \
+  ; done \
+  && rm -rf "${GNUPGHOME}" \
+  && rm -rf /usr/src/python* \
+  && rm -rf ~/.cache/
+
+RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \
+  && python3.7 /tmp/get-pip.py \
+  && python3.8 /tmp/get-pip.py \
+  && rm /tmp/get-pip.py
+
+CMD ["python3.7"]
diff --git a/.kokoro/docker/docs/fetch_gpg_keys.sh b/.kokoro/docker/docs/fetch_gpg_keys.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A script to fetch gpg keys with retry.
+# Avoid jinja parsing the file.
+# 
+
+function retry {
+    if [[ "${#}" -le 1 ]]; then
+	echo "Usage: ${0} retry_count commands.."
+	exit 1
+    fi
+    local retries=${1}
+    local command="${@:2}"
+    until [[ "${retries}" -le 0 ]]; do
+	$command && return 0
+	if [[ $? -ne 0 ]]; then
+	    echo "command failed, retrying"
+	    ((retries--))
+	fi
+    done
+    return 1
+}
+
+# 3.6.9, 3.7.5 (Ned Deily)
+retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
+      0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D
+
+# 3.8.0 (Łukasz Langa)
+retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
+      E3FF2839C048B25C084DEBE9B26995E310250568
+
+# 
diff --git a/.kokoro/docs/common.cfg b/.kokoro/docs/common.cfg
@@ -0,0 +1,65 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Build logs will be here
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+  }
+}
+
+# Download trampoline resources.
+gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
+
+# Use the trampoline script to run in docker.
+build_file: "python-spanner-django/.kokoro/trampoline_v2.sh"
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+    key: "TRAMPOLINE_IMAGE"
+    value: "gcr.io/cloud-devrel-kokoro-resources/python-lib-docs"
+}
+env_vars: {
+    key: "TRAMPOLINE_BUILD_FILE"
+    value: "github/python-spanner-django/.kokoro/publish-docs.sh"
+}
+
+env_vars: {
+    key: "STAGING_BUCKET"
+    value: "docs-staging"
+}
+
+env_vars: {
+    key: "V2_STAGING_BUCKET"
+    value: "docs-staging-v2"
+}
+
+# It will upload the docker image after successful builds.
+env_vars: {
+    key: "TRAMPOLINE_IMAGE_UPLOAD"
+    value: "true"
+}
+
+# It will always build the docker image.
+env_vars: {
+    key: "TRAMPOLINE_DOCKERFILE"
+    value: ".kokoro/docker/docs/Dockerfile"
+}
+
+# Fetch the token needed for reporting release status to GitHub
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73713
+      keyname: "yoshi-automation-github-key"
+    }
+  }
+}
+
+before_action {
+  fetch_keystore {
+    keystore_resource {
+      keystore_config_id: 73713
+      keyname: "docuploader_service_account"
+    }
+  }
+}
diff --git a/.kokoro/docs/docs-presubmit.cfg b/.kokoro/docs/docs-presubmit.cfg
@@ -0,0 +1,28 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+env_vars: {
+    key: "STAGING_BUCKET"
+    value: "gcloud-python-test"
+}
+
+env_vars: {
+    key: "V2_STAGING_BUCKET"
+    value: "gcloud-python-test"
+}
+
+# We only upload the image in the main `docs` build.
+env_vars: {
+    key: "TRAMPOLINE_IMAGE_UPLOAD"
+    value: "false"
+}
+
+env_vars: {
+    key: "TRAMPOLINE_BUILD_FILE"
+    value: "github/python-spanner-django/.kokoro/build.sh"
+}
+
+# Only run this nox session.
+env_vars: {
+    key: "NOX_SESSION"
+    value: "docs docfx"
+}
diff --git a/.kokoro/docs/docs.cfg b/.kokoro/docs/docs.cfg
@@ -0,0 +1 @@
+# Format: //devtools/kokoro/config/proto/build.proto
diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+
+# Disable buffering, so that the logs stream through.
+export PYTHONUNBUFFERED=1
+
+export PATH="${HOME}/.local/bin:${PATH}"
+
+# Install nox
+python3 -m pip install --user --upgrade --quiet nox
+python3 -m nox --version
+
+# build docs
+nox -s docs
+
+python3 -m pip install --user gcp-docuploader
+
+# create metadata
+python3 -m docuploader create-metadata \
+  --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
+  --version=$(python3 setup.py --version) \
+  --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
+  --distribution-name=$(python3 setup.py --name) \
+  --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
+  --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
+  --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
+
+cat docs.metadata
+
+# upload docs
+python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}"
+
+
+# docfx yaml files
+nox -s docfx
+
+# create metadata.
+python3 -m docuploader create-metadata \
+  --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
+  --version=$(python3 setup.py --version) \
+  --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
+  --distribution-name=$(python3 setup.py --name) \
+  --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
+  --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
+  --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
+
+cat docs.metadata
+
+# upload docs
+python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}"