Updates CI config to remove Xenial references

A few scripts have been updated, mostly it's the default for BASE_OS and similar vars that now defaults to Focal. The big change is snipping out all the special cases for Focal within the CircleCI config, and using it by default everywhere.
freedomofpress · Apr 28, 2021 · 0105b3b · 0105b3b
1 parent 80b6d10
commit 0105b3b
Show file tree

Hide file tree

Showing 7 changed files with 26 additions and 192 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -18,7 +18,7 @@ common-steps:
 
   - &restorecache
     restore_cache:
-      key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }}
+      key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/focal/python3/Dockerfile" }}
       paths:
         - /caches/layers.tar.gz
 
@@ -35,63 +35,21 @@ common-steps:
       command: |
         set +o pipefail
         docker images
-        fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
-        cd securedrop && DOCKER_BUILD_VERBOSE=true DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" ./bin/dev-shell true
+        fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
+        cd securedrop && DOCKER_BUILD_VERBOSE=true DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" ./bin/dev-shell true
 
   - &saveimagelayers
     run:
       name: Save Docker image layer cache
       command: |
         docker images
-        docker save -o /caches/layers.tar securedrop-test-xenial-py3:latest
+        docker save -o /caches/layers.tar securedrop-test-focal-py3:latest
 
   - &savecache
-    save_cache:
-      key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }}
-      paths:
-        - /caches/layers.tar
-
-  - &focalcreatecachedir
-    run:
-      name: Ensure cache dir exists and permissions are good
-      command: |
-        sudo mkdir -p /focalcaches && sudo chown circleci: -R /focalcaches
-
-  - &focalrestorecache
-    restore_cache:
-      key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/focal/python3/Dockerfile" }}
-      paths:
-        - /focalcaches/layers.tar.gz
-
-  - &focalloadimagelayers
-    run:
-      name: Load image layer cache on Focal
-      command: |
-        set +o pipefail
-        docker load -i /focalcaches/layers.tar || true
-
-  - &focaldockerimagebuild
-    run:
-      name: Build Docker images for Focal
-      command: |
-        set +o pipefail
-        docker images
-        fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
-        cd securedrop && DOCKER_BUILD_VERBOSE=true DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" BASE_OS=focal ./bin/dev-shell true
-
-  - &focalsaveimagelayers
-    run:
-      name: Save Docker image layer cache on Focal
-      command: |
-        docker images
-        docker save -o /focalcaches/layers.tar securedrop-test-focal-py3:latest
-
-  - &focalsavecache
     save_cache:
       key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/focal/python3/Dockerfile" }}
       paths:
-        - /focalcaches/layers.tar
-
+        - /caches/layers.tar
 
 version: 2
 jobs:
@@ -123,47 +81,13 @@ jobs:
           name: Run shellcheck
           command: make shellcheck
 
-  focal-app-tests:
-    machine:
-      image: ubuntu-2004:202010-01
-      enabled: true
-    environment:
-      DOCKER_API_VERSION: 1.23
-      BASE_OS: focal
-    parallelism: 3
-    steps:
-      - checkout
-      - *rebaseontarget
-      - *focalcreatecachedir
-      - *focalrestorecache
-      - *focalloadimagelayers
-      - *focaldockerimagebuild
-      - *focalsaveimagelayers
-      - *focalsavecache
-
-      - run:
-          name: Run tests on Focal
-          command: |
-            BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|update-builder)")
-            if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
-            export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo)
-            fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
-            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" make test-focal
-          no_output_timeout: 15m
-
-      - store_test_results:
-          path: ~/project/test-results-focal
-
-      - store_artifacts:
-          path: ~/project/test-results-focal
-
   app-tests:
     machine:
       image: ubuntu-2004:202010-01
       enabled: true
     environment:
       DOCKER_API_VERSION: 1.23
-      BASE_OS: xenial
+      BASE_OS: focal
     parallelism: 3
     steps:
       - checkout
@@ -181,8 +105,8 @@ jobs:
             BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|update-builder)")
             if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
             export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo)
-            fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
-            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" make test
+            fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
+            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" make test
           no_output_timeout: 15m
 
       - store_test_results:
@@ -197,7 +121,7 @@ jobs:
       enabled: true
     environment:
       DOCKER_API_VERSION: 1.23
-      BASE_OS: xenial
+      BASE_OS: focal
     parallelism: 20
     steps:
       - checkout
@@ -216,8 +140,8 @@ jobs:
             if ! [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
             sudo apt update && sudo apt install python3-sh python3-babel
             export LOCALES="$(/usr/bin/python3 securedrop/i18n_tool.py list-locales --lines | circleci tests split | tr '\n' ' ')"
-            fromtag=$(docker images | grep securedrop-test-xenial-py3 | head -n1 | awk '{print $2}')
-            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" make translation-test
+            fromtag=$(docker images | grep securedrop-test-focal-py3 | head -n1 | awk '{print $2}')
+            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" make translation-test
 
       - store_test_results:
           path: ~/project/test-results
@@ -284,7 +208,7 @@ jobs:
       enabled: true
     environment:
       DOCKER_API_VERSION: 1.23
-      BASE_OS: xenial
+      BASE_OS: focal
     steps:
       - checkout
       - *rebaseontarget
@@ -298,14 +222,14 @@ jobs:
       - run:
           name: Check Python dependencies for CVEs
           command: |
-            fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
-            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade safety && make -C .. safety"
+            fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
+            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade safety && make -C .. safety"
 
       - run:
           name: Run static security testing on source code
           command: |
-            fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
-            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade pip && pip3 install -U -q --upgrade bandit && make -C .. bandit"
+            fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
+            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade pip && pip3 install -U -q --upgrade bandit && make -C .. bandit"
 
 
   staging-test-with-rebase:
@@ -340,38 +264,6 @@ jobs:
       - store_artifacts:
           path: ~/sd/junit
 
-  staging-test-with-rebase-focal:
-    machine:
-      image: ubuntu-2004:202010-01
-      enabled: true
-
-    working_directory: ~/sd
-    steps:
-      - checkout
-      - *rebaseontarget
-      - *installenchant
-
-      - run:
-          name: Run Staging tests on GCE
-          command: |
-            BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n)")
-            if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
-            BASE_OS=focal make ci-go
-          no_output_timeout: 35m
-
-      - run:
-          name: Ensure environment torn down
-          # Always report true, since env should will destroyed already
-          # if all tests passed.
-          command: make ci-teardown || true
-          when: always
-
-      - store_test_results:
-          path: ~/sd/junit
-
-      - store_artifacts:
-          path: ~/sd/junit
-
   deb-tests:
     docker:
       - image: cimg/python:3.7
@@ -389,21 +281,6 @@ jobs:
             if ! [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
             make ci-deb-tests
 
-  deb-tests-focal:
-    docker:
-      - image: cimg/python:3.7
-    environment:
-      LC_ALL: C.UTF-8
-      LANG: C.UTF-8
-    steps:
-      - run: sudo apt-get update && sudo apt-get install -y make virtualenv enchant jq python3-dev build-essential rsync
-      - checkout
-      - setup_remote_docker
-      - run:
-          name: Test Debian package build on Focal
-          command: |
-            make ci-deb-tests-focal
-
 workflows:
   version: 2
   securedrop_ci:
@@ -417,14 +294,6 @@ workflows:
                 - /update-builder-.*/
           requires:
             - lint
-      - focal-app-tests:
-          filters:
-            branches:
-              ignore:
-                - /i18n-.*/
-                - /update-builder-.*/
-          requires:
-            - lint
       - admin-tests:
           filters:
             branches:
@@ -451,13 +320,6 @@ workflows:
                 - /i18n-.*/
           requires:
             - lint
-      - staging-test-with-rebase-focal:
-          filters:
-            branches:
-              ignore:
-                - /i18n-.*/
-          requires:
-            - lint
       - translation-tests:
           requires:
             - lint
@@ -468,13 +330,6 @@ workflows:
                 - /update-builder-.*/
           requires:
             - lint
-      - deb-tests-focal:
-          filters:
-            branches:
-              ignore:
-                - /i18n-.*/
-          requires:
-            - lint
 
   nightly:
     triggers:
@@ -497,6 +352,5 @@ workflows:
                 - develop
     jobs:
       - deb-tests
-      - deb-tests-focal
       - translation-tests
       - fetch-tor-debs
diff --git a/Makefile b/Makefile
@@ -219,18 +219,14 @@ test:  ## Run the test suite in a Docker container.
 	@echo
 
 .PHONY: test-focal
-test-focal:  ## Run the test suite in a Docker container.
-	@echo "███ Running SecureDrop application tests..."
-	@BASE_OS='focal' $(DEVSHELL) $(SDBIN)/run-test -v $${TESTFILES:-tests}
-	@echo
+test-focal:  test
 
 .PHONY: docker-vnc
 docker-vnc:  ## Open a VNC connection to a running Docker instance.
 	@echo "███ Opening VNC connection to dev container..."
 	@$(SDROOT)/devops/scripts/vnc-docker-connect.sh
 	@echo
 
-# Xenial upgrade targets
 .PHONY: upgrade-start
 upgrade-start:  ## Boot an upgrade test environment using libvirt.
 	@echo "███ Starting upgrade test environment..."
@@ -344,14 +340,6 @@ ci-deb-tests:  ## Test SecureDrop Debian packages in CI environment.
 	@$(SDROOT)/devops/scripts/test-built-packages.sh
 	@echo
 
-.PHONY: ci-deb-tests-focal
-ci-deb-tests-focal:  ## Test SecureDrop Debian packages in CI environment.
-	@echo "███ Running Debian package tests in CI..."
-	@$(SDROOT)/devops/scripts/test-built-packages.sh focal
-	@echo
-
-
-
 .PHONY: build-gcloud-docker
 build-gcloud-docker:  ## Build Docker container for Google Cloud SDK.
 	@echo "Building Docker container for Google Cloud SDK..."
@@ -368,7 +356,7 @@ vagrant-package:  ## Package a Vagrant box of the last stable SecureDrop release
 	@echo
 
 .PHONY: fetch-tor-packages
-fetch-tor-packages:  ## Retrieves the most recent Tor packages for Xenial, for apt repo.
+fetch-tor-packages:  ## Retrieves the most recent Tor packages, for apt repo.
 	@echo "Fetching most recent Tor packages..."
 	@$(SDROOT)/devops/scripts/fetch-tor-packages.sh
 	@echo

diff --git a/devops/gce-nested/ci-go.sh b/devops/gce-nested/ci-go.sh
@@ -12,7 +12,7 @@ set -e
 set -u
 set -o pipefail
 
-export BASE_OS="${BASE_OS:-xenial}"
+export BASE_OS="${BASE_OS:-focal}"
 
 ./devops/gce-nested/gce-start.sh
 ./devops/gce-nested/gce-runner.sh

diff --git a/devops/gce-nested/gce-runner.sh b/devops/gce-nested/gce-runner.sh
@@ -4,7 +4,7 @@
 # for storage as artifacts on the build, so devs can review via web.
 set -e
 set -u
-BASE_OS="${BASE_OS:-xenial}"
+BASE_OS="${BASE_OS:-focal}"
 
 
 TOPLEVEL="$(git rev-parse --show-toplevel)"
@@ -56,11 +56,6 @@ copy_securedrop_repo
 # The test results should be collected regardless of pass/fail,
 # so register a trap to ensure the fetch always runs.
 trap fetch_junit_test_results EXIT
-if [ "${BASE_OS:-'xenial'}" = "xenial" ]
-then
-	ssh_gce "make build-debs-notest"
-	ssh_gce "make staging"
-else
-	ssh_gce "make build-debs-notest-focal"
-	ssh_gce "make staging-focal"
-fi
+
+ssh_gce "make build-debs-notest"
+ssh_gce "make staging"
diff --git a/devops/scripts/select-staging-env b/devops/scripts/select-staging-env
@@ -13,14 +13,13 @@ set -o pipefail
 
 
 # Support overrides for LTS version
-securedrop_platform_suffix="-${1:-xenial}"
+securedrop_platform_suffix="-${1:-focal}"
 
 # Respect explicit choice of Vagrant provider if set.
 if [[ -n "${VAGRANT_DEFAULT_PROVIDER:-}" ]] ; then
     securedrop_vm_provider="${VAGRANT_DEFAULT_PROVIDER}"
 # Check whether it appears we're running in Qubes, in which case the standard Vagrant
-# logic will not work at all, due to lack of nested virt support. The Qubes staging
-# environment uses Xenial template VMs only, so we also suppress the platform suffix.
+# logic will not work at all, due to lack of nested virt support.
 elif printenv | grep -q ^QUBES_ ; then
     securedrop_vm_provider="qubes"
 elif [[ "${OSTYPE:-}" == "linux-gnu" ]]; then

diff --git a/devops/scripts/test-built-packages.sh b/devops/scripts/test-built-packages.sh
@@ -5,7 +5,7 @@
 
 set -e
 set -o pipefail
-TARGET_PLATFORM="${1:-xenial}"
+TARGET_PLATFORM="${1:-focal}"
 . ./devops/scripts/boot-strap-venv.sh
 
 virtualenv_bootstrap