Skip to content

Commit

Permalink
Merge pull request #5911 from freedomofpress/cancel_xenial
Browse files Browse the repository at this point in the history 
Removes Xenial related parts from codebase
  • Loading branch information
@conorsch
conorsch committed May 3, 2021
2 parents e37c6c0 + 7fa6f65 commit 84b7042
Show file tree
Hide file tree
Showing 122 changed files with 476 additions and 2,729 deletions.
178 changes: 16 additions & 162 deletions .circleci/config.yml
View file
Expand Up @@ -18,7 +18,7 @@ common-steps:
- &restorecache
restore_cache:
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }}
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/focal/python3/Dockerfile" }}
paths:
- /caches/layers.tar.gz

Expand All @@ -35,63 +35,21 @@ common-steps:
command: |
set +o pipefail
docker images
fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
cd securedrop && DOCKER_BUILD_VERBOSE=true DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" ./bin/dev-shell true
fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
cd securedrop && DOCKER_BUILD_VERBOSE=true DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" ./bin/dev-shell true
- &saveimagelayers
run:
name: Save Docker image layer cache
command: |
docker images
docker save -o /caches/layers.tar securedrop-test-xenial-py3:latest
docker save -o /caches/layers.tar securedrop-test-focal-py3:latest
- &savecache
save_cache:
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }}
paths:
- /caches/layers.tar

- &focalcreatecachedir
run:
name: Ensure cache dir exists and permissions are good
command: |
sudo mkdir -p /focalcaches && sudo chown circleci: -R /focalcaches
- &focalrestorecache
restore_cache:
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/focal/python3/Dockerfile" }}
paths:
- /focalcaches/layers.tar.gz

- &focalloadimagelayers
run:
name: Load image layer cache on Focal
command: |
set +o pipefail
docker load -i /focalcaches/layers.tar || true
- &focaldockerimagebuild
run:
name: Build Docker images for Focal
command: |
set +o pipefail
docker images
fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
cd securedrop && DOCKER_BUILD_VERBOSE=true DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" BASE_OS=focal ./bin/dev-shell true
- &focalsaveimagelayers
run:
name: Save Docker image layer cache on Focal
command: |
docker images
docker save -o /focalcaches/layers.tar securedrop-test-focal-py3:latest
- &focalsavecache
save_cache:
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/focal/python3/Dockerfile" }}
paths:
- /focalcaches/layers.tar

- /caches/layers.tar

version: 2
jobs:
Expand Down Expand Up @@ -123,47 +81,13 @@ jobs:
name: Run shellcheck
command: make shellcheck

focal-app-tests:
machine:
image: ubuntu-2004:202010-01
enabled: true
environment:
DOCKER_API_VERSION: 1.23
BASE_OS: focal
parallelism: 3
steps:
- checkout
- *rebaseontarget
- *focalcreatecachedir
- *focalrestorecache
- *focalloadimagelayers
- *focaldockerimagebuild
- *focalsaveimagelayers
- *focalsavecache

- run:
name: Run tests on Focal
command: |
BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|update-builder)")
if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo)
fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" make test-focal
no_output_timeout: 15m

- store_test_results:
path: ~/project/test-results-focal

- store_artifacts:
path: ~/project/test-results-focal

app-tests:
machine:
image: ubuntu-2004:202010-01
enabled: true
environment:
DOCKER_API_VERSION: 1.23
BASE_OS: xenial
BASE_OS: focal
parallelism: 3
steps:
- checkout
Expand All @@ -181,8 +105,8 @@ jobs:
BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|update-builder)")
if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo)
fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" make test
fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" make test
no_output_timeout: 15m

- store_test_results:
Expand All @@ -197,7 +121,7 @@ jobs:
enabled: true
environment:
DOCKER_API_VERSION: 1.23
BASE_OS: xenial
BASE_OS: focal
parallelism: 20
steps:
- checkout
Expand All @@ -216,8 +140,8 @@ jobs:
if ! [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
sudo apt update && sudo apt install python3-sh python3-babel
export LOCALES="$(/usr/bin/python3 securedrop/i18n_tool.py list-locales --lines | circleci tests split | tr '\n' ' ')"
fromtag=$(docker images | grep securedrop-test-xenial-py3 | head -n1 | awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" make translation-test
fromtag=$(docker images | grep securedrop-test-focal-py3 | head -n1 | awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" make translation-test
- store_test_results:
path: ~/project/test-results
Expand Down Expand Up @@ -284,7 +208,7 @@ jobs:
enabled: true
environment:
DOCKER_API_VERSION: 1.23
BASE_OS: xenial
BASE_OS: focal
steps:
- checkout
- *rebaseontarget
Expand All @@ -298,14 +222,14 @@ jobs:
- run:
name: Check Python dependencies for CVEs
command: |
fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade safety && make -C .. safety"
fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade safety && make -C .. safety"
- run:
name: Run static security testing on source code
command: |
fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade pip && pip3 install -U -q --upgrade bandit && make -C .. bandit"
fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade pip && pip3 install -U -q --upgrade bandit && make -C .. bandit"
staging-test-with-rebase:
Expand Down Expand Up @@ -340,38 +264,6 @@ jobs:
- store_artifacts:
path: ~/sd/junit

staging-test-with-rebase-focal:
machine:
image: ubuntu-2004:202010-01
enabled: true

working_directory: ~/sd
steps:
- checkout
- *rebaseontarget
- *installenchant

- run:
name: Run Staging tests on GCE
command: |
BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n)")
if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
BASE_OS=focal make ci-go
no_output_timeout: 35m

- run:
name: Ensure environment torn down
# Always report true, since env should will destroyed already
# if all tests passed.
command: make ci-teardown || true
when: always

- store_test_results:
path: ~/sd/junit

- store_artifacts:
path: ~/sd/junit

deb-tests:
docker:
- image: cimg/python:3.7
Expand All @@ -389,21 +281,6 @@ jobs:
if ! [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
make ci-deb-tests
deb-tests-focal:
docker:
- image: cimg/python:3.7
environment:
LC_ALL: C.UTF-8
LANG: C.UTF-8
steps:
- run: sudo apt-get update && sudo apt-get install -y make virtualenv enchant jq python3-dev build-essential rsync
- checkout
- setup_remote_docker
- run:
name: Test Debian package build on Focal
command: |
make ci-deb-tests-focal
workflows:
version: 2
securedrop_ci:
Expand All @@ -417,14 +294,6 @@ workflows:
- /update-builder-.*/
requires:
- lint
- focal-app-tests:
filters:
branches:
ignore:
- /i18n-.*/
- /update-builder-.*/
requires:
- lint
- admin-tests:
filters:
branches:
Expand All @@ -451,13 +320,6 @@ workflows:
- /i18n-.*/
requires:
- lint
- staging-test-with-rebase-focal:
filters:
branches:
ignore:
- /i18n-.*/
requires:
- lint
- translation-tests:
requires:
- lint
Expand All @@ -468,13 +330,6 @@ workflows:
- /update-builder-.*/
requires:
- lint
- deb-tests-focal:
filters:
branches:
ignore:
- /i18n-.*/
requires:
- lint

nightly:
triggers:
Expand All @@ -497,6 +352,5 @@ workflows:
- develop
jobs:
- deb-tests
- deb-tests-focal
- translation-tests
- fetch-tor-debs
50 changes: 7 additions & 43 deletions Makefile
View file
Expand Up @@ -181,21 +181,9 @@ dev: ## Run the development server in a Docker container.
@OFFSET_PORTS='false' DOCKER_BUILD_VERBOSE='true' $(DEVSHELL) $(SDBIN)/run
@echo

.PHONY: dev-focal
dev-focal: ## Run the development server in a Docker container.
@echo "███ Starting development server..."
@OFFSET_PORTS='false' DOCKER_BUILD_VERBOSE='true' BASE_OS='focal' $(DEVSHELL) $(SDBIN)/run
@echo


.PHONY: staging
staging: ## Create a local staging environment in virtual machines (Xenial)
@echo "███ Creating staging environment on Ubuntu Xenial..."
@$(SDROOT)/devops/scripts/create-staging-env xenial
@echo

.PHONY: staging-focal
staging-focal: ## Create a local staging environment in virtual machines (Focal)
staging: ## Create a local staging environment in virtual machines (Focal)
@echo "███ Creating staging environment on Ubuntu Focal..."
@$(SDROOT)/devops/scripts/create-staging-env focal
@echo
Expand Down Expand Up @@ -231,18 +219,14 @@ test: ## Run the test suite in a Docker container.
@echo

.PHONY: test-focal
test-focal: ## Run the test suite in a Docker container.
@echo "███ Running SecureDrop application tests..."
@BASE_OS='focal' $(DEVSHELL) $(SDBIN)/run-test -v $${TESTFILES:-tests}
@echo
test-focal: test

.PHONY: docker-vnc
docker-vnc: ## Open a VNC connection to a running Docker instance.
@echo "███ Opening VNC connection to dev container..."
@$(SDROOT)/devops/scripts/vnc-docker-connect.sh
@echo

# Xenial upgrade targets
.PHONY: upgrade-start
upgrade-start: ## Boot an upgrade test environment using libvirt.
@echo "███ Starting upgrade test environment..."
Expand Down Expand Up @@ -320,27 +304,15 @@ endif
###########

.PHONY: build-debs
build-debs: ## Build and test SecureDrop Debian packages (for Xenial)
@echo "Building SecureDrop Debian packages for Xenial..."
build-debs: ## Build and test SecureDrop Debian packages (for Focal)
@echo "Building SecureDrop Debian packages for Focal..."
@$(SDROOT)/devops/scripts/build-debs.sh
@echo

.PHONY: build-debs-notest
build-debs-notest: ## Build SecureDrop Debian packages (for Xenial) without running tests.
@echo "Building SecureDrop Debian packages for Xenial; skipping tests..."
@$(SDROOT)/devops/scripts/build-debs.sh notest
@echo

.PHONY: build-debs-focal
build-debs-focal: ## Build and test SecureDrop Debian packages (for Focal)
@echo "Building SecureDrop Debian packages for Focal..."
@$(SDROOT)/devops/scripts/build-debs.sh test focal
@echo

.PHONY: build-debs-notest-focal
build-debs-notest-focal: ## Build SecureDrop Debian packages (for Focal) without running tests.
build-debs-notest: ## Build SecureDrop Debian packages (for Focal) without running tests.
@echo "Building SecureDrop Debian packages for Focal; skipping tests..."
@$(SDROOT)/devops/scripts/build-debs.sh notest focal
@$(SDROOT)/devops/scripts/build-debs.sh notest
@echo


Expand Down Expand Up @@ -368,14 +340,6 @@ ci-deb-tests: ## Test SecureDrop Debian packages in CI environment.
@$(SDROOT)/devops/scripts/test-built-packages.sh
@echo

.PHONY: ci-deb-tests-focal
ci-deb-tests-focal: ## Test SecureDrop Debian packages in CI environment.
@echo "███ Running Debian package tests in CI..."
@$(SDROOT)/devops/scripts/test-built-packages.sh focal
@echo



.PHONY: build-gcloud-docker
build-gcloud-docker: ## Build Docker container for Google Cloud SDK.
@echo "Building Docker container for Google Cloud SDK..."
Expand All @@ -392,7 +356,7 @@ vagrant-package: ## Package a Vagrant box of the last stable SecureDrop release
@echo

.PHONY: fetch-tor-packages
fetch-tor-packages: ## Retrieves the most recent Tor packages for Xenial, for apt repo.
fetch-tor-packages: ## Retrieves the most recent Tor packages, for apt repo.
@echo "Fetching most recent Tor packages..."
@$(SDROOT)/devops/scripts/fetch-tor-packages.sh
@echo
Expand Down

0 comments on commit 84b7042

Please sign in to comment.