Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Removes Xenial related parts from codebase #5911

Merged
merged 12 commits into from May 3, 2021
Merged

Removes Xenial related parts from codebase #5911

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
f8b9d40
Removes Xenial from Vagrantfile
kushaldas Apr 21, 2021
8955bd1
Make staging now runs Focal by default
kushaldas Apr 21, 2021
2063451
Updates make dev to Focal
kushaldas Apr 21, 2021
798cf9c
Make build-debs now has Focal as default
kushaldas Apr 21, 2021
0bb420b
Removes xenial scenarios from molecule
kushaldas Apr 21, 2021
5a48683
Update the Focal Dockerfile
rmol Apr 21, 2021
b3c8a43
Removes all EOL related code and tests
kushaldas Apr 22, 2021
b8ef5d6
Removes Xenial-specific Ansible logic
Apr 28, 2021
bf8cefb
Removes Xenial-specific packaging logic
Apr 28, 2021
80b6d10
Removes Xenial-specific logic from config tests
Apr 28, 2021
0105b3b
Updates CI config to remove Xenial references
Apr 28, 2021
7fa6f65
Removes EOL related strings from journalist interface
kushaldas Apr 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
178 changes: 16 additions & 162 deletions .circleci/config.yml
View file
Open in desktop
Expand Up @@ -18,7 +18,7 @@ common-steps:

- &restorecache
restore_cache:
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }}
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/focal/python3/Dockerfile" }}
paths:
- /caches/layers.tar.gz

Expand All @@ -35,63 +35,21 @@ common-steps:
command: |
set +o pipefail
docker images
fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
cd securedrop && DOCKER_BUILD_VERBOSE=true DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" ./bin/dev-shell true
fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
cd securedrop && DOCKER_BUILD_VERBOSE=true DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" ./bin/dev-shell true

- &saveimagelayers
run:
name: Save Docker image layer cache
command: |
docker images
docker save -o /caches/layers.tar securedrop-test-xenial-py3:latest
docker save -o /caches/layers.tar securedrop-test-focal-py3:latest

- &savecache
save_cache:
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/xenial/python3/Dockerfile" }}
paths:
- /caches/layers.tar

- &focalcreatecachedir
run:
name: Ensure cache dir exists and permissions are good
command: |
sudo mkdir -p /focalcaches && sudo chown circleci: -R /focalcaches

- &focalrestorecache
restore_cache:
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/focal/python3/Dockerfile" }}
paths:
- /focalcaches/layers.tar.gz

- &focalloadimagelayers
run:
name: Load image layer cache on Focal
command: |
set +o pipefail
docker load -i /focalcaches/layers.tar || true

- &focaldockerimagebuild
run:
name: Build Docker images for Focal
command: |
set +o pipefail
docker images
fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
cd securedrop && DOCKER_BUILD_VERBOSE=true DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" BASE_OS=focal ./bin/dev-shell true

- &focalsaveimagelayers
run:
name: Save Docker image layer cache on Focal
command: |
docker images
docker save -o /focalcaches/layers.tar securedrop-test-focal-py3:latest

- &focalsavecache
save_cache:
key: v1-sd-layers-{{ checksum "securedrop/dockerfiles/focal/python3/Dockerfile" }}
paths:
- /focalcaches/layers.tar

- /caches/layers.tar

version: 2
jobs:
Expand Down Expand Up @@ -123,47 +81,13 @@ jobs:
name: Run shellcheck
command: make shellcheck

focal-app-tests:
machine:
image: ubuntu-2004:202010-01
enabled: true
environment:
DOCKER_API_VERSION: 1.23
BASE_OS: focal
parallelism: 3
steps:
- checkout
- *rebaseontarget
- *focalcreatecachedir
- *focalrestorecache
- *focalloadimagelayers
- *focaldockerimagebuild
- *focalsaveimagelayers
- *focalsavecache

- run:
name: Run tests on Focal
command: |
BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|update-builder)")
if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo)
fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" make test-focal
no_output_timeout: 15m

- store_test_results:
path: ~/project/test-results-focal

- store_artifacts:
path: ~/project/test-results-focal

app-tests:
machine:
image: ubuntu-2004:202010-01
enabled: true
environment:
DOCKER_API_VERSION: 1.23
BASE_OS: xenial
BASE_OS: focal
parallelism: 3
steps:
- checkout
Expand All @@ -181,8 +105,8 @@ jobs:
BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n|update-builder)")
if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
export TESTFILES=$(cd securedrop; circleci tests glob 'tests/test*py' 'tests/**/test*py' |circleci tests split --split-by=timings |xargs echo)
fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" make test
fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" make test
no_output_timeout: 15m

- store_test_results:
Expand All @@ -197,7 +121,7 @@ jobs:
enabled: true
environment:
DOCKER_API_VERSION: 1.23
BASE_OS: xenial
BASE_OS: focal
parallelism: 20
steps:
- checkout
Expand All @@ -216,8 +140,8 @@ jobs:
if ! [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
sudo apt update && sudo apt install python3-sh python3-babel
export LOCALES="$(/usr/bin/python3 securedrop/i18n_tool.py list-locales --lines | circleci tests split | tr '\n' ' ')"
fromtag=$(docker images | grep securedrop-test-xenial-py3 | head -n1 | awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" make translation-test
fromtag=$(docker images | grep securedrop-test-focal-py3 | head -n1 | awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" make translation-test

- store_test_results:
path: ~/project/test-results
Expand Down Expand Up @@ -284,7 +208,7 @@ jobs:
enabled: true
environment:
DOCKER_API_VERSION: 1.23
BASE_OS: xenial
BASE_OS: focal
steps:
- checkout
- *rebaseontarget
Expand All @@ -298,14 +222,14 @@ jobs:
- run:
name: Check Python dependencies for CVEs
command: |
fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade safety && make -C .. safety"
fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade safety && make -C .. safety"

- run:
name: Run static security testing on source code
command: |
fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade pip && pip3 install -U -q --upgrade bandit && make -C .. bandit"
fromtag=$(docker images |grep securedrop-test-focal-py3 |head -n1 |awk '{print $2}')
DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-focal-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "pip3 install -U -q --upgrade pip && pip3 install -U -q --upgrade bandit && make -C .. bandit"


staging-test-with-rebase:
Expand Down Expand Up @@ -340,38 +264,6 @@ jobs:
- store_artifacts:
path: ~/sd/junit

staging-test-with-rebase-focal:
machine:
image: ubuntu-2004:202010-01
enabled: true

working_directory: ~/sd
steps:
- checkout
- *rebaseontarget
- *installenchant

- run:
name: Run Staging tests on GCE
command: |
BRANCH_MATCH=$(devops/scripts/match-ci-branch.sh "^(i18n)")
if [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
BASE_OS=focal make ci-go
no_output_timeout: 35m

- run:
name: Ensure environment torn down
# Always report true, since env should will destroyed already
# if all tests passed.
command: make ci-teardown || true
when: always

- store_test_results:
path: ~/sd/junit

- store_artifacts:
path: ~/sd/junit

deb-tests:
docker:
- image: cimg/python:3.7
Expand All @@ -389,21 +281,6 @@ jobs:
if ! [[ $BRANCH_MATCH =~ ^found ]]; then echo "Skipping: ${BRANCH_MATCH}"; exit 0; fi
make ci-deb-tests

deb-tests-focal:
docker:
- image: cimg/python:3.7
environment:
LC_ALL: C.UTF-8
LANG: C.UTF-8
steps:
- run: sudo apt-get update && sudo apt-get install -y make virtualenv enchant jq python3-dev build-essential rsync
- checkout
- setup_remote_docker
- run:
name: Test Debian package build on Focal
command: |
make ci-deb-tests-focal

workflows:
version: 2
securedrop_ci:
Expand All @@ -417,14 +294,6 @@ workflows:
- /update-builder-.*/
requires:
- lint
- focal-app-tests:
filters:
branches:
ignore:
- /i18n-.*/
- /update-builder-.*/
requires:
- lint
- admin-tests:
filters:
branches:
Expand All @@ -451,13 +320,6 @@ workflows:
- /i18n-.*/
requires:
- lint
- staging-test-with-rebase-focal:
filters:
branches:
ignore:
- /i18n-.*/
requires:
- lint
- translation-tests:
requires:
- lint
Expand All @@ -468,13 +330,6 @@ workflows:
- /update-builder-.*/
requires:
- lint
- deb-tests-focal:
filters:
branches:
ignore:
- /i18n-.*/
requires:
- lint

nightly:
triggers:
Expand All @@ -497,6 +352,5 @@ workflows:
- develop
jobs:
- deb-tests
- deb-tests-focal
- translation-tests
- fetch-tor-debs
50 changes: 7 additions & 43 deletions Makefile
View file
Open in desktop
Expand Up @@ -181,21 +181,9 @@ dev: ## Run the development server in a Docker container.
@OFFSET_PORTS='false' DOCKER_BUILD_VERBOSE='true' $(DEVSHELL) $(SDBIN)/run
@echo

.PHONY: dev-focal
dev-focal: ## Run the development server in a Docker container.
@echo "███ Starting development server..."
@OFFSET_PORTS='false' DOCKER_BUILD_VERBOSE='true' BASE_OS='focal' $(DEVSHELL) $(SDBIN)/run
@echo


.PHONY: staging
staging: ## Create a local staging environment in virtual machines (Xenial)
@echo "███ Creating staging environment on Ubuntu Xenial..."
@$(SDROOT)/devops/scripts/create-staging-env xenial
@echo

.PHONY: staging-focal
staging-focal: ## Create a local staging environment in virtual machines (Focal)
staging: ## Create a local staging environment in virtual machines (Focal)
@echo "███ Creating staging environment on Ubuntu Focal..."
@$(SDROOT)/devops/scripts/create-staging-env focal
@echo
Expand Down Expand Up @@ -231,18 +219,14 @@ test: ## Run the test suite in a Docker container.
@echo

.PHONY: test-focal
test-focal: ## Run the test suite in a Docker container.
@echo "███ Running SecureDrop application tests..."
@BASE_OS='focal' $(DEVSHELL) $(SDBIN)/run-test -v $${TESTFILES:-tests}
@echo
test-focal: test

.PHONY: docker-vnc
docker-vnc: ## Open a VNC connection to a running Docker instance.
@echo "███ Opening VNC connection to dev container..."
@$(SDROOT)/devops/scripts/vnc-docker-connect.sh
@echo

# Xenial upgrade targets
.PHONY: upgrade-start
upgrade-start: ## Boot an upgrade test environment using libvirt.
@echo "███ Starting upgrade test environment..."
Expand Down Expand Up @@ -320,27 +304,15 @@ endif
###########

.PHONY: build-debs
build-debs: ## Build and test SecureDrop Debian packages (for Xenial)
@echo "Building SecureDrop Debian packages for Xenial..."
build-debs: ## Build and test SecureDrop Debian packages (for Focal)
@echo "Building SecureDrop Debian packages for Focal..."
@$(SDROOT)/devops/scripts/build-debs.sh
@echo

.PHONY: build-debs-notest
build-debs-notest: ## Build SecureDrop Debian packages (for Xenial) without running tests.
@echo "Building SecureDrop Debian packages for Xenial; skipping tests..."
@$(SDROOT)/devops/scripts/build-debs.sh notest
@echo

.PHONY: build-debs-focal
build-debs-focal: ## Build and test SecureDrop Debian packages (for Focal)
@echo "Building SecureDrop Debian packages for Focal..."
@$(SDROOT)/devops/scripts/build-debs.sh test focal
@echo

.PHONY: build-debs-notest-focal
build-debs-notest-focal: ## Build SecureDrop Debian packages (for Focal) without running tests.
build-debs-notest: ## Build SecureDrop Debian packages (for Focal) without running tests.
@echo "Building SecureDrop Debian packages for Focal; skipping tests..."
@$(SDROOT)/devops/scripts/build-debs.sh notest focal
@$(SDROOT)/devops/scripts/build-debs.sh notest
@echo


Expand Down Expand Up @@ -368,14 +340,6 @@ ci-deb-tests: ## Test SecureDrop Debian packages in CI environment.
@$(SDROOT)/devops/scripts/test-built-packages.sh
@echo

.PHONY: ci-deb-tests-focal
ci-deb-tests-focal: ## Test SecureDrop Debian packages in CI environment.
@echo "███ Running Debian package tests in CI..."
@$(SDROOT)/devops/scripts/test-built-packages.sh focal
@echo



.PHONY: build-gcloud-docker
build-gcloud-docker: ## Build Docker container for Google Cloud SDK.
@echo "Building Docker container for Google Cloud SDK..."
Expand All @@ -392,7 +356,7 @@ vagrant-package: ## Package a Vagrant box of the last stable SecureDrop release
@echo

.PHONY: fetch-tor-packages
fetch-tor-packages: ## Retrieves the most recent Tor packages for Xenial, for apt repo.
fetch-tor-packages: ## Retrieves the most recent Tor packages, for apt repo.
@echo "Fetching most recent Tor packages..."
@$(SDROOT)/devops/scripts/fetch-tor-packages.sh
@echo
Expand Down