[US_IX][Workflows] Add criteria to exclude clients serving their firs… #13902

Workflow file for this run

	name: CI Tests

	on: push

	jobs:
	cancel-previous-runs:
	name: Cancel previous runs
	runs-on: ubuntu-20.04
	steps:
	- name: Cancel previous runs
	if: ${{ github.ref != 'refs/heads/main' }}
	uses: styfle/cancel-workflow-action@0.6.0
	with:
	access_token: ${{ github.token }}

	black:
	name: Black
	runs-on: ubuntu-20.04
	steps:
	- uses: actions/checkout@v2
	- name: Python Setup
	uses: ./.github/actions/python-setup
	- name: Black
	run: pipenv run black . --check

	check-if-pipenv-locked:
	name: Check if Pipenv is locked and synced
	runs-on: ubuntu-20.04
	steps:
	- uses: actions/checkout@v2
	- name: Python Setup
	uses: ./.github/actions/python-setup
	- name: Check pipenv is locked
	run: pipenv run python -m recidiviz.tools.is_pipenv_locked
	- name: Check pipenv is synced
	run: pipenv run recidiviz/tools/diff_pipenv.sh

	mypy:
	name: Mypy
	runs-on: ubuntu-20.04
	steps:
	- uses: actions/checkout@v2
	- name: Python Setup
	uses: ./.github/actions/python-setup
	- name: Mypy check
	run: pipenv run mypy recidiviz

	# Verify that all YAML files are properly formatted, according to prettier
	prettier:
	name: Prettier [yaml]
	runs-on: ubuntu-20.04
	steps:
	- uses: actions/checkout@v2
	- uses: actions/setup-node@v2
	- run: npm install prettier
	- run: npx prettier --check "*/.{yaml,yml}"

	pylint:
	name: Pylint
	runs-on: ubuntu-20.04
	steps:
	- uses: actions/checkout@v2
	with:
	# This clones the repo and checks out the SHA that triggered this action.
	# We set fetch-depth 0 to fetch all branches and history so that merge-base
	# is guaranteed to be able to find the common ancestor with the base branch.
	# See https://github.com/actions/checkout#readme
	fetch-depth: 0
	- name: Python Setup
	uses: ./.github/actions/python-setup
	- name: Install Pylint
	run: pipenv run pip install pylint
	- name: Pylint
	run: pipenv run ./recidiviz/tools/lint/run_pylint.sh

	terraform-validation:
	name: Terraform Validation
	runs-on: ubuntu-20.04
	steps:
	- uses: actions/checkout@v2
	- uses: hashicorp/setup-terraform@v1
	with:
	# Note: this verison number should be kept in sync with the ones in Dockerfile,
	# .devcontainer/devcontainer.json, recidiviz/tools/deploy/terraform/terraform.tf, and
	# recidiviz/tools/deploy/deploy_helpers.sh
	terraform_version: ~1.7.0
	- name: Validate terraform
	run: \|
	terraform -chdir=recidiviz/tools/deploy/terraform init -backend=false
	terraform -chdir=recidiviz/tools/deploy/terraform validate

	# TODO(#6431): pre-commit was flaky in CI, so we turned it off. It should be
	# re-enabled soon.
	# - name: pre-commit
	# # If pre-commit proves useful, we can remove redundant checks (e.g. black).
	# uses: pre-commit/action@v2.0.0
	# # This can also create a new commit with changes, but it merges in latest
	# # main and can't re-trigger CI, so we don't do so at this time.
	# with:
	# # Pass extra_args to only run on changed files. The default is '--all-files'.
	# # TODO(#6234): Remove once PA files can be generated again.
	# extra_args: --from-ref ${{ env.merge_base }} --to-ref HEAD

	validate-source-visibility:
	name: Validate source visibility
	runs-on: ubuntu-20.04
	steps:
	- uses: actions/checkout@v2
	- name: Python Setup
	uses: ./.github/actions/python-setup
	- name: Validate source visibility
	run: pipenv run python -m recidiviz.tools.validate_source_visibility

	frontend-tests:
	name: Frontend tests
	runs-on: ubuntu-20.04
	steps:
	- uses: actions/checkout@v2
	with:
	# This clones the repo and checks out the SHA that triggered this action.
	# We set fetch-depth 0 to fetch all branches and history so that merge-base
	# is guaranteed to be able to find the common ancestor with the base branch.
	# See https://github.com/actions/checkout#readme
	fetch-depth: 0
	- name: Yarn caching
	uses: actions/cache@v2
	id: yarn-caching
	with:
	path: "**/node_modules"
	key: yarn-v2-${{ hashFiles('**/yarn.lock') }}
	restore-keys: \|
	yarn-v2-${{ hashFiles('**/yarn.lock') }}
	yarn-v2-
	yarn-
	- name: Yarn install (admin panel)
	if: steps.yarn-caching.outputs.cache-hit != 'true'
	working-directory: ./frontends/admin-panel/
	run: yarn install --frozen-lockfile
	- name: Yarn lint (admin panel)
	working-directory: ./frontends/admin-panel/
	run: yarn lint --max-warnings=0
	- name: Yarn test (admin panel)
	working-directory: ./frontends/admin-panel/
	run: yarn test

	asset-generation-tests:
	name: Asset Generation Service tests
	runs-on: ubuntu-20.04
	defaults:
	run:
	working-directory: ./nodejs/asset-generation/
	steps:
	- uses: actions/checkout@v2
	- uses: actions/setup-node@v3
	with:
	node-version: "18"
	cache: "yarn"
	cache-dependency-path: nodejs/asset-generation/yarn.lock
	- name: Install fonts
	run: \|
	cp src/fonts/* /usr/share/fonts/
	fc-cache -fv
	- name: Yarn install
	run: yarn install --frozen-lockfile
	- name: Lint
	run: yarn lint --max-warnings=0
	- name: Test
	run: yarn test

	unit-tests:
	name: Base unit tests
	runs-on: ubuntu-20.04
	strategy:
	fail-fast: false
	matrix:
	suite: [1, 2, 3]
	suite-count: [3]
	steps:
	- uses: actions/checkout@v2
	- name: Python Setup
	uses: ./.github/actions/python-setup
	- name: Run unit tests in parallel
	run: \|
	pipenv run pytest recidiviz/tests \
	-n logical \
	-m 'not uses_db and not isolated and not uses_bq_emulator' \
	--suite-count ${{matrix.suite-count}} \
	--suite=${{ matrix.suite }} \
	--durations=25

	# These unit tests need to be run separately because they depend on a different Python
	# environment defined in the Pipfile in the recidiviz/airflow package.
	airflow-tests:
	name: Airflow unit tests
	runs-on: ubuntu-20.04
	steps:
	- uses: actions/checkout@v2
	- name: Airflow Python Setup
	uses: ./.github/actions/airflow-python-setup
	- name: Cache postgres
	uses: actions/cache@v2
	id: cache-postgres
	with:
	path: "~/postgres"
	key: ${{ runner.os }}-postgres-13
	- name: Install postgres
	env:
	CACHE_HIT: ${{steps.cache-postgres.outputs.cache-hit}}
	shell: bash
	run: \|
	if [[ "$CACHE_HIT" == 'true' ]]; then
	sudo cp --force --recursive ~/postgres/* /
	else
	sudo apt-get update
	sudo apt-get -y upgrade
	wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc \| sudo apt-key add -
	echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" \| sudo tee /etc/apt/sources.list.d/postgresql-pgdg.list > /dev/null
	sudo apt-get update
	sudo apt-get install postgresql-13
	mkdir -p ~/postgres
	for dep in postgresql-13; do
	dpkg -L $dep \| while IFS= read -r f; do if test -f $f; then echo $f; fi; done \| xargs cp --parents --target-directory ~/postgres/
	done
	fi
	echo "/usr/lib/postgresql/13/bin" >> $GITHUB_PATH
	sudo chown -R $USER /var/run/postgresql
	- name: Run airflow tests in parallel
	working-directory: ./recidiviz/airflow
	run: \|
	pipenv run pytest tests

	db-tests:
	name: Database tests
	runs-on: ubuntu-20.04
	strategy:
	fail-fast: false
	matrix:
	suite: [1, 2, 3]
	suite-count: [3]
	steps:
	- uses: actions/checkout@v2
	- name: Python Setup
	uses: ./.github/actions/python-setup
	# Postgres caching strategy adapted from https://raymii.org/s/articles/Github_Actions_cpp_boost_cmake_speedup.html
	- name: Cache postgres
	uses: actions/cache@v2
	id: cache-postgres
	with:
	path: "~/postgres"
	key: ${{ runner.os }}-postgres-13
	- name: Install postgres
	env:
	CACHE_HIT: ${{steps.cache-postgres.outputs.cache-hit}}
	shell: bash
	run: \|
	if [[ "$CACHE_HIT" == 'true' ]]; then
	sudo cp --force --recursive ~/postgres/* /
	else
	sudo apt-get update
	sudo apt-get -y upgrade
	wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc \| sudo apt-key add -
	echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" \| sudo tee /etc/apt/sources.list.d/postgresql-pgdg.list > /dev/null
	sudo apt-get update
	sudo apt-get install postgresql-13
	mkdir -p ~/postgres
	for dep in postgresql-13; do
	dpkg -L $dep \| while IFS= read -r f; do if test -f $f; then echo $f; fi; done \| xargs cp --parents --target-directory ~/postgres/
	done
	fi
	echo "/usr/lib/postgresql/13/bin" >> $GITHUB_PATH
	sudo chown -R $USER /var/run/postgresql

	- name: Run db tests
	run: \|
	pipenv run pytest recidiviz/tests \
	-n logical \
	-m 'uses_db' \
	--suite-count ${{matrix.suite-count}} \
	--suite=${{ matrix.suite }} \
	--durations=25

	big-query-emulator-tests:
	name: BigQuery Emulator Tests
	runs-on: ubuntu-20.04
	strategy:
	fail-fast: false
	matrix:
	suite: [1, 2, 3]
	suite-count: [3]
	steps:
	- uses: actions/checkout@v2
	- name: Python Setup
	uses: ./.github/actions/python-setup
	# TODO(#20786): This step can be removed when moving back to the upstream bigquery emulator.
	- name: Docker auth with ghcr
	run: \|
	echo ${{ secrets.github_token }} \| docker login ghcr.io -u ${{ github.actor }} --password-stdin
	- name: Pull emulator Docker image
	run: pipenv run pull-bq-emulator
	# TODO(#20786): Replace ghcr.io/recidiviz with ghcr.io/goccy to move back to the
	# upstream bigquery emulator.
	# Start two emulator processes for each of the running pytest processes.
	# We must run one emulator per process specified in `pytest -n` below.
	- name: Start emulator (1)
	run: \|
	docker run -d -p 9050:9050 \
	--name=recidiviz-bq-emulator-1 \
	--cpus 1 \
	--cpuset-cpus 0 \
	ghcr.io/recidiviz/bigquery-emulator:0.4.4-recidiviz.8 \
	/bin/bigquery-emulator \
	--port=9050 \
	--project=recidiviz-bq-emulator-project \
	--log-level=info
	- name: Start emulator (2)
	run: \|
	docker run -d -p 9051:9051 \
	--name=recidiviz-bq-emulator-2 \
	--cpus 1 \
	--cpuset-cpus 1 \
	ghcr.io/recidiviz/bigquery-emulator:0.4.4-recidiviz.8 \
	/bin/bigquery-emulator \
	--port=9051 \
	--project=recidiviz-bq-emulator-project \
	--log-level=info
	- name: Run unittests
	run: \|
	pipenv run pytest recidiviz/tests \
	-m 'uses_bq_emulator' \
	-n 2 \
	--suite-count ${{matrix.suite-count}} \
	--suite=${{ matrix.suite }} \
	--durations=25
	- name: Show logs (1)
	if: ${{ failure() \|\| cancelled() }}
	run: docker logs recidiviz-bq-emulator-1
	- name: Show logs (2)
	if: ${{ failure() \|\| cancelled() }}
	run: docker logs recidiviz-bq-emulator-2
	- name: Stop the container
	if: ${{ always() }}
	run: docker stop recidiviz-bq-emulator-1 && docker rm recidiviz-bq-emulator-1 && docker stop recidiviz-bq-emulator-2 && docker rm recidiviz-bq-emulator-2

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[US_IX][Workflows] Add criteria to exclude clients serving their firs… #13902

Workflow file

[US_IX][Workflows] Add criteria to exclude clients serving their firs… #13902

Jobs

Run details

Workflow file for this run