[US_ND] Add PROL
to supervision delegate (Recidiviz/recidiviz-data#…
#13907
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: CI Tests | |
on: push | |
jobs: | |
cancel-previous-runs: | |
name: Cancel previous runs | |
runs-on: ubuntu-20.04 | |
steps: | |
- name: Cancel previous runs | |
if: ${{ github.ref != 'refs/heads/main' }} | |
uses: styfle/cancel-workflow-action@0.6.0 | |
with: | |
access_token: ${{ github.token }} | |
black: | |
name: Black | |
runs-on: ubuntu-20.04 | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Python Setup | |
uses: ./.github/actions/python-setup | |
- name: Black | |
run: pipenv run black . --check | |
check-if-pipenv-locked: | |
name: Check if Pipenv is locked and synced | |
runs-on: ubuntu-20.04 | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Python Setup | |
uses: ./.github/actions/python-setup | |
- name: Check pipenv is locked | |
run: pipenv run python -m recidiviz.tools.is_pipenv_locked | |
- name: Check pipenv is synced | |
run: pipenv run recidiviz/tools/diff_pipenv.sh | |
mypy: | |
name: Mypy | |
runs-on: ubuntu-20.04 | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Python Setup | |
uses: ./.github/actions/python-setup | |
- name: Mypy check | |
run: pipenv run mypy recidiviz | |
# Verify that all YAML files are properly formatted, according to prettier | |
prettier: | |
name: Prettier [yaml] | |
runs-on: ubuntu-20.04 | |
steps: | |
- uses: actions/checkout@v2 | |
- uses: actions/setup-node@v2 | |
- run: npm install prettier | |
- run: npx prettier --check "**/*.{yaml,yml}" | |
pylint: | |
name: Pylint | |
runs-on: ubuntu-20.04 | |
steps: | |
- uses: actions/checkout@v2 | |
with: | |
# This clones the repo and checks out the SHA that triggered this action. | |
# We set fetch-depth 0 to fetch all branches and history so that merge-base | |
# is guaranteed to be able to find the common ancestor with the base branch. | |
# See https://github.com/actions/checkout#readme | |
fetch-depth: 0 | |
- name: Python Setup | |
uses: ./.github/actions/python-setup | |
- name: Install Pylint | |
run: pipenv run pip install pylint | |
- name: Pylint | |
run: pipenv run ./recidiviz/tools/lint/run_pylint.sh | |
terraform-validation: | |
name: Terraform Validation | |
runs-on: ubuntu-20.04 | |
steps: | |
- uses: actions/checkout@v2 | |
- uses: hashicorp/setup-terraform@v1 | |
with: | |
# Note: this verison number should be kept in sync with the ones in Dockerfile, | |
# .devcontainer/devcontainer.json, recidiviz/tools/deploy/terraform/terraform.tf, and | |
# recidiviz/tools/deploy/deploy_helpers.sh | |
terraform_version: ~1.7.0 | |
- name: Validate terraform | |
run: | | |
terraform -chdir=recidiviz/tools/deploy/terraform init -backend=false | |
terraform -chdir=recidiviz/tools/deploy/terraform validate | |
# TODO(#6431): pre-commit was flaky in CI, so we turned it off. It should be | |
# re-enabled soon. | |
# - name: pre-commit | |
# # If pre-commit proves useful, we can remove redundant checks (e.g. black). | |
# uses: pre-commit/action@v2.0.0 | |
# # This can also create a new commit with changes, but it merges in latest | |
# # main and can't re-trigger CI, so we don't do so at this time. | |
# with: | |
# # Pass extra_args to only run on changed files. The default is '--all-files'. | |
# # TODO(#6234): Remove once PA files can be generated again. | |
# extra_args: --from-ref ${{ env.merge_base }} --to-ref HEAD | |
validate-source-visibility: | |
name: Validate source visibility | |
runs-on: ubuntu-20.04 | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Python Setup | |
uses: ./.github/actions/python-setup | |
- name: Validate source visibility | |
run: pipenv run python -m recidiviz.tools.validate_source_visibility | |
frontend-tests: | |
name: Frontend tests | |
runs-on: ubuntu-20.04 | |
steps: | |
- uses: actions/checkout@v2 | |
with: | |
# This clones the repo and checks out the SHA that triggered this action. | |
# We set fetch-depth 0 to fetch all branches and history so that merge-base | |
# is guaranteed to be able to find the common ancestor with the base branch. | |
# See https://github.com/actions/checkout#readme | |
fetch-depth: 0 | |
- name: Yarn caching | |
uses: actions/cache@v2 | |
id: yarn-caching | |
with: | |
path: "**/node_modules" | |
key: yarn-v2-${{ hashFiles('**/yarn.lock') }} | |
restore-keys: | | |
yarn-v2-${{ hashFiles('**/yarn.lock') }} | |
yarn-v2- | |
yarn- | |
- name: Yarn install (admin panel) | |
if: steps.yarn-caching.outputs.cache-hit != 'true' | |
working-directory: ./frontends/admin-panel/ | |
run: yarn install --frozen-lockfile | |
- name: Yarn lint (admin panel) | |
working-directory: ./frontends/admin-panel/ | |
run: yarn lint --max-warnings=0 | |
- name: Yarn test (admin panel) | |
working-directory: ./frontends/admin-panel/ | |
run: yarn test | |
asset-generation-tests: | |
name: Asset Generation Service tests | |
runs-on: ubuntu-20.04 | |
defaults: | |
run: | |
working-directory: ./nodejs/asset-generation/ | |
steps: | |
- uses: actions/checkout@v2 | |
- uses: actions/setup-node@v3 | |
with: | |
node-version: "18" | |
cache: "yarn" | |
cache-dependency-path: nodejs/asset-generation/yarn.lock | |
- name: Install fonts | |
run: | | |
cp src/fonts/* /usr/share/fonts/ | |
fc-cache -fv | |
- name: Yarn install | |
run: yarn install --frozen-lockfile | |
- name: Lint | |
run: yarn lint --max-warnings=0 | |
- name: Test | |
run: yarn test | |
unit-tests: | |
name: Base unit tests | |
runs-on: ubuntu-20.04 | |
strategy: | |
fail-fast: false | |
matrix: | |
suite: [1, 2, 3] | |
suite-count: [3] | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Python Setup | |
uses: ./.github/actions/python-setup | |
- name: Run unit tests in parallel | |
run: | | |
pipenv run pytest recidiviz/tests \ | |
-n logical \ | |
-m 'not uses_db and not isolated and not uses_bq_emulator' \ | |
--suite-count ${{matrix.suite-count}} \ | |
--suite=${{ matrix.suite }} \ | |
--durations=25 | |
# These unit tests need to be run separately because they depend on a different Python | |
# environment defined in the Pipfile in the recidiviz/airflow package. | |
airflow-tests: | |
name: Airflow unit tests | |
runs-on: ubuntu-20.04 | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Airflow Python Setup | |
uses: ./.github/actions/airflow-python-setup | |
- name: Cache postgres | |
uses: actions/cache@v2 | |
id: cache-postgres | |
with: | |
path: "~/postgres" | |
key: ${{ runner.os }}-postgres-13 | |
- name: Install postgres | |
env: | |
CACHE_HIT: ${{steps.cache-postgres.outputs.cache-hit}} | |
shell: bash | |
run: | | |
if [[ "$CACHE_HIT" == 'true' ]]; then | |
sudo cp --force --recursive ~/postgres/* / | |
else | |
sudo apt-get update | |
sudo apt-get -y upgrade | |
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - | |
echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/postgresql-pgdg.list > /dev/null | |
sudo apt-get update | |
sudo apt-get install postgresql-13 | |
mkdir -p ~/postgres | |
for dep in postgresql-13; do | |
dpkg -L $dep | while IFS= read -r f; do if test -f $f; then echo $f; fi; done | xargs cp --parents --target-directory ~/postgres/ | |
done | |
fi | |
echo "/usr/lib/postgresql/13/bin" >> $GITHUB_PATH | |
sudo chown -R $USER /var/run/postgresql | |
- name: Run airflow tests in parallel | |
working-directory: ./recidiviz/airflow | |
run: | | |
pipenv run pytest tests | |
db-tests: | |
name: Database tests | |
runs-on: ubuntu-20.04 | |
strategy: | |
fail-fast: false | |
matrix: | |
suite: [1, 2, 3] | |
suite-count: [3] | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Python Setup | |
uses: ./.github/actions/python-setup | |
# Postgres caching strategy adapted from https://raymii.org/s/articles/Github_Actions_cpp_boost_cmake_speedup.html | |
- name: Cache postgres | |
uses: actions/cache@v2 | |
id: cache-postgres | |
with: | |
path: "~/postgres" | |
key: ${{ runner.os }}-postgres-13 | |
- name: Install postgres | |
env: | |
CACHE_HIT: ${{steps.cache-postgres.outputs.cache-hit}} | |
shell: bash | |
run: | | |
if [[ "$CACHE_HIT" == 'true' ]]; then | |
sudo cp --force --recursive ~/postgres/* / | |
else | |
sudo apt-get update | |
sudo apt-get -y upgrade | |
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - | |
echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/postgresql-pgdg.list > /dev/null | |
sudo apt-get update | |
sudo apt-get install postgresql-13 | |
mkdir -p ~/postgres | |
for dep in postgresql-13; do | |
dpkg -L $dep | while IFS= read -r f; do if test -f $f; then echo $f; fi; done | xargs cp --parents --target-directory ~/postgres/ | |
done | |
fi | |
echo "/usr/lib/postgresql/13/bin" >> $GITHUB_PATH | |
sudo chown -R $USER /var/run/postgresql | |
- name: Run db tests | |
run: | | |
pipenv run pytest recidiviz/tests \ | |
-n logical \ | |
-m 'uses_db' \ | |
--suite-count ${{matrix.suite-count}} \ | |
--suite=${{ matrix.suite }} \ | |
--durations=25 | |
big-query-emulator-tests: | |
name: BigQuery Emulator Tests | |
runs-on: ubuntu-20.04 | |
strategy: | |
fail-fast: false | |
matrix: | |
suite: [1, 2, 3] | |
suite-count: [3] | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Python Setup | |
uses: ./.github/actions/python-setup | |
# TODO(#20786): This step can be removed when moving back to the upstream bigquery emulator. | |
- name: Docker auth with ghcr | |
run: | | |
echo ${{ secrets.github_token }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin | |
- name: Pull emulator Docker image | |
run: pipenv run pull-bq-emulator | |
# TODO(#20786): Replace ghcr.io/recidiviz with ghcr.io/goccy to move back to the | |
# upstream bigquery emulator. | |
# Start two emulator processes for each of the running pytest processes. | |
# We must run one emulator per process specified in `pytest -n` below. | |
- name: Start emulator (1) | |
run: | | |
docker run -d -p 9050:9050 \ | |
--name=recidiviz-bq-emulator-1 \ | |
--cpus 1 \ | |
--cpuset-cpus 0 \ | |
ghcr.io/recidiviz/bigquery-emulator:0.4.4-recidiviz.8 \ | |
/bin/bigquery-emulator \ | |
--port=9050 \ | |
--project=recidiviz-bq-emulator-project \ | |
--log-level=info | |
- name: Start emulator (2) | |
run: | | |
docker run -d -p 9051:9051 \ | |
--name=recidiviz-bq-emulator-2 \ | |
--cpus 1 \ | |
--cpuset-cpus 1 \ | |
ghcr.io/recidiviz/bigquery-emulator:0.4.4-recidiviz.8 \ | |
/bin/bigquery-emulator \ | |
--port=9051 \ | |
--project=recidiviz-bq-emulator-project \ | |
--log-level=info | |
- name: Run unittests | |
run: | | |
pipenv run pytest recidiviz/tests \ | |
-m 'uses_bq_emulator' \ | |
-n 2 \ | |
--suite-count ${{matrix.suite-count}} \ | |
--suite=${{ matrix.suite }} \ | |
--durations=25 | |
- name: Show logs (1) | |
if: ${{ failure() || cancelled() }} | |
run: docker logs recidiviz-bq-emulator-1 | |
- name: Show logs (2) | |
if: ${{ failure() || cancelled() }} | |
run: docker logs recidiviz-bq-emulator-2 | |
- name: Stop the container | |
if: ${{ always() }} | |
run: docker stop recidiviz-bq-emulator-1 && docker rm recidiviz-bq-emulator-1 && docker stop recidiviz-bq-emulator-2 && docker rm recidiviz-bq-emulator-2 |