Add separate parsing endpoint #11296
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| push: | |
| pull_request: | |
| jobs: | |
| changes: | |
| name: Check if back-end or front-end files changed | |
| # Don't run this for pushes generated by push.yml on weblate-localization | |
| # branch → prevents an infinite loop of actions triggering actions | |
| # | |
| # Also, "push" event is not triggered for forks, so need to listen for | |
| # pull_requests, but only for external ones, so as not to run the action | |
| # twice | |
| if: | | |
| github.event.head_commit.committer.name != 'Hosted Weblate' && | |
| github.event.head_commit.committer.name != 'github-actions' && | |
| ( | |
| github.event_name != 'pull_request' || | |
| github.event.pull_request.head.repo.fork == true | |
| ) | |
| runs-on: ubuntu-latest | |
| outputs: | |
| backend_changed: ${{ steps.back-end-check.outputs.changed }} | |
| frontend: ${{ steps.filter.outputs.frontend }} | |
| frontend_changes: ${{ steps.filter.outputs.frontend_files }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Find all changed files | |
| uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| base: ${{ github.ref }} | |
| list-files: escape | |
| filters: | | |
| frontend: | |
| - 'specifyweb/frontend/**' | |
| backend: | |
| - '**' | |
| - name: Check if any non-front-end files changed | |
| id: back-end-check | |
| run: | | |
| changed_files=`echo "${{steps.filter.outputs.backend_files}}" | tr " " "\n" | grep -v 'specifyweb/frontend/' || echo ""` | |
| echo "Changed back-end files: ${changed_files}" | |
| echo "changed=$([[ -z "${changed_files}" ]] && echo "" || echo "1")" >> $GITHUB_OUTPUT | |
| test-back-end: | |
| name: Run back-end tests | |
| needs: changes | |
| if: needs.changes.outputs.backend_changed | |
| runs-on: ubuntu-latest | |
| services: | |
| mariadb: | |
| image: mariadb:latest | |
| ports: | |
| - 3306 | |
| env: | |
| MYSQL_USER: MasterUser | |
| MYSQL_PASSWORD: MasterPassword | |
| MYSQL_DATABASE: test_SpecifyDB | |
| MYSQL_ROOT_PASSWORD: password | |
| options: | |
| --health-cmd="mariadb-admin ping" --health-interval=5s | |
| --health-timeout=2s --health-retries=3 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get Specify 6 from cache | |
| id: cache-specify6 | |
| uses: actions/cache@v3 | |
| with: | |
| path: Specify6 | |
| key: specify6.8.02-cache | |
| - name: Install Specify 6 | |
| if: steps.cache-specify6.outputs.cache-hit != 'true' | |
| run: | | |
| wget https://update.specifysoftware.org/6802/Specify_unix_64.sh | |
| sh Specify_unix_64.sh -q -dir ./Specify6 | |
| - name: | |
| Patch Specify datamodel (Sam, you made the Attachment.origFilename too | |
| long) | |
| run: | |
| sed -i 's/name="origFilename" | |
| type="java.lang.String"/name="origFilename" type="text"/' | |
| ./Specify6/config/specify_datamodel.xml | |
| - name: Install ubuntu dependencies | |
| run: | | |
| sudo apt update | |
| sudo apt install -y libmysqlclient-dev libsasl2-dev libldap2-dev libssl-dev | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8 | |
| - name: Install python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m venv ve | |
| ve/bin/pip install -r requirements.txt | |
| - name: Install testing python dependencies | |
| run: ve/bin/pip install -r requirements-testing.txt | |
| - name: Create settings file | |
| run: | | |
| echo "THICK_CLIENT_LOCATION = '${{ github.workspace }}/Specify6'" >> specifyweb/settings/local_specify_settings.py | |
| echo "DATABASE_HOST = '127.0.0.1'" >> specifyweb/settings/local_specify_settings.py | |
| echo "DATABASE_PORT = ${{ job.services.mariadb.ports[3306] }}" >> specifyweb/settings/local_specify_settings.py | |
| - name: Need these files to be present | |
| run: | |
| make specifyweb/settings/build_version.py | |
| specifyweb/settings/secret_key.py | |
| - name: Verify MariaDB connection | |
| env: | |
| PORT: ${{ job.services.mariadb.ports[3306] }} | |
| run: | | |
| while ! mysqladmin ping -h"127.0.0.1" -P"$PORT" --silent; do | |
| sleep 1 | |
| done | |
| - name: Run Mypy type checker | |
| run: VIRTUAL_ENV=./ve make typecheck | |
| # - name: Delete existing test database (if any). Use if database is corrupted | |
| # env: | |
| # PORT: ${{ job.services.mariadb.ports[3306] }} | |
| # run: | | |
| # mysql -h 127.0.0.1 -P $PORT -u MasterUser -p'MasterPassword' -e 'DROP DATABASE IF EXISTS test_SpecifyDB;'; | |
| # mysql -h 127.0.0.1 -P $PORT -u MasterUser -p'MasterPassword' -e 'SHOW DATABASES;' | |
| - name: Run test suite | |
| run: ./ve/bin/python manage.py test --verbosity=3 --keepdb | |
| # run: ./ve/bin/python manage.py test --verbosity=3 --noinput | |
| test-front-end: | |
| name: Run front-end tests | |
| needs: changes | |
| if: ${{ needs.changes.outputs.frontend == 'true' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # Using a Personal Access Token from an admin account to bypass branch | |
| # protection rules and allow direct pushes to production. Inspirited | |
| # by: https://github.com/community/community/discussions/13836 | |
| token: ${{ secrets.TESTS_PUSH_TO_GITHUB }} | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: specifyweb/frontend/js_src/package.json | |
| cache: 'npm' | |
| cache-dependency-path: specifyweb/frontend/js_src/package-lock.json | |
| - name: Build frontend | |
| run: make frontend | |
| # See https://jestjs.io/docs/troubleshooting#tests-are-extremely-slow-on-docker-andor-continuous-integration-ci-server | |
| - name: Get number of CPU cores | |
| id: cpu-cores | |
| uses: SimenB/github-actions-cpu-cores@v1 | |
| - name: Run JS test suite | |
| working-directory: specifyweb/frontend/js_src | |
| run: | | |
| npm run typecheck && \ | |
| npm run unitTests -- --maxWorkers ${{ steps.cpu-cores.outputs.count }} | |
| - name: Clone branch that stores localization strings | |
| # Listen for changes to production branch | |
| if: | |
| github.ref == format('refs/heads/{0}', | |
| github.event.repository.default_branch) | |
| id: weblate | |
| uses: actions/checkout@v3 | |
| with: | |
| path: weblate-localization | |
| ref: weblate-localization | |
| fetch-depth: 0 | |
| - name: Localization Tests | |
| working-directory: specifyweb/frontend/js_src | |
| run: | | |
| npm run localizationTests -- --emit ../../../weblate-localization/strings | |
| - name: Sync localization strings with Weblate (only on production branch) | |
| if: | |
| github.ref == format('refs/heads/{0}', | |
| github.event.repository.default_branch) | |
| working-directory: weblate-localization | |
| run: | | |
| git add strings | |
| if git diff-index --quiet HEAD --; then | |
| echo "Localization strings are unchanged" | |
| else | |
| git config --local user.name "github-actions" | |
| git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| # Set the committer to the name and email of the person who | |
| # triggered the action | |
| git commit \ | |
| --author '${{ github.event.head_commit.author.name }} <${{ github.event.head_commit.author.email }}>' \ | |
| -m 'Sync localization strings with Weblate' \ | |
| -m "Triggered by ${{ github.sha }} on branch ${{ github.ref }}" | |
| git push | |
| fi | |
| # This step must be run after pushing to github or else it will fail in | |
| # cases when new components are to be created (Weblate requires files to | |
| # be pushed before component can be created) | |
| - name: Make sure Weblate settings are up to date | |
| # Only run this step if previous step was not skipped | |
| if: steps.weblate.outcome == 'success' | |
| working-directory: specifyweb/frontend/js_src | |
| env: | |
| WEBLATE_API_TOKEN: ${{ secrets.WEBLATE_API_TOKEN }} | |
| run: npm run validateWeblate | |
| - name: Cleanup localization outfiles | |
| working-directory: specifyweb/frontend/js_src | |
| run: rm -rf ./localization-strings | |
| - name: Get relative path of all changed files | |
| working-directory: specifyweb/frontend/js_src | |
| id: changed | |
| run: | | |
| # Strip specifyweb/frontend/js_src/ from every file name | |
| changed_files=`sed 's/specifyweb\/frontend\/js_src\///g' <<< "${{ needs.changes.outputs.frontend_changes }}"` | |
| # Convert to array | |
| IFS=' ' read -r -a split_files <<< "$changed_files" | |
| # Exclude files that were removed | |
| for file in "${split_files[@]}"; do | |
| if [ ! -e "$file" ]; then | |
| echo "File was removed: $file" | |
| split_files=("${split_files[@]/$file}") | |
| fi | |
| done | |
| # Convert back to string | |
| changed="${split_files[*]}" | |
| echo "Changed front-end files: ${changed}" | |
| echo "changed=${changed}" >> $GITHUB_OUTPUT | |
| - name: Run ESLint auto fixes | |
| # Don't run this for production branch. Reasons: | |
| # - ESLint auto fixes may break code. Don't want it to break production | |
| # code | |
| # - This would have already run on the feature branch by the time code | |
| # is pushed into production. | |
| # - There shouldn't be many direct pushes to production anyway | |
| if: | |
| github.ref != format('refs/heads/{0}', | |
| github.event.repository.default_branch) | |
| working-directory: specifyweb/frontend/js_src | |
| run: | | |
| # TODO: Once most errors are fixed remove "set +e" to not ignore errors | |
| set +e | |
| npx eslint --fix --color `echo "${{steps.changed.outputs.changed}}" | tr " " "\n"` | |
| set -e | |
| - name: Reformat all code with Prettier | |
| if: steps.changed.outputs.changed | |
| working-directory: specifyweb/frontend/js_src | |
| run: | | |
| npx prettier --write --minify `echo "${{steps.changed.outputs.changed}}" | tr " " "\n"` || true | |
| - name: Commit linted files (if made any changes) | |
| working-directory: specifyweb/frontend/js_src | |
| # Creates a new commit. Amending an existing commit is a bad idea | |
| # because: | |
| # - Would require original committer to force pull. May cause merge | |
| # conflicts | |
| # - Changes that require linting might have been made over several | |
| # commits. If you amend the last commit, than the fixes for all | |
| # commits would be in the last commit. | |
| run: | | |
| git add . | |
| if git diff-index --quiet HEAD --; then | |
| echo "Linters did not detect any issues. Good job!" | |
| else | |
| git config --local user.name "github-actions" | |
| git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| # Set the committer to the name and email of the person who | |
| # triggered the action | |
| git commit \ | |
| --author '${{ github.event.head_commit.author.name }} <${{ github.event.head_commit.author.email }}>' \ | |
| -m 'Lint code with ESLint and Prettier' \ | |
| -m "Triggered by ${{ github.sha }} on branch ${{ github.ref }}" | |
| git push | |
| fi |