Continuous Integration #3289
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # -*- coding: utf-8 -*- | |
| # | |
| # This file is part of HEPData. | |
| # Copyright (C) 2020 CERN. | |
| # | |
| # HEPData is free software; you can redistribute it | |
| # and/or modify it under the terms of the GNU General Public License as | |
| # published by the Free Software Foundation; either version 2 of the | |
| # License, or (at your option) any later version. | |
| # | |
| # HEPData is distributed in the hope that it will be | |
| # useful, but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
| # General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU General Public License | |
| # along with HEPData; if not, write to the | |
| # Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, | |
| # MA 02111-1307, USA. | |
| # | |
| # In applying this license, CERN does not | |
| # waive the privileges and immunities granted to it by virtue of its status | |
| # as an Intergovernmental Organization or submit itself to any jurisdiction. | |
| name: Continuous Integration | |
| on: | |
| push: | |
| pull_request: | |
| branches: [ main ] | |
| release: | |
| types: [ published ] | |
| schedule: | |
| - cron: '35 1 * * *' | |
| concurrency: | |
| group: ${{github.event_name}}-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| permissions: write-all | |
| env: | |
| node-version: '18' | |
| strategy: | |
| matrix: | |
| postgres-version: [ 14 ] | |
| os-version: [ '2.11.0' ] | |
| python-version: [ '3.8', '3.9' ] | |
| exclude: | |
| - python-version: ${{ github.event.act && '3.8' }} | |
| # Service containers to run with `runner-job` | |
| services: | |
| postgres: | |
| image: postgres:${{ matrix.postgres-version }} | |
| # Provide the password for postgres | |
| env: | |
| POSTGRES_USER: hepdata | |
| POSTGRES_PASSWORD: hepdata | |
| POSTGRES_DB: hepdata_test | |
| # Set health checks to wait until postgres has started | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| redis: | |
| image: redis | |
| ports: | |
| - "6379:6379" | |
| opensearch: | |
| image: opensearchproject/opensearch:${{ matrix.os-version }} | |
| env: | |
| node.name: os01 | |
| cluster.name: hepdata | |
| cluster.initial_master_nodes: os01 | |
| OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m | |
| plugins.security.disabled: true | |
| ports: | |
| - "9200:9200" | |
| - "9600:9600" | |
| steps: | |
| # Services are not currently supported by act (https://github.com/nektos/act#services) | |
| - name: Set up services for act | |
| if: ${{ env.ACT }} | |
| run: | | |
| docker run --name act-postgres -e POSTGRES_USER=hepdata -e POSTGRES_PASSWORD=hepdata -e POSTGRES_DB=hepdata_test -p 5432:5432 --rm -d postgres:${{ matrix.postgres-version }} | |
| docker run --name act-redis --rm -d -p 6379:6379 redis | |
| docker run --name act-opensearch --rm -d -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -e "plugins.security.disabled=true" opensearchproject/opensearch:${{ matrix.os-version }} | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.node-version }} | |
| - name: Cache pip modules | |
| uses: actions/cache@v4 | |
| if: ${{ !env.ACT }} | |
| env: | |
| cache-name: cache-pip-modules | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-test-${{ env.cache-name }}- | |
| - name: Cache node modules | |
| uses: actions/cache@v4 | |
| if: ${{ !env.ACT }} | |
| env: | |
| cache-name: cache-node-modules | |
| with: | |
| path: ~/.npm | |
| key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/bundles.py') }} | |
| restore-keys: | | |
| ${{ runner.os }}-test-${{ env.cache-name }}- | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip setuptools py | |
| python -m pip install twine wheel coveralls requirements-builder | |
| python -m pip install --force-reinstall -r requirements.txt | |
| python -m pip install -e .[tests] | |
| - name: Initialise hepdata | |
| run: | | |
| hepdata db init | |
| hepdata db create | |
| hepdata utils reindex -rc True | |
| - name: Setup npm and build assets | |
| run: | | |
| hepdata collect -v | |
| hepdata webpack buildall | |
| cp hepdata/config_local.gh.py hepdata/config_local.py | |
| - name: Run tests | |
| env: | |
| COVERAGE_FILE: '.coverage_func' | |
| run: | | |
| py.test -vv tests/*_test.py | |
| - name: Setup Sauce Connect | |
| uses: saucelabs/sauce-connect-action@v2 | |
| if: startsWith(matrix.python-version, '3.9') | |
| with: | |
| username: ${{ secrets.SAUCE_USERNAME }} | |
| accessKey: ${{ secrets.SAUCE_ACCESS_KEY }} | |
| region: eu-central | |
| tunnelName: ${{ github.run_id }} | |
| scVersion: 4.9.1 | |
| verbose: true | |
| - name: Run end-to-end tests | |
| if: startsWith(matrix.python-version, '3.9') | |
| env: | |
| SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} | |
| SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} | |
| COVERAGE_FILE: '.coverage_e2e' | |
| run: | | |
| if [[ -n ${{ secrets.SAUCE_USERNAME }} && -n ${{ secrets.SAUCE_ACCESS_KEY}} ]]; then py.test -vv tests/e2e; fi | |
| - name: Run coveralls | |
| if: ${{ startsWith(matrix.python-version, '3.9') && !env.ACT }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| COVERALLS_SERVICE_NAME: github | |
| run: | | |
| coverage combine .coverage_func .coverage_e2e | |
| coveralls | |
| - name: Build docs | |
| run: | | |
| python -m pip install -e .[docs] | |
| cd docs && make html | |
| - uses: actions/upload-artifact@v4 | |
| if: ${{ failure() && !env.ACT }} | |
| with: | |
| name: sauce-connect-log | |
| path: ${{ env.SAUCE_CONNECT_DIR_IN_HOST }}/sauce-connect.log | |
| - name: Stop service containers for act | |
| if: ${{ always() && env.ACT }} | |
| run: | | |
| docker stop act-postgres act-redis act-opensearch | |
| deploy: | |
| needs: test | |
| if: contains(github.ref, 'main') || github.event_name == 'release' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Deploy to DockerHub | |
| env: | |
| CI_TAG: ${{ github.event.release.tag_name }} | |
| DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| run: | | |
| ./build-and-deploy.sh |