QA #83
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Runs the entire testsuite | |
| name: QA | |
| on: | |
| pull_request: | |
| schedule: | |
| - cron: '0 18 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| pcp_qa_args: | |
| description: '(optional) overwrite ./check args (e.g. "-g pmproxy -g pmseries")' | |
| jobs: | |
| run_qa: | |
| name: ${{ matrix.platform }} | |
| # do not run this workflow on schedule for forks of the main repository | |
| if: github.event_name != 'schedule' || github.repository == 'performancecopilot/pcp' | |
| runs-on: ${{ matrix.os }} | |
| # do not mark the workflow as failed if an experimental distro (e.g. rawhide) fails | |
| continue-on-error: ${{ matrix.experimental }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - {platform: debian10-container, os: ubuntu-20.04, experimental: false} | |
| - {platform: debian11-container, os: ubuntu-20.04, experimental: false} | |
| - {platform: debian12-container, os: ubuntu-20.04, experimental: false} | |
| - {platform: ubuntu1804-container, os: ubuntu-20.04, experimental: false} | |
| - {platform: ubuntu1804-i386-container, os: ubuntu-20.04, experimental: false} | |
| - {platform: ubuntu2004-container, os: ubuntu-20.04, experimental: false} | |
| - {platform: ubuntu2004, os: ubuntu-20.04, experimental: false} | |
| - {platform: ubuntu2204-container, os: ubuntu-20.04, experimental: false} | |
| - {platform: ubuntu2204, os: ubuntu-22.04, experimental: false} | |
| - {platform: fedora37-container, os: ubuntu-20.04, experimental: false} | |
| - {platform: fedora38-container, os: ubuntu-20.04, experimental: false} | |
| - {platform: fedora39-container, os: ubuntu-20.04, experimental: false} | |
| - {platform: fedora-rawhide-container, os: ubuntu-20.04, experimental: true } | |
| - {platform: centos6-container, os: ubuntu-20.04, experimental: false} | |
| - {platform: centos7-container, os: ubuntu-20.04, experimental: false} | |
| - {platform: centos-stream8-container, os: ubuntu-20.04, experimental: false} | |
| - {platform: centos-stream9-container, os: ubuntu-20.04, experimental: false} | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| - name: Fix local hostname DNS lookup | |
| run: echo 127.0.1.1 $(hostname --fqdn) $(hostname) | sudo tee -a /etc/hosts | |
| - name: Update version information | |
| run: | | |
| . VERSION.pcp | |
| PACKAGE_BUILD="0.$(date +'%Y%m%d').$(git rev-parse --short HEAD)" | |
| PCP_VERSION=${PACKAGE_MAJOR}.${PACKAGE_MINOR}.${PACKAGE_REVISION} | |
| PCP_BUILD_VERSION=${PCP_VERSION}-${PACKAGE_BUILD} | |
| sed -i "s/PACKAGE_BUILD=.*/PACKAGE_BUILD=${PACKAGE_BUILD}/" VERSION.pcp | |
| sed -i "1 s/(.*)/(${PCP_BUILD_VERSION})/" debian/changelog | |
| - name: Setup | |
| run: | | |
| python3 -c 'import yaml' || pip3 install pyyaml | |
| mkdir -p artifacts/build artifacts/test | |
| touch artifacts/build/.keep | |
| # crun >= 1.9.1 is required on Ubuntu 20.04.6 | |
| # this workaround came from ... | |
| # https://github.com/kubernetes-sigs/kind/pull/3527 | |
| curl -sLo /tmp/crun https://github.com/containers/crun/releases/download/1.14.3/crun-1.14.3-linux-amd64 | |
| chmod +x /tmp/crun | |
| sudo cp /tmp/crun /usr/bin/crun | |
| build/ci/ci-run.py ${{ matrix.platform }} setup | |
| - name: Build | |
| run: build/ci/ci-run.py ${{ matrix.platform }} task build | |
| - name: Copy build artifacts | |
| run: build/ci/ci-run.py ${{ matrix.platform }} artifacts build --path artifacts/build | |
| - name: Upload build artifacts | |
| # always() is required here to run this step even if the build fails | |
| # otherwise the platform will be skipped in the test report (it should be flagged as broken) | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: build-${{ matrix.platform }} | |
| path: artifacts/build | |
| - name: Install | |
| run: build/ci/ci-run.py ${{ matrix.platform }} task install | |
| - name: Initialize QA | |
| id: init_qa | |
| run: build/ci/ci-run.py ${{ matrix.platform }} task init_qa | |
| - name: QA | |
| run: build/ci/ci-run.py ${{ matrix.platform }} task qa || true | |
| env: | |
| PCP_QA_ARGS: ${{ github.event.inputs.pcp_qa_args }} | |
| - name: Copy test results | |
| # always() is required here to run this step even if the QA step fails or times out | |
| if: always() && steps.init_qa.outcome == 'success' | |
| run: build/ci/ci-run.py ${{ matrix.platform }} artifacts test --path artifacts/test | |
| - name: Upload test results | |
| if: always() && steps.init_qa.outcome == 'success' | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: test-${{ matrix.platform }} | |
| path: artifacts/test | |
| create_report: | |
| name: Create Test Report | |
| # always() is required here to run this job even if any of the QA jobs fails | |
| if: always() && (github.event_name != 'schedule' || github.repository == 'performancecopilot/pcp') | |
| needs: run_qa | |
| runs-on: ubuntu-latest | |
| outputs: | |
| report_id: ${{ steps.parse_test_results.outputs.report_id }} | |
| summary_url: ${{ steps.publish_test_summary.outputs.summary_url }} | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| - name: Checkout performancecopilot/qa-reports | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: performancecopilot/qa-reports | |
| path: qa-reports | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: artifacts | |
| - name: Setup Allure2 | |
| run: | | |
| wget -q -O allure-commandline.zip https://repo1.maven.org/maven2/io/qameta/allure/allure-commandline/2.19.0/allure-commandline-2.19.0.zip | |
| unzip -q -d /opt allure-commandline.zip | |
| - name: Parse test results | |
| id: parse_test_results | |
| run: | | |
| REPORT_ID=$(date +'%Y%m%d_%H%M%S')-${GITHUB_SHA::8} | |
| echo "report_id=${REPORT_ID}" >> $GITHUB_OUTPUT | |
| # generate Allure inputs | |
| build/ci/test-report.py \ | |
| --artifacts artifacts \ | |
| --allure-results allure-results \ | |
| --summary summary.txt \ | |
| --source-url "https://github.com/${{ github.repository }}/tree/${GITHUB_SHA}" \ | |
| --build-name "GitHub Actions Run #${{ github.run_id }}" \ | |
| --build-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ | |
| --report-url "https://${{ github.repository_owner }}.github.io/qa-reports/reports/${REPORT_ID}/" | |
| cat summary.txt | |
| - name: Upload test summary | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: qasummary | |
| path: summary.txt | |
| - name: Publish test summary | |
| id: publish_test_summary | |
| if: github.event_name != 'pull_request' | |
| uses: actions/github-script@v6 | |
| continue-on-error: true | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const head_sha = context.payload.pull_request ? context.payload.pull_request.head.sha : context.sha; | |
| const response = await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| head_sha, | |
| name: 'Test Summary', | |
| conclusion: 'success', | |
| output: { | |
| title: 'Test Summary', | |
| summary: '```\n' + fs.readFileSync('summary.txt', 'utf8') + '\n```', | |
| } | |
| }); | |
| core.setOutput('summary_url', response.data.html_url); | |
| console.log(`::notice::Test summary is available at ${response.data.html_url}`); | |
| - name: Create interactive test report | |
| run: | | |
| cp build/ci/allure/categories.json allure-results | |
| if [[ -f qa-reports/reports/latest/history/history.json.gz ]]; then | |
| gunzip qa-reports/reports/latest/history/history.json.gz | |
| cp -r qa-reports/reports/latest/history allure-results | |
| fi | |
| /opt/allure-*/bin/allure generate | |
| - name: Create qareport.zip | |
| run: | | |
| cp build/ci/allure/README.md allure-report | |
| cp build/ci/allure/open_report.sh allure-report | |
| gzip allure-report/history/history.json | |
| # zip report before uploading, because GitHub uploads files in individual HTTP calls | |
| cd allure-report && zip -q -r ../qareport.zip . && cd .. | |
| - name: Upload test report | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: qareport | |
| path: qareport.zip | |
| - name: Check for regressions | |
| run: build/ci/check-trend.py --history-trend allure-report/history/history-trend.json | |
| continue-on-error: true | |
| publish_report: | |
| name: Publish Test Report | |
| # always() is required here to run this job even if any of the QA jobs fails | |
| if: always() && (github.repository == 'performancecopilot/pcp' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) | |
| needs: create_report | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout sources | |
| uses: actions/checkout@v4 | |
| - name: Checkout ${{ github.repository_owner }}/qa-reports | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.repository_owner }}/qa-reports | |
| path: qa-reports | |
| ssh-key: ${{ secrets.QA_REPORT_DEPLOY_KEY }} | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: artifacts | |
| - name: Prepare test report on GitHub Pages | |
| run: | | |
| rm -rf qa-reports/test-* | |
| cp -r artifacts/test-* qa-reports/ | |
| unzip -q -d qa-reports/reports/${REPORT_ID} artifacts/qareport/qareport.zip | |
| pushd qa-reports/reports | |
| rm latest | |
| rm README.md | |
| ls -1r | tail -n +16 | xargs --no-run-if-empty rm -r # keep last 15 reports | |
| ln -s ${REPORT_ID} latest | |
| ls -1r -I README.md | awk 'BEGIN {print "# Recent Reports"} { print "## [" $1 "](" $1 "/)" }' > README.md | |
| popd | |
| env: | |
| REPORT_ID: ${{ needs.create_report.outputs.report_id }} | |
| - name: Publish test report on GitHub Pages | |
| run: | | |
| pushd qa-reports | |
| git config --local user.name "GitHub Actions" | |
| git config --local user.email "action@github.com" | |
| git add . | |
| git commit --amend -m "Added QA report of ${{ github.repository }}@${GITHUB_SHA}" | |
| git push -f | |
| popd | |
| echo "::notice::Test report is available at https://${{ github.repository_owner }}.github.io/qa-reports/reports/${REPORT_ID}" | |
| env: | |
| REPORT_ID: ${{ needs.create_report.outputs.report_id }} | |
| - name: Send test results to Slack channel | |
| run: | | |
| build/ci/test-report.py \ | |
| --artifacts artifacts \ | |
| --slack-channel "${SLACK_CHANNEL}" \ | |
| --build-url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ | |
| --report-url "https://${{ github.repository_owner }}.github.io/qa-reports/reports/${REPORT_ID}/" \ | |
| --summary-url "${SUMMARY_URL}" | |
| env: | |
| SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| REPORT_ID: ${{ needs.create_report.outputs.report_id }} | |
| SUMMARY_URL: ${{ needs.create_report.outputs.summary_url }} |