From a73c659520fe8aad68ac1b290686f8f1d8441d12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Sun, 10 Mar 2024 16:16:22 +0100 Subject: [PATCH 01/13] try to send to pixel eagle --- .github/workflows/example-report.yml | 304 ++------------------------- 1 file changed, 20 insertions(+), 284 deletions(-) diff --git a/.github/workflows/example-report.yml b/.github/workflows/example-report.yml index c08329b..5427e5e 100644 --- a/.github/workflows/example-report.yml +++ b/.github/workflows/example-report.yml @@ -19,11 +19,6 @@ jobs: pages: ${{ steps.env.outputs.pages }} gitref: ${{ steps.env.outputs.gitref }} date: ${{ steps.env.outputs.date }} - linux_percy_project: ${{ steps.env.outputs.linux_percy_project }} - windows_percy_project: ${{ steps.env.outputs.windows_percy_project }} - macos_percy_project: ${{ steps.env.outputs.macos_percy_project }} - mobile_percy_project: ${{ steps.env.outputs.mobile_percy_project }} - mobile_nonce: ${{ steps.env.outputs.mobile_nonce }} updated: ${{ steps.version-check.outputs.updated }} steps: - name: Checkout Bevy main branch @@ -39,12 +34,7 @@ jobs: echo "gitref=`git rev-parse HEAD`" >> $GITHUB_OUTPUT echo "date=`date +%Y%m%d%H%M`" >> $GITHUB_OUTPUT - echo "linux_percy_project=dede4209/Screenshots-Linux-Vulkan" >> $GITHUB_OUTPUT - echo "windows_percy_project=dede4209/Screenshots-Windows-DX12" >> $GITHUB_OUTPUT - echo "macos_percy_project=dede4209/Screenshots-macOS-Metal" >> $GITHUB_OUTPUT - echo "mobile_percy_project=dede4209/Bevy-Mobile-Example" >> $GITHUB_OUTPUT echo "pages=`python -c \"import json; print(json.dumps([i for i in range($page_count)]))\"`" >> $GITHUB_OUTPUT - echo "mobile_nonce=${{ github.run_id }}-$(date +%s)" >> $GITHUB_OUTPUT - uses: actions/checkout@v4 with: ref: 'results' @@ -177,8 +167,8 @@ jobs: failures no_screenshots - send-to-percy: - name: Send screenshots to Percy + send-to-pixel-eagle: + name: Send screenshots to Pixel Eagle runs-on: ubuntu-latest needs: [take-screenshots, get-environment] strategy: @@ -186,14 +176,8 @@ jobs: matrix: include: - os: Linux - percy_key: PERCY_TOKEN_LINUX_VULKAN - percy_project: ${{ needs.get-environment.outputs.linux_percy_project }} - os: Windows - percy_key: PERCY_TOKEN_WINDOWS_DX12 - percy_project: ${{ needs.get-environment.outputs.windows_percy_project }} - os: macOS - percy_key: PERCY_TOKEN_MACOS_METAL - percy_project: ${{ needs.get-environment.outputs.macos_percy_project }} steps: - name: Download all artifacts @@ -231,274 +215,26 @@ jobs: fi done - - name: Remove example known to be random - if: steps.gather-examples.outcome == 'success' - run: | - rm "screenshots-${{ matrix.os }}/Animation/animated_fox.png" || true - rm "screenshots-${{ matrix.os }}/Animation/morph_targets.png" || true - rm "screenshots-${{ matrix.os }}/Async Tasks/async_compute.png" || true - rm "screenshots-${{ matrix.os }}/Async Tasks/external_source_external_thread.png" || true - rm "screenshots-${{ matrix.os }}/Games/alien_cake_addict.png" || true - rm "screenshots-${{ matrix.os }}/Games/contributors.png" || true - rm "screenshots-${{ matrix.os }}/UI (User Interface)/font_atlas_debug.png" || true - rm "screenshots-${{ matrix.os }}/Shaders/compute_shader_game_of_life.png" || true - - - name: Send to Percy + - name: Send to Pixel Eagle if: steps.gather-examples.outcome == 'success' run: | - npm install -g @percy/cli@latest - npx percy upload screenshots-${{ matrix.os }} - env: - PERCY_TOKEN: ${{ secrets[matrix.percy_key] }} - PERCY_COMMIT: ${{ needs.get-environment.outputs.gitref }} - - - name: Wait for result - if: steps.gather-examples.outcome == 'success' - run: | - npx percy build:wait --project ${{ matrix.percy_project }} --commit ${{ needs.get-environment.outputs.gitref }} - env: - PERCY_TOKEN: ${{ secrets[matrix.percy_key] }} - - update-results: - name: Update Results - runs-on: ubuntu-latest - needs: [send-to-percy, get-environment, mobile-run] - if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch') - permissions: - contents: write - steps: - - - uses: actions/checkout@v4 - with: - ref: 'results' - path: 'results' - - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - pattern: status-* + uuid="B25A040A-A980-4602-B90C-D480AB84076D" - - name: Concatenate status - run: | - set -x - - mkdir results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }} - - for report in status-Linux-* - do - (cat $report/successes; echo) >> Linux-successes-concat - (cat $report/failures; echo) >> Linux-failures-concat - (cat $report/no_screenshots; echo) >> Linux-no_screenshots-concat - done - # remove empty lines - grep . Linux-successes-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Linux-successes || true - grep . Linux-failures-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Linux-failures || true - grep . Linux-no_screenshots-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Linux-no_screenshots || true - - for report in status-Windows-* - do - (cat $report/successes; echo) >> Windows-successes-concat - (cat $report/failures; echo) >> Windows-failures-concat - (cat $report/no_screenshots; echo) >> Windows-no_screenshots-concat - done - # remove empty lines - grep . Windows-successes-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Windows-successes || true - grep . Windows-failures-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Windows-failures || true - grep . Windows-no_screenshots-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Windows-no_screenshots || true - - for report in status-macOS-* + id=`curl https://pixel-eagle.vleue.com/$uuid/runs --json '{"os":"${{ matrix.os }}", "gitref": "${{ needs.get-environment.outputs.gitref }}"}' | jq '.id'` + + SAVEIFS=$IFS + IFS=$'\n' + + for screenshot in $(find ./screenshots -type f -name "*.png"); do - (cat $report/successes; echo) >> macOS-successes-concat - (cat $report/failures; echo) >> macOS-failures-concat - (cat $report/no_screenshots; echo) >> macOS-no_screenshots-concat + echo $screenshot + sha=`shasum -a 256 $screenshot | awk '{print $1}'` + to_upload=`curl https://pixel-eagle.vleue.com/$uuid/runs/$id/hashes --json "[ [\"$screenshot\", \"$sha\"] ]" | jq '. | length'` + + if [ $to_upload -eq 1 ]; then + curl https://pixel-eagle.vleue.com/$uuid/runs/$id/screenshots -F "data=@$screenshot" -F "screenshot=$screenshot" | jq + fi + done - # remove empty lines - grep . macOS-successes-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/macOS-successes || true - grep . macOS-failures-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/macOS-failures || true - grep . macOS-no_screenshots-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/macOS-no_screenshots || true - - - name: Save Percy results - run: | - curl 'https://percy.io/api/v1/projects/${{ needs.get-environment.outputs.windows_percy_project }}/builds?filter\[sha\]=${{ needs.get-environment.outputs.gitref }}' | jq '.data[0].attributes' > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Windows-percy - curl 'https://percy.io/api/v1/projects/${{ needs.get-environment.outputs.linux_percy_project }}/builds?filter\[sha\]=${{ needs.get-environment.outputs.gitref }}' | jq '.data[0].attributes' > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Linux-percy - curl 'https://percy.io/api/v1/projects/${{ needs.get-environment.outputs.macos_percy_project }}/builds?filter\[sha\]=${{ needs.get-environment.outputs.gitref }}' | jq '.data[0].attributes' > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/macOS-percy - curl 'https://percy.io/api/v1/projects/${{ needs.get-environment.outputs.mobile_percy_project }}/builds?filter\[sha\]=${{ needs.get-environment.outputs.gitref }}' | jq '.data[0].attributes' > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/mobile-percy - - - name: Store results in git - run: | - cd results - - git config user.name 'Workflow' - git config user.email '<>' - - git add . - git commit -m "Update Results" - git push - - - name: Upload Aggregated Status - uses: actions/upload-artifact@v4 - with: - name: aggregated-status - path: | - results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }} - - rerun-failed-examples: - name: Rerun Failed Examples (without screenshot) - needs: [get-environment, update-results] - if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch') - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-14] - steps: - - - name: Checkout Bevy main branch - uses: actions/checkout@v4 - with: - repository: 'bevyengine/bevy' - ref: ${{ needs.get-environment.outputs.gitref }} - - - name: Checkout patches - uses: actions/checkout@v4 - with: - path: 'runner-patches' - - - name: Apply patches - shell: pwsh - run: | - Get-ChildItem "runner-patches/patches" -Filter *.patch | - Foreach-Object { - Write-Output "Processing $($_.FullName)" - git apply --ignore-whitespace $($_.FullName) - } - - - name: Setup Rust - uses: dtolnay/rust-toolchain@stable - - - name: Download Aggregated Status - uses: actions/download-artifact@v4 - with: - name: aggregated-status - path: aggregated-status - - - name: Clean Up failures list - run: | - sed 's/.*\/\(.*\) - [.0-9]*/\1/g' aggregated-status/${{ runner.os }}-failures > failure-list - - - name: Install Bevy dependencies - if: runner.os == 'linux' - run: | - sudo apt-get update; - DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends -yq \ - libasound2-dev libudev-dev; - - - name: install xvfb, llvmpipe and lavapipe - if: runner.os == 'linux' - run: | - sudo apt-get update -y -qq - sudo add-apt-repository ppa:kisak/turtle -y - sudo apt-get update - sudo apt install -y xvfb libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers - - - uses: actions/cache/restore@v4 - id: restore-cache - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} - restore-keys: ${{ runner.os }}-cargo- - - - name: Take Screenshots (Linux) - if: runner.os == 'linux' - continue-on-error: true - run: xvfb-run -s "-screen 0 1280x1024x24" cargo run -p example-showcase -- run --screenshot-at 0 --frame-duration 0.02 --stop-at 200 --in-ci --ignore-stress-tests --report-details --example-list failure-list - - - name: Take Screenshots (Windows) - if: runner.os == 'windows' - continue-on-error: true - run: cargo run -p example-showcase -- run --screenshot-at 0 --frame-duration 0.02 --stop-at 200 --in-ci --ignore-stress-tests --report-details --example-list failure-list - - - name: Take Screenshots (macOS) - if: runner.os == 'macos' - continue-on-error: true - run: cargo run -p example-showcase -- run --screenshot-at 0 --frame-duration 0.02 --stop-at 200 --in-ci --ignore-stress-tests --report-details --example-list failure-list - - - name: Upload Rerun Status - uses: actions/upload-artifact@v4 - with: - name: status-rerun-${{ runner.os }} - path: | - successes - failures - no_screenshots - *.log - - update-results-with-rerun: - name: Update Results with Rerun - needs: [rerun-failed-examples, get-environment] - if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch') - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - with: - ref: 'results' - path: 'results' - - - name: Download Rerun Status on Linux - uses: actions/download-artifact@v4 - with: - name: status-rerun-Linux - path: status-rerun-Linux - - - name: Download Rerun Status on Windows - uses: actions/download-artifact@v4 - with: - name: status-rerun-Windows - path: status-rerun-Windows - - - name: Download Rerun Status on macOS - uses: actions/download-artifact@v4 - with: - name: status-rerun-macOS - path: status-rerun-macOS - - - name: Store results in git - run: | - mv status-rerun-Windows results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/ - mv status-rerun-Linux results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/ - mv status-rerun-macOS results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/ - - cd results - - git config user.name 'Workflow' - git config user.email '<>' - - git add . - git commit -m "Update Results" - git push - - update-website: - name: Update Website - needs: [update-results-with-rerun, get-environment] - if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch') - uses: ./.github/workflows/rebuild-website.yml - permissions: - contents: read - pages: write - id-token: write - - mobile-run: - needs: [get-environment] - if: needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch' - uses: ./.github/workflows/workflow-mobile.yml - with: - gitref: ${{ needs.get-environment.outputs.gitref }} - nonce: ${{ needs.get-environment.outputs.mobile_nonce }} - mobile_percy_project: ${{ needs.get-environment.outputs.mobile_percy_project }} - secrets: inherit + + IFS=$SAVEIFS From 25cc2d9bf3751136e6d432ced6bfd210c55effc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Sun, 10 Mar 2024 17:13:38 +0100 Subject: [PATCH 02/13] shorter run --- .github/workflows/example-report.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/example-report.yml b/.github/workflows/example-report.yml index 5427e5e..a706ca1 100644 --- a/.github/workflows/example-report.yml +++ b/.github/workflows/example-report.yml @@ -53,8 +53,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-14] - page: ${{ fromJSON(needs.get-environment.outputs.pages) }} + os: [ubuntu-latest, macos-14] + page: [1] steps: @@ -176,7 +176,6 @@ jobs: matrix: include: - os: Linux - - os: Windows - os: macOS steps: From 9c341e74bf27ee70ddb44ca92393931fc7eaed98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Sun, 10 Mar 2024 17:13:46 +0100 Subject: [PATCH 03/13] get a recent curl --- .github/workflows/example-report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/example-report.yml b/.github/workflows/example-report.yml index a706ca1..5340549 100644 --- a/.github/workflows/example-report.yml +++ b/.github/workflows/example-report.yml @@ -169,7 +169,7 @@ jobs: send-to-pixel-eagle: name: Send screenshots to Pixel Eagle - runs-on: ubuntu-latest + runs-on: macos-14 needs: [take-screenshots, get-environment] strategy: fail-fast: false From 1b2bd2abdd53f9a0f51c2975fa34dd36b691ca17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Sun, 10 Mar 2024 17:13:57 +0100 Subject: [PATCH 04/13] in the correct directory --- .github/workflows/example-report.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/example-report.yml b/.github/workflows/example-report.yml index 5340549..d9e9eb0 100644 --- a/.github/workflows/example-report.yml +++ b/.github/workflows/example-report.yml @@ -224,7 +224,8 @@ jobs: SAVEIFS=$IFS IFS=$'\n' - for screenshot in $(find ./screenshots -type f -name "*.png"); + cd screenshots-${{ matrix.os }} + for screenshot in $(find ./ -type f -name "*.png"); do echo $screenshot sha=`shasum -a 256 $screenshot | awk '{print $1}'` From 6acddac778b4417ee917782f69474181d0e91af4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Sun, 10 Mar 2024 23:22:19 +0100 Subject: [PATCH 05/13] prettier path --- .github/workflows/example-report.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/example-report.yml b/.github/workflows/example-report.yml index d9e9eb0..37db2ff 100644 --- a/.github/workflows/example-report.yml +++ b/.github/workflows/example-report.yml @@ -227,6 +227,7 @@ jobs: cd screenshots-${{ matrix.os }} for screenshot in $(find ./ -type f -name "*.png"); do + screenshot=${screenshot:2} echo $screenshot sha=`shasum -a 256 $screenshot | awk '{print $1}'` to_upload=`curl https://pixel-eagle.vleue.com/$uuid/runs/$id/hashes --json "[ [\"$screenshot\", \"$sha\"] ]" | jq '. | length'` From 975696635c6b70d0bd8bd4d15293f1e8fc1396a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Sun, 10 Mar 2024 23:22:30 +0100 Subject: [PATCH 06/13] log more info --- .github/workflows/example-report.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/example-report.yml b/.github/workflows/example-report.yml index 37db2ff..83a0b47 100644 --- a/.github/workflows/example-report.yml +++ b/.github/workflows/example-report.yml @@ -233,9 +233,14 @@ jobs: to_upload=`curl https://pixel-eagle.vleue.com/$uuid/runs/$id/hashes --json "[ [\"$screenshot\", \"$sha\"] ]" | jq '. | length'` if [ $to_upload -eq 1 ]; then - curl https://pixel-eagle.vleue.com/$uuid/runs/$id/screenshots -F "data=@$screenshot" -F "screenshot=$screenshot" | jq + echo " uploading $screenshot" + curl https://pixel-eagle.vleue.com/$uuid/runs/$id/screenshots -F "data=@$screenshot" -F "screenshot=$screenshot" + else + echo " skipping $screenshot" fi done - + + echo "created run $id" + IFS=$SAVEIFS From 6a2f3042be9112c088a559c6a32064a9741b9f16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Sun, 10 Mar 2024 23:22:42 +0100 Subject: [PATCH 07/13] don't skip black screenshots --- .github/workflows/example-report.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.github/workflows/example-report.yml b/.github/workflows/example-report.yml index 83a0b47..8efe508 100644 --- a/.github/workflows/example-report.yml +++ b/.github/workflows/example-report.yml @@ -196,24 +196,6 @@ jobs: rm -rf $screenshotfolder done - - name: Remove images that are all black - if: steps.gather-examples.outcome == 'success' - run: | - set +e - sudo apt install -y imagemagick - for image in screenshots-${{ matrix.os }}/*/*.png - do - mean=`convert "$image" -format "%[mean]" info:` - if [[ "$?" = "1" ]]; then - echo "Error reading $image" - rm "$image" - fi - if [ "$mean" = 0 ]; then - echo "$image is all black" - rm "$image" - fi - done - - name: Send to Pixel Eagle if: steps.gather-examples.outcome == 'success' run: | From 40a3683d33a8e8aeafbc82a7ad5ef013efb1619e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Sun, 10 Mar 2024 23:44:42 +0100 Subject: [PATCH 08/13] keep the path --- .github/workflows/example-report.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/example-report.yml b/.github/workflows/example-report.yml index 8efe508..6099599 100644 --- a/.github/workflows/example-report.yml +++ b/.github/workflows/example-report.yml @@ -209,14 +209,14 @@ jobs: cd screenshots-${{ matrix.os }} for screenshot in $(find ./ -type f -name "*.png"); do - screenshot=${screenshot:2} - echo $screenshot + name=${screenshot:2} + echo $name sha=`shasum -a 256 $screenshot | awk '{print $1}'` - to_upload=`curl https://pixel-eagle.vleue.com/$uuid/runs/$id/hashes --json "[ [\"$screenshot\", \"$sha\"] ]" | jq '. | length'` + to_upload=`curl https://pixel-eagle.vleue.com/$uuid/runs/$id/hashes --json "[ [\"$name\", \"$sha\"] ]" | jq '. | length'` if [ $to_upload -eq 1 ]; then echo " uploading $screenshot" - curl https://pixel-eagle.vleue.com/$uuid/runs/$id/screenshots -F "data=@$screenshot" -F "screenshot=$screenshot" + curl https://pixel-eagle.vleue.com/$uuid/runs/$id/screenshots -F "data=@$screenshot" -F "screenshot=$name" else echo " skipping $screenshot" fi From 3da3ad0822f5a71c12c2e85a0fa5cd2b2f285654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Sun, 10 Mar 2024 23:44:48 +0100 Subject: [PATCH 09/13] one more page to test --- .github/workflows/example-report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/example-report.yml b/.github/workflows/example-report.yml index 6099599..9986674 100644 --- a/.github/workflows/example-report.yml +++ b/.github/workflows/example-report.yml @@ -54,7 +54,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-14] - page: [1] + page: [1, 2] steps: From 72c1eda9f03ae2886a5a12bc0e169c003619699e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Mon, 11 Mar 2024 00:05:22 +0100 Subject: [PATCH 10/13] better path --- .github/workflows/example-report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/example-report.yml b/.github/workflows/example-report.yml index 9986674..ec54d55 100644 --- a/.github/workflows/example-report.yml +++ b/.github/workflows/example-report.yml @@ -207,7 +207,7 @@ jobs: IFS=$'\n' cd screenshots-${{ matrix.os }} - for screenshot in $(find ./ -type f -name "*.png"); + for screenshot in $(find . -type f -name "*.png"); do name=${screenshot:2} echo $name From 1c540690b3826c1406b24c053cca278b4ffefc1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Mon, 11 Mar 2024 00:34:34 +0100 Subject: [PATCH 11/13] restore --- .github/workflows/example-report.yml | 339 ++++++++++++++++++++++++++- 1 file changed, 337 insertions(+), 2 deletions(-) diff --git a/.github/workflows/example-report.yml b/.github/workflows/example-report.yml index ec54d55..715e0ea 100644 --- a/.github/workflows/example-report.yml +++ b/.github/workflows/example-report.yml @@ -19,6 +19,11 @@ jobs: pages: ${{ steps.env.outputs.pages }} gitref: ${{ steps.env.outputs.gitref }} date: ${{ steps.env.outputs.date }} + linux_percy_project: ${{ steps.env.outputs.linux_percy_project }} + windows_percy_project: ${{ steps.env.outputs.windows_percy_project }} + macos_percy_project: ${{ steps.env.outputs.macos_percy_project }} + mobile_percy_project: ${{ steps.env.outputs.mobile_percy_project }} + mobile_nonce: ${{ steps.env.outputs.mobile_nonce }} updated: ${{ steps.version-check.outputs.updated }} steps: - name: Checkout Bevy main branch @@ -34,7 +39,12 @@ jobs: echo "gitref=`git rev-parse HEAD`" >> $GITHUB_OUTPUT echo "date=`date +%Y%m%d%H%M`" >> $GITHUB_OUTPUT + echo "linux_percy_project=dede4209/Screenshots-Linux-Vulkan" >> $GITHUB_OUTPUT + echo "windows_percy_project=dede4209/Screenshots-Windows-DX12" >> $GITHUB_OUTPUT + echo "macos_percy_project=dede4209/Screenshots-macOS-Metal" >> $GITHUB_OUTPUT + echo "mobile_percy_project=dede4209/Bevy-Mobile-Example" >> $GITHUB_OUTPUT echo "pages=`python -c \"import json; print(json.dumps([i for i in range($page_count)]))\"`" >> $GITHUB_OUTPUT + echo "mobile_nonce=${{ github.run_id }}-$(date +%s)" >> $GITHUB_OUTPUT - uses: actions/checkout@v4 with: ref: 'results' @@ -53,8 +63,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-14] - page: [1, 2] + os: [ubuntu-latest, windows-latest, macos-14] + page: ${{ fromJSON(needs.get-environment.outputs.pages) }} steps: @@ -167,6 +177,331 @@ jobs: failures no_screenshots + send-to-percy: + name: Send screenshots to Percy + runs-on: ubuntu-latest + needs: [take-screenshots, get-environment] + strategy: + fail-fast: false + matrix: + include: + - os: Linux + percy_key: PERCY_TOKEN_LINUX_VULKAN + percy_project: ${{ needs.get-environment.outputs.linux_percy_project }} + - os: Windows + percy_key: PERCY_TOKEN_WINDOWS_DX12 + percy_project: ${{ needs.get-environment.outputs.windows_percy_project }} + - os: macOS + percy_key: PERCY_TOKEN_MACOS_METAL + percy_project: ${{ needs.get-environment.outputs.macos_percy_project }} + steps: + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + pattern: screenshots-${{ matrix.os }}-* + + - name: Move examples to the correct folder + id: gather-examples + continue-on-error: true + run: | + mkdir screenshots-${{ matrix.os }} + for screenshotfolder in screenshots-${{ matrix.os }}-* + do + echo $screenshotfolder + rsync --verbose --archive $screenshotfolder/* screenshots-${{ matrix.os }}/ + rm -rf $screenshotfolder + done + + - name: Remove images that are all black + if: steps.gather-examples.outcome == 'success' + run: | + set +e + sudo apt install -y imagemagick + for image in screenshots-${{ matrix.os }}/*/*.png + do + mean=`convert "$image" -format "%[mean]" info:` + if [[ "$?" = "1" ]]; then + echo "Error reading $image" + rm "$image" + fi + if [ "$mean" = 0 ]; then + echo "$image is all black" + rm "$image" + fi + done + + - name: Remove example known to be random + if: steps.gather-examples.outcome == 'success' + run: | + rm "screenshots-${{ matrix.os }}/Animation/animated_fox.png" || true + rm "screenshots-${{ matrix.os }}/Animation/morph_targets.png" || true + rm "screenshots-${{ matrix.os }}/Async Tasks/async_compute.png" || true + rm "screenshots-${{ matrix.os }}/Async Tasks/external_source_external_thread.png" || true + rm "screenshots-${{ matrix.os }}/Games/alien_cake_addict.png" || true + rm "screenshots-${{ matrix.os }}/Games/contributors.png" || true + rm "screenshots-${{ matrix.os }}/UI (User Interface)/font_atlas_debug.png" || true + rm "screenshots-${{ matrix.os }}/Shaders/compute_shader_game_of_life.png" || true + + - name: Send to Percy + if: steps.gather-examples.outcome == 'success' + run: | + npm install -g @percy/cli@latest + npx percy upload screenshots-${{ matrix.os }} + env: + PERCY_TOKEN: ${{ secrets[matrix.percy_key] }} + PERCY_COMMIT: ${{ needs.get-environment.outputs.gitref }} + + - name: Wait for result + if: steps.gather-examples.outcome == 'success' + run: | + npx percy build:wait --project ${{ matrix.percy_project }} --commit ${{ needs.get-environment.outputs.gitref }} + env: + PERCY_TOKEN: ${{ secrets[matrix.percy_key] }} + + update-results: + name: Update Results + runs-on: ubuntu-latest + needs: [send-to-percy, get-environment, mobile-run] + if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch') + permissions: + contents: write + steps: + + - uses: actions/checkout@v4 + with: + ref: 'results' + path: 'results' + + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + pattern: status-* + + - name: Concatenate status + run: | + set -x + + mkdir results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }} + + for report in status-Linux-* + do + (cat $report/successes; echo) >> Linux-successes-concat + (cat $report/failures; echo) >> Linux-failures-concat + (cat $report/no_screenshots; echo) >> Linux-no_screenshots-concat + done + # remove empty lines + grep . Linux-successes-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Linux-successes || true + grep . Linux-failures-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Linux-failures || true + grep . Linux-no_screenshots-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Linux-no_screenshots || true + + for report in status-Windows-* + do + (cat $report/successes; echo) >> Windows-successes-concat + (cat $report/failures; echo) >> Windows-failures-concat + (cat $report/no_screenshots; echo) >> Windows-no_screenshots-concat + done + # remove empty lines + grep . Windows-successes-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Windows-successes || true + grep . Windows-failures-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Windows-failures || true + grep . Windows-no_screenshots-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Windows-no_screenshots || true + + for report in status-macOS-* + do + (cat $report/successes; echo) >> macOS-successes-concat + (cat $report/failures; echo) >> macOS-failures-concat + (cat $report/no_screenshots; echo) >> macOS-no_screenshots-concat + done + # remove empty lines + grep . macOS-successes-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/macOS-successes || true + grep . macOS-failures-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/macOS-failures || true + grep . macOS-no_screenshots-concat > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/macOS-no_screenshots || true + + - name: Save Percy results + run: | + curl 'https://percy.io/api/v1/projects/${{ needs.get-environment.outputs.windows_percy_project }}/builds?filter\[sha\]=${{ needs.get-environment.outputs.gitref }}' | jq '.data[0].attributes' > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Windows-percy + curl 'https://percy.io/api/v1/projects/${{ needs.get-environment.outputs.linux_percy_project }}/builds?filter\[sha\]=${{ needs.get-environment.outputs.gitref }}' | jq '.data[0].attributes' > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/Linux-percy + curl 'https://percy.io/api/v1/projects/${{ needs.get-environment.outputs.macos_percy_project }}/builds?filter\[sha\]=${{ needs.get-environment.outputs.gitref }}' | jq '.data[0].attributes' > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/macOS-percy + curl 'https://percy.io/api/v1/projects/${{ needs.get-environment.outputs.mobile_percy_project }}/builds?filter\[sha\]=${{ needs.get-environment.outputs.gitref }}' | jq '.data[0].attributes' > results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/mobile-percy + + - name: Store results in git + run: | + cd results + + git config user.name 'Workflow' + git config user.email '<>' + + git add . + git commit -m "Update Results" + git push + + - name: Upload Aggregated Status + uses: actions/upload-artifact@v4 + with: + name: aggregated-status + path: | + results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }} + + rerun-failed-examples: + name: Rerun Failed Examples (without screenshot) + needs: [get-environment, update-results] + if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch') + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-14] + steps: + + - name: Checkout Bevy main branch + uses: actions/checkout@v4 + with: + repository: 'bevyengine/bevy' + ref: ${{ needs.get-environment.outputs.gitref }} + + - name: Checkout patches + uses: actions/checkout@v4 + with: + path: 'runner-patches' + + - name: Apply patches + shell: pwsh + run: | + Get-ChildItem "runner-patches/patches" -Filter *.patch | + Foreach-Object { + Write-Output "Processing $($_.FullName)" + git apply --ignore-whitespace $($_.FullName) + } + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + + - name: Download Aggregated Status + uses: actions/download-artifact@v4 + with: + name: aggregated-status + path: aggregated-status + + - name: Clean Up failures list + run: | + sed 's/.*\/\(.*\) - [.0-9]*/\1/g' aggregated-status/${{ runner.os }}-failures > failure-list + + - name: Install Bevy dependencies + if: runner.os == 'linux' + run: | + sudo apt-get update; + DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends -yq \ + libasound2-dev libudev-dev; + + - name: install xvfb, llvmpipe and lavapipe + if: runner.os == 'linux' + run: | + sudo apt-get update -y -qq + sudo add-apt-repository ppa:kisak/turtle -y + sudo apt-get update + sudo apt install -y xvfb libegl1-mesa libgl1-mesa-dri libxcb-xfixes0-dev mesa-vulkan-drivers + + - uses: actions/cache/restore@v4 + id: restore-cache + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} + restore-keys: ${{ runner.os }}-cargo- + + - name: Take Screenshots (Linux) + if: runner.os == 'linux' + continue-on-error: true + run: xvfb-run -s "-screen 0 1280x1024x24" cargo run -p example-showcase -- run --screenshot-at 0 --frame-duration 0.02 --stop-at 200 --in-ci --ignore-stress-tests --report-details --example-list failure-list + + - name: Take Screenshots (Windows) + if: runner.os == 'windows' + continue-on-error: true + run: cargo run -p example-showcase -- run --screenshot-at 0 --frame-duration 0.02 --stop-at 200 --in-ci --ignore-stress-tests --report-details --example-list failure-list + + - name: Take Screenshots (macOS) + if: runner.os == 'macos' + continue-on-error: true + run: cargo run -p example-showcase -- run --screenshot-at 0 --frame-duration 0.02 --stop-at 200 --in-ci --ignore-stress-tests --report-details --example-list failure-list + + - name: Upload Rerun Status + uses: actions/upload-artifact@v4 + with: + name: status-rerun-${{ runner.os }} + path: | + successes + failures + no_screenshots + *.log + + update-results-with-rerun: + name: Update Results with Rerun + needs: [rerun-failed-examples, get-environment] + if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + ref: 'results' + path: 'results' + + - name: Download Rerun Status on Linux + uses: actions/download-artifact@v4 + with: + name: status-rerun-Linux + path: status-rerun-Linux + + - name: Download Rerun Status on Windows + uses: actions/download-artifact@v4 + with: + name: status-rerun-Windows + path: status-rerun-Windows + + - name: Download Rerun Status on macOS + uses: actions/download-artifact@v4 + with: + name: status-rerun-macOS + path: status-rerun-macOS + + - name: Store results in git + run: | + mv status-rerun-Windows results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/ + mv status-rerun-Linux results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/ + mv status-rerun-macOS results/${{ needs.get-environment.outputs.date }}-${{ needs.get-environment.outputs.gitref }}/ + + cd results + + git config user.name 'Workflow' + git config user.email '<>' + + git add . + git commit -m "Update Results" + git push + + update-website: + name: Update Website + needs: [update-results-with-rerun, get-environment] + if: always() && (needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch') + uses: ./.github/workflows/rebuild-website.yml + permissions: + contents: read + pages: write + id-token: write + + mobile-run: + needs: [get-environment] + if: needs.get-environment.outputs.updated == 'true' || github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/workflow-mobile.yml + with: + gitref: ${{ needs.get-environment.outputs.gitref }} + nonce: ${{ needs.get-environment.outputs.mobile_nonce }} + mobile_percy_project: ${{ needs.get-environment.outputs.mobile_percy_project }} + secrets: inherit send-to-pixel-eagle: name: Send screenshots to Pixel Eagle runs-on: macos-14 From 269044a678298ed9c571523ece9eb92d301be3ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Mon, 11 Mar 2024 00:35:16 +0100 Subject: [PATCH 12/13] extra newline --- .github/workflows/example-report.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/example-report.yml b/.github/workflows/example-report.yml index 715e0ea..6b68acd 100644 --- a/.github/workflows/example-report.yml +++ b/.github/workflows/example-report.yml @@ -502,6 +502,7 @@ jobs: nonce: ${{ needs.get-environment.outputs.mobile_nonce }} mobile_percy_project: ${{ needs.get-environment.outputs.mobile_percy_project }} secrets: inherit + send-to-pixel-eagle: name: Send screenshots to Pixel Eagle runs-on: macos-14 From b577d8b79b6669b3d4c436b16a5a91c59a042b86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Mon, 11 Mar 2024 00:36:23 +0100 Subject: [PATCH 13/13] windows --- .github/workflows/example-report.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/example-report.yml b/.github/workflows/example-report.yml index 6b68acd..99a356e 100644 --- a/.github/workflows/example-report.yml +++ b/.github/workflows/example-report.yml @@ -513,6 +513,7 @@ jobs: include: - os: Linux - os: macOS + - os: Windows steps: - name: Download all artifacts