Merge pull request #64105 from ClickHouse/backport/24.2/64024 #3237
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # yamllint disable rule:comments-indentation | |
| name: ReleaseBranchCI | |
| env: | |
| # Force the stdout and stderr streams to be unbuffered | |
| PYTHONUNBUFFERED: 1 | |
| on: # yamllint disable-line rule:truthy | |
| push: | |
| branches: | |
| # 22.1 and 22.10 | |
| - '2[1-9].[1-9][0-9]' | |
| - '2[1-9].[1-9]' | |
| jobs: | |
| RunConfig: | |
| runs-on: [self-hosted, style-checker-aarch64] | |
| outputs: | |
| data: ${{ steps.runconfig.outputs.CI_DATA }} | |
| steps: | |
| - name: Check out repository code | |
| uses: ClickHouse/checkout@v1 | |
| with: | |
| clear-repository: true # to ensure correct digests | |
| fetch-depth: 0 # to get version | |
| filter: tree:0 | |
| - name: Labels check | |
| run: | | |
| cd "$GITHUB_WORKSPACE/tests/ci" | |
| python3 run_check.py | |
| - name: Python unit tests | |
| run: | | |
| cd "$GITHUB_WORKSPACE/tests/ci" | |
| echo "Testing the main ci directory" | |
| python3 -m unittest discover -s . -p 'test_*.py' | |
| for dir in *_lambda/; do | |
| echo "Testing $dir" | |
| python3 -m unittest discover -s "$dir" -p 'test_*.py' | |
| done | |
| - name: PrepareRunConfig | |
| id: runconfig | |
| run: | | |
| echo "::group::configure CI run" | |
| python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json | |
| echo "::endgroup::" | |
| echo "::group::CI run configure results" | |
| python3 -m json.tool ${{ runner.temp }}/ci_run_data.json | |
| echo "::endgroup::" | |
| { | |
| echo 'CI_DATA<<EOF' | |
| cat ${{ runner.temp }}/ci_run_data.json | |
| echo 'EOF' | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Re-create GH statuses for skipped jobs if any | |
| run: | | |
| python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --update-gh-statuses | |
| BuildDockers: | |
| needs: [RunConfig] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_docker.yml | |
| with: | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| CompatibilityCheckX86: | |
| needs: [RunConfig, BuilderDebRelease] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Compatibility check (amd64) | |
| runner_type: style-checker | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| CompatibilityCheckAarch64: | |
| needs: [RunConfig, BuilderDebAarch64] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Compatibility check (aarch64) | |
| runner_type: style-checker | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| ######################################################################################### | |
| #################################### ORDINARY BUILDS #################################### | |
| ######################################################################################### | |
| BuilderDebRelease: | |
| needs: [RunConfig, BuildDockers] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_build.yml | |
| with: | |
| build_name: package_release | |
| checkout_depth: 0 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| # always rebuild on release branches to be able to publish from any commit | |
| force: true | |
| BuilderDebAarch64: | |
| needs: [RunConfig, BuildDockers] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_build.yml | |
| with: | |
| build_name: package_aarch64 | |
| checkout_depth: 0 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| # always rebuild on release branches to be able to publish from any commit | |
| force: true | |
| BuilderDebAsan: | |
| needs: [RunConfig, BuildDockers] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_build.yml | |
| with: | |
| build_name: package_asan | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| BuilderDebUBsan: | |
| needs: [RunConfig, BuildDockers] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_build.yml | |
| with: | |
| build_name: package_ubsan | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| BuilderDebTsan: | |
| needs: [RunConfig, BuildDockers] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_build.yml | |
| with: | |
| build_name: package_tsan | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| BuilderDebMsan: | |
| needs: [RunConfig, BuildDockers] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_build.yml | |
| with: | |
| build_name: package_msan | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| BuilderDebDebug: | |
| needs: [RunConfig, BuildDockers] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_build.yml | |
| with: | |
| build_name: package_debug | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| BuilderBinDarwin: | |
| needs: [RunConfig, BuildDockers] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_build.yml | |
| with: | |
| build_name: binary_darwin | |
| checkout_depth: 0 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| # always rebuild on release branches to be able to publish from any commit | |
| force: true | |
| BuilderBinDarwinAarch64: | |
| needs: [RunConfig, BuildDockers] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_build.yml | |
| with: | |
| build_name: binary_darwin_aarch64 | |
| checkout_depth: 0 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| # always rebuild on release branches to be able to publish from any commit | |
| force: true | |
| ############################################################################################ | |
| ##################################### Docker images ####################################### | |
| ############################################################################################ | |
| DockerServerImage: | |
| needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Docker server image | |
| runner_type: style-checker | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| DockerKeeperImage: | |
| needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Docker keeper image | |
| runner_type: style-checker | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| ############################################################################################ | |
| ##################################### BUILD REPORTER ####################################### | |
| ############################################################################################ | |
| BuilderReport: | |
| # run report check for failed builds to indicate the CI error | |
| if: ${{ !cancelled() }} | |
| needs: | |
| - RunConfig | |
| - BuilderDebRelease | |
| - BuilderDebAarch64 | |
| - BuilderDebAsan | |
| - BuilderDebTsan | |
| - BuilderDebUBsan | |
| - BuilderDebMsan | |
| - BuilderDebDebug | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: ClickHouse build check | |
| runner_type: style-checker-aarch64 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| BuilderSpecialReport: | |
| # run report check for failed builds to indicate the CI error | |
| if: ${{ !cancelled() }} | |
| needs: | |
| - RunConfig | |
| - BuilderBinDarwin | |
| - BuilderBinDarwinAarch64 | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: ClickHouse special build check | |
| runner_type: style-checker-aarch64 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| MarkReleaseReady: | |
| if: ${{ !failure() && !cancelled() }} | |
| needs: | |
| - BuilderBinDarwin | |
| - BuilderBinDarwinAarch64 | |
| - BuilderDebRelease | |
| - BuilderDebAarch64 | |
| runs-on: [self-hosted, style-checker-aarch64] | |
| steps: | |
| - name: Debug | |
| run: | | |
| echo need with different filters | |
| cat << 'EOF' | |
| ${{ toJSON(needs) }} | |
| ${{ toJSON(needs.*.result) }} | |
| no failures ${{ !contains(needs.*.result, 'failure') }} | |
| no skips ${{ !contains(needs.*.result, 'skipped') }} | |
| no both ${{ !(contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }} | |
| EOF | |
| - name: Not ready | |
| # fail the job to be able restart it | |
| if: ${{ contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure') }} | |
| run: exit 1 | |
| - name: Check out repository code | |
| if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }} | |
| uses: ClickHouse/checkout@v1 | |
| - name: Mark Commit Release Ready | |
| if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }} | |
| run: | | |
| cd "$GITHUB_WORKSPACE/tests/ci" | |
| python3 mark_release_ready.py | |
| ############################################################################################ | |
| #################################### INSTALL PACKAGES ###################################### | |
| ############################################################################################ | |
| InstallPackagesTestRelease: | |
| needs: [RunConfig, BuilderDebRelease] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Install packages (amd64) | |
| runner_type: style-checker | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| run_command: | | |
| python3 install_check.py "$CHECK_NAME" | |
| InstallPackagesTestAarch64: | |
| needs: [RunConfig, BuilderDebAarch64] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Install packages (arm64) | |
| runner_type: style-checker-aarch64 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| run_command: | | |
| python3 install_check.py "$CHECK_NAME" | |
| ############################################################################################## | |
| ########################### FUNCTIONAl STATELESS TESTS ####################################### | |
| ############################################################################################## | |
| FunctionalStatelessTestRelease: | |
| needs: [RunConfig, BuilderDebRelease] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stateless tests (release) | |
| runner_type: func-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatelessTestAarch64: | |
| needs: [RunConfig, BuilderDebAarch64] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stateless tests (aarch64) | |
| runner_type: func-tester-aarch64 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatelessTestAsan: | |
| needs: [RunConfig, BuilderDebAsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stateless tests (asan) | |
| runner_type: func-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatelessTestTsan: | |
| needs: [RunConfig, BuilderDebTsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stateless tests (tsan) | |
| runner_type: func-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatelessTestMsan: | |
| needs: [RunConfig, BuilderDebMsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stateless tests (msan) | |
| runner_type: func-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatelessTestUBsan: | |
| needs: [RunConfig, BuilderDebUBsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stateless tests (ubsan) | |
| runner_type: func-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatelessTestDebug: | |
| needs: [RunConfig, BuilderDebDebug] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stateless tests (debug) | |
| runner_type: func-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| ############################################################################################## | |
| ############################ FUNCTIONAl STATEFUL TESTS ####################################### | |
| ############################################################################################## | |
| FunctionalStatefulTestRelease: | |
| needs: [RunConfig, BuilderDebRelease] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stateful tests (release) | |
| runner_type: func-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatefulTestAarch64: | |
| needs: [RunConfig, BuilderDebAarch64] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stateful tests (aarch64) | |
| runner_type: func-tester-aarch64 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatefulTestAsan: | |
| needs: [RunConfig, BuilderDebAsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stateful tests (asan) | |
| runner_type: func-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatefulTestTsan: | |
| needs: [RunConfig, BuilderDebTsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stateful tests (tsan) | |
| runner_type: func-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatefulTestMsan: | |
| needs: [RunConfig, BuilderDebMsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stateful tests (msan) | |
| runner_type: func-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatefulTestUBsan: | |
| needs: [RunConfig, BuilderDebUBsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stateful tests (ubsan) | |
| runner_type: func-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FunctionalStatefulTestDebug: | |
| needs: [RunConfig, BuilderDebDebug] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stateful tests (debug) | |
| runner_type: func-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| ############################################################################################## | |
| ######################################### STRESS TESTS ####################################### | |
| ############################################################################################## | |
| StressTestAsan: | |
| needs: [RunConfig, BuilderDebAsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stress test (asan) | |
| runner_type: stress-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| StressTestTsan: | |
| needs: [RunConfig, BuilderDebTsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stress test (tsan) | |
| runner_type: stress-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| StressTestMsan: | |
| needs: [RunConfig, BuilderDebMsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stress test (msan) | |
| runner_type: stress-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| StressTestUBsan: | |
| needs: [RunConfig, BuilderDebUBsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stress test (ubsan) | |
| runner_type: stress-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| StressTestDebug: | |
| needs: [RunConfig, BuilderDebDebug] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Stress test (debug) | |
| runner_type: stress-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| ############################################################################################# | |
| ############################# INTEGRATION TESTS ############################################# | |
| ############################################################################################# | |
| IntegrationTestsAsan: | |
| needs: [RunConfig, BuilderDebAsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Integration tests (asan) | |
| runner_type: stress-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| IntegrationTestsAnalyzerAsan: | |
| needs: [RunConfig, BuilderDebAsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Integration tests (asan, analyzer) | |
| runner_type: stress-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| IntegrationTestsTsan: | |
| needs: [RunConfig, BuilderDebTsan] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Integration tests (tsan) | |
| runner_type: stress-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| IntegrationTestsRelease: | |
| needs: [RunConfig, BuilderDebRelease] | |
| if: ${{ !failure() && !cancelled() }} | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: Integration tests (release) | |
| runner_type: stress-tester | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| FinishCheck: | |
| if: ${{ !failure() && !cancelled() }} | |
| needs: | |
| - DockerServerImage | |
| - DockerKeeperImage | |
| - BuilderReport | |
| - BuilderSpecialReport | |
| - MarkReleaseReady | |
| - FunctionalStatelessTestDebug | |
| - FunctionalStatelessTestRelease | |
| - FunctionalStatelessTestAarch64 | |
| - FunctionalStatelessTestAsan | |
| - FunctionalStatelessTestTsan | |
| - FunctionalStatelessTestMsan | |
| - FunctionalStatelessTestUBsan | |
| - FunctionalStatefulTestDebug | |
| - FunctionalStatefulTestRelease | |
| - FunctionalStatefulTestAarch64 | |
| - FunctionalStatefulTestAsan | |
| - FunctionalStatefulTestTsan | |
| - FunctionalStatefulTestMsan | |
| - FunctionalStatefulTestUBsan | |
| - StressTestDebug | |
| - StressTestAsan | |
| - StressTestTsan | |
| - StressTestMsan | |
| - StressTestUBsan | |
| - IntegrationTestsAsan | |
| - IntegrationTestsTsan | |
| - IntegrationTestsRelease | |
| - CompatibilityCheckX86 | |
| - CompatibilityCheckAarch64 | |
| runs-on: [self-hosted, style-checker] | |
| steps: | |
| - name: Check out repository code | |
| uses: ClickHouse/checkout@v1 | |
| with: | |
| clear-repository: true | |
| - name: Finish label | |
| run: | | |
| cd "$GITHUB_WORKSPACE/tests/ci" | |
| python3 finish_check.py |