CI: mergeable check redesign #61643
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# yamllint disable rule:comments-indentation | |
name: PullRequestCI | |
env: | |
# Force the stdout and stderr streams to be unbuffered | |
PYTHONUNBUFFERED: 1 | |
on: # yamllint disable-line rule:truthy | |
pull_request: | |
types: | |
- synchronize | |
- reopened | |
- opened | |
branches: | |
- master | |
# Cancel the previous wf run in PRs. | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.ref }} | |
cancel-in-progress: true | |
jobs: | |
RunConfig: | |
runs-on: [self-hosted, style-checker-aarch64] | |
outputs: | |
data: ${{ steps.runconfig.outputs.CI_DATA }} | |
steps: | |
- name: DebugInfo | |
uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 | |
- name: Check out repository code | |
uses: ClickHouse/checkout@v1 | |
with: | |
clear-repository: true # to ensure correct digests | |
fetch-depth: 0 # to get version | |
filter: tree:0 | |
- name: Labels check | |
run: | | |
cd "$GITHUB_WORKSPACE/tests/ci" | |
python3 run_check.py | |
- name: Python unit tests | |
run: | | |
cd "$GITHUB_WORKSPACE/tests/ci" | |
echo "Testing the main ci directory" | |
python3 -m unittest discover -s . -p 'test_*.py' | |
for dir in *_lambda/; do | |
echo "Testing $dir" | |
python3 -m unittest discover -s "$dir" -p 'test_*.py' | |
done | |
- name: PrepareRunConfig | |
id: runconfig | |
run: | | |
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json | |
echo "::group::CI configuration" | |
python3 -m json.tool ${{ runner.temp }}/ci_run_data.json | |
echo "::endgroup::" | |
{ | |
echo 'CI_DATA<<EOF' | |
cat ${{ runner.temp }}/ci_run_data.json | |
echo 'EOF' | |
} >> "$GITHUB_OUTPUT" | |
- name: Re-create GH statuses for skipped jobs if any | |
run: | | |
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --update-gh-statuses | |
BuildDockers: | |
needs: [RunConfig] | |
if: ${{ !failure() && !cancelled() && toJson(fromJson(needs.RunConfig.outputs.data).docker_data.missing_multi) != '[]' }} | |
uses: ./.github/workflows/reusable_docker.yml | |
with: | |
data: ${{ needs.RunConfig.outputs.data }} | |
StyleCheck: | |
needs: [RunConfig, BuildDockers] | |
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Style check')}} | |
uses: ./.github/workflows/reusable_test.yml | |
with: | |
test_name: Style check | |
runner_type: style-checker | |
run_command: | | |
python3 style_check.py | |
data: ${{ needs.RunConfig.outputs.data }} | |
secrets: | |
secret_envs: | | |
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK | |
${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}} | |
RCSK | |
FastTest: | |
needs: [RunConfig, BuildDockers, StyleCheck] | |
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }} | |
uses: ./.github/workflows/reusable_test.yml | |
with: | |
test_name: Fast test | |
runner_type: builder | |
data: ${{ needs.RunConfig.outputs.data }} | |
run_command: | | |
python3 fast_test_check.py | |
################################# Main statges ################################# | |
# for main CI chain | |
# | |
Builds_1: | |
needs: [RunConfig, StyleCheck, FastTest] | |
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_1') }} | |
# using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab | |
uses: ./.github/workflows/reusable_build_stage.yml | |
with: | |
stage: Builds_1 | |
data: ${{ needs.RunConfig.outputs.data }} | |
Tests_1: | |
needs: [RunConfig, Builds_1] | |
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_1') }} | |
# using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab | |
uses: ./.github/workflows/reusable_test_stage.yml | |
with: | |
stage: Tests_1 | |
data: ${{ needs.RunConfig.outputs.data }} | |
Builds_2: | |
needs: [RunConfig, Builds_1] | |
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_2') }} | |
# using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab | |
uses: ./.github/workflows/reusable_build_stage.yml | |
with: | |
stage: Builds_2 | |
data: ${{ needs.RunConfig.outputs.data }} | |
Tests_2: | |
needs: [RunConfig, Builds_2] | |
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }} | |
# using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab | |
uses: ./.github/workflows/reusable_test_stage.yml | |
with: | |
stage: Tests_2 | |
data: ${{ needs.RunConfig.outputs.data }} | |
# stage for jobs that do not prohibit merge | |
Tests_3: | |
needs: [RunConfig, Tests_1, Tests_2] | |
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }} | |
uses: ./.github/workflows/reusable_test_stage.yml | |
with: | |
stage: Tests_3 | |
data: ${{ needs.RunConfig.outputs.data }} | |
################################# Reports ################################# | |
# Reports should by run even if Builds_1/2 fail, so put them separately in wf (not in Tests_1/2) | |
Builds_1_Report: | |
# run report check for failed builds to indicate the CI error | |
if: ${{ !cancelled() && needs.StyleCheck.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse build check') }} | |
needs: [RunConfig, StyleCheck, Builds_1] | |
uses: ./.github/workflows/reusable_test.yml | |
with: | |
test_name: ClickHouse build check | |
runner_type: style-checker-aarch64 | |
data: ${{ needs.RunConfig.outputs.data }} | |
Builds_2_Report: | |
# run report check for failed builds to indicate the CI error | |
if: ${{ !cancelled() && needs.StyleCheck.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse special build check') }} | |
needs: [RunConfig, StyleCheck, Builds_2] | |
uses: ./.github/workflows/reusable_test.yml | |
with: | |
test_name: ClickHouse special build check | |
runner_type: style-checker-aarch64 | |
data: ${{ needs.RunConfig.outputs.data }} | |
CheckReadyForMerge: | |
if: ${{ !cancelled() && needs.StyleCheck.result == 'success' }} | |
needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2] | |
runs-on: [self-hosted, style-checker-aarch64] | |
steps: | |
- name: Check out repository code | |
uses: ClickHouse/checkout@v1 | |
with: | |
filter: tree:0 | |
- name: Check and set merge status | |
run: | | |
cd "$GITHUB_WORKSPACE/tests/ci" | |
python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }} | |
################################# Stage Final ################################# | |
# | |
FinishCheck: | |
if: ${{ !failure() && !cancelled() }} | |
needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2, Tests_3, CheckReadyForMerge] | |
runs-on: [self-hosted, style-checker] | |
steps: | |
- name: Check out repository code | |
uses: ClickHouse/checkout@v1 | |
with: | |
filter: tree:0 | |
- name: Finish label | |
run: | | |
cd "$GITHUB_WORKSPACE/tests/ci" | |
python3 finish_check.py | |
# FIXME: merge on approval does not work with MQ. Could be fixed by using defaul GH's automerge after some corrections in Mergeable Check status | |
# - name: Auto merge if approved | |
# if: ${{ github.event_name != 'merge_group' }} | |
# run: | | |
# cd "$GITHUB_WORKSPACE/tests/ci" | |
# python3 merge_pr.py --check-approved | |
############################################################################################# | |
###################################### JEPSEN TESTS ######################################### | |
############################################################################################# | |
# This is special test NOT INCLUDED in FinishCheck | |
# When it's skipped, all dependent tasks will be skipped too. | |
# DO NOT add it there | |
Jepsen: | |
# we need concurrency as the job uses dedicated instances in the cloud | |
concurrency: | |
group: jepsen | |
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse Keeper Jepsen') }} | |
# jepsen needs binary_release build which is in Builds_2 | |
needs: [RunConfig, Builds_2] | |
uses: ./.github/workflows/reusable_test.yml | |
with: | |
test_name: ClickHouse Keeper Jepsen | |
runner_type: style-checker | |
data: ${{ needs.RunConfig.outputs.data }} |