.gitlab-ci.yml

# polkadot-sdk | CI definitions (via GitLab CI)
#
# FYI: Pipelines can be triggered manually through the web UI (if you have enough permissions)
#
# Currently, entire CI instructions are split into different subfiles. Each CI stage has a corresponding
# file which can be found here: .gitlab/pipeline/<stage_name>.yml

stages:
  - check
  - test
  - build
  - publish
  - short-benchmarks
  - zombienet
  - deploy
  - notify

workflow:
  rules:
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH

variables:
  CI_IMAGE: !reference [.ci-unified, variables, CI_IMAGE]
  # BUILDAH_IMAGE is defined in group variables
  BUILDAH_COMMAND: "buildah --storage-driver overlay2"
  RELENG_SCRIPTS_BRANCH: "master"
  RUSTY_CACHIER_SINGLE_BRANCH: master
  RUSTY_CACHIER_DONT_OPERATE_ON_MAIN_BRANCH: "true"
  RUSTY_CACHIER_COMPRESSION_METHOD: zstd
  NEXTEST_FAILURE_OUTPUT: immediate-final
  NEXTEST_SUCCESS_OUTPUT: final
  DOCKER_IMAGES_VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"

default:
  retry:
    max: 2
    when:
      - runner_system_failure
      - unknown_failure
      - api_failure
  cache: {}
  interruptible: true

.collect-artifacts:
  artifacts:
    name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
    when: on_success
    expire_in: 1 days
    paths:
      - artifacts/

.collect-artifacts-short:
  artifacts:
    name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
    when: on_failure
    expire_in: 3 hours
    paths:
      - artifacts/

.prepare-env:
  before_script:
    # $WASM_BUILD_WORKSPACE_HINT enables wasm-builder to find the Cargo.lock from within generated
    # packages
    - export WASM_BUILD_WORKSPACE_HINT="$PWD"
    # ensure that RUSTFLAGS are set correctly
    - echo $RUSTFLAGS

.common-before-script:
  before_script:
    - !reference [.job-switcher, before_script]
    - !reference [.pipeline-stopper-vars, script]

.job-switcher:
  before_script:
    - if echo "$CI_DISABLED_JOBS" | grep -xF "$CI_JOB_NAME"; then echo "The job has been cancelled in CI settings"; exit 0; fi

.kubernetes-env:
  image: "${CI_IMAGE}"
  before_script:
    - !reference [.common-before-script, before_script]
    - !reference [.prepare-env, before_script]
  tags:
    - kubernetes-parity-build

.rust-info-script:
  script:
    - rustup show
    - cargo --version
    - rustup +nightly show
    - cargo +nightly --version

# collecting vars for pipeline stopper
# they will be used if the job fails
.pipeline-stopper-vars:
  script:
    - echo "Collecting env variables for the cancel-pipeline job"
    - echo "FAILED_JOB_URL=${CI_JOB_URL}" > pipeline-stopper.env
    - echo "FAILED_JOB_NAME=${CI_JOB_NAME}" >> pipeline-stopper.env
    - echo "PR_NUM=${CI_COMMIT_REF_NAME}" >> pipeline-stopper.env

.pipeline-stopper-artifacts:
  artifacts:
    reports:
      dotenv: pipeline-stopper.env

.docker-env:
  image: "${CI_IMAGE}"
  variables:
    FL_FORKLIFT_VERSION: !reference [.forklift, variables, FL_FORKLIFT_VERSION]
  before_script:
    - !reference [.common-before-script, before_script]
    - !reference [.prepare-env, before_script]
    - !reference [.rust-info-script, script]
    - !reference [.forklift-cache, before_script]
  tags:
    - linux-docker

#
.forklift-cache:
  before_script:
    - mkdir ~/.forklift
    - cp $FL_FORKLIFT_CONFIG ~/.forklift/config.toml
    - >
      if [ "$FORKLIFT_BYPASS" != "true" ]; then
        echo "FORKLIFT_BYPASS not set"; 
        if command -v forklift >/dev/null 2>&1; then
          echo "forklift already exists"; 
          forklift version
        else
          echo "forklift does not exist, downloading";
          curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift  -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION}/forklift_${FL_FORKLIFT_VERSION}_linux_amd64"; 
          chmod +x forklift;
          export PATH=$PATH:$(pwd);
          echo ${FL_FORKLIFT_VERSION};
        fi
        echo "Creating alias cargo='forklift cargo'"; 
        shopt -s expand_aliases;
        alias cargo="forklift cargo";
      fi
    #

.common-refs:
  rules:
    - if: $CI_PIPELINE_SOURCE == "web"
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
    - if: $CI_COMMIT_REF_NAME =~ /^gh-readonly-queue.*$/ # merge queues
    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1

.test-pr-refs:
  rules:
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
    - if: $CI_COMMIT_REF_NAME =~ /^gh-readonly-queue.*$/ # merge queues

.publish-gh-pages-refs:
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME == "master"

# handle the specific case where benches could store incorrect bench data because of the downstream staging runs
# exclude cargo-check-benches from such runs
.test-refs-check-benches:
  rules:
    - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "pipeline"  && $CI_IMAGE =~ /staging$/
      when: never
    - if: $CI_PIPELINE_SOURCE == "web"
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
    - if: $CI_COMMIT_REF_NAME =~ /^gh-readonly-queue.*$/ # merge queues
    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1

.test-refs-no-trigger:
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: $CI_PIPELINE_SOURCE == "web"
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
    - if: $CI_COMMIT_REF_NAME =~ /^gh-readonly-queue.*$/ # merge queues
    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
    - if: $CI_COMMIT_REF_NAME =~ /^ci-release-.*$/

.test-refs-no-trigger-prs-only:
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: $CI_PIPELINE_SOURCE == "web"
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
    - if: $CI_COMMIT_REF_NAME =~ /^gh-readonly-queue.*$/ # merge queues

.publish-refs:
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: $CI_PIPELINE_SOURCE == "web"
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1

.build-refs:
  # publish-refs + PRs
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
    - if: $CI_PIPELINE_SOURCE == "web"
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
    - if: $CI_COMMIT_REF_NAME =~ /^gh-readonly-queue.*$/ # merge queues

include:
  # check jobs
  - .gitlab/pipeline/check.yml
  # test jobs
  - .gitlab/pipeline/test.yml
  # build jobs
  - .gitlab/pipeline/build.yml
  # short-benchmarks jobs
  - .gitlab/pipeline/short-benchmarks.yml
  # publish jobs
  - .gitlab/pipeline/publish.yml
  # zombienet jobs
  - .gitlab/pipeline/zombienet.yml
  # ci image
  - project: parity/infrastructure/ci_cd/shared
    ref: main
    file: /common/ci-unified.yml
  - project: parity/infrastructure/ci_cd/shared
    ref: main
    file: /common/forklift.yml
# This job cancels the whole pipeline if any of provided jobs fail.
# In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests
# to fail the pipeline as soon as possible to shorten the feedback loop.
.cancel-pipeline-template:
  stage: .post
  rules:
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
      when: on_failure
  variables:
    PROJECT_ID: "${CI_PROJECT_ID}"
    PROJECT_NAME: "${CI_PROJECT_NAME}"
    PIPELINE_ID: "${CI_PIPELINE_ID}"
    FAILED_JOB_URL: "${FAILED_JOB_URL}"
    FAILED_JOB_NAME: "${FAILED_JOB_NAME}"
    PR_NUM: "${PR_NUM}"
  trigger:
    project: "parity/infrastructure/ci_cd/pipeline-stopper"

remove-cancel-pipeline-message:
  stage: .post
  rules:
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
  variables:
    PROJECT_ID: "${CI_PROJECT_ID}"
    PROJECT_NAME: "${CI_PROJECT_NAME}"
    PIPELINE_ID: "${CI_PIPELINE_ID}"
    FAILED_JOB_URL: "https://gitlab.com"
    FAILED_JOB_NAME: "nope"
    PR_NUM: "${CI_COMMIT_REF_NAME}"
  trigger:
    project: "parity/infrastructure/ci_cd/pipeline-stopper"
# need to copy jobs this way because otherwise gitlab will wait
# for all 3 jobs to finish instead of cancelling if one fails
cancel-pipeline-test-linux-stable1:
  extends: .cancel-pipeline-template
  needs:
    - job: "test-linux-stable 1/3"

cancel-pipeline-test-linux-stable2:
  extends: .cancel-pipeline-template
  needs:
    - job: "test-linux-stable 2/3"

cancel-pipeline-test-linux-stable3:
  extends: .cancel-pipeline-template
  needs:
    - job: "test-linux-stable 3/3"

cancel-pipeline-cargo-check-benches1:
  extends: .cancel-pipeline-template
  needs:
    - job: "cargo-check-benches 1/2"

cancel-pipeline-cargo-check-benches2:
  extends: .cancel-pipeline-template
  needs:
    - job: "cargo-check-benches 2/2"

cancel-pipeline-test-linux-stable-int:
  extends: .cancel-pipeline-template
  needs:
    - job: test-linux-stable-int

cancel-pipeline-cargo-check-each-crate-1:
  extends: .cancel-pipeline-template
  needs:
    - job: "cargo-check-each-crate 1/6"

cancel-pipeline-cargo-check-each-crate-2:
  extends: .cancel-pipeline-template
  needs:
    - job: "cargo-check-each-crate 2/6"

cancel-pipeline-cargo-check-each-crate-3:
  extends: .cancel-pipeline-template
  needs:
    - job: "cargo-check-each-crate 3/6"

cancel-pipeline-cargo-check-each-crate-4:
  extends: .cancel-pipeline-template
  needs:
    - job: "cargo-check-each-crate 4/6"

cancel-pipeline-cargo-check-each-crate-5:
  extends: .cancel-pipeline-template
  needs:
    - job: "cargo-check-each-crate 5/6"

cancel-pipeline-cargo-check-each-crate-6:
  extends: .cancel-pipeline-template
  needs:
    - job: "cargo-check-each-crate 6/6"

cancel-pipeline-cargo-check-each-crate-macos:
  extends: .cancel-pipeline-template
  needs:
    - job: cargo-check-each-crate-macos

cancel-pipeline-check-tracing:
  extends: .cancel-pipeline-template
  needs:
    - job: check-tracing

cancel-pipeline-cargo-clippy:
  extends: .cancel-pipeline-template
  needs:
    - job: cargo-clippy

cancel-pipeline-build-linux-stable:
  extends: .cancel-pipeline-template
  needs:
    - job: build-linux-stable

cancel-pipeline-build-linux-stable-cumulus:
  extends: .cancel-pipeline-template
  needs:
    - job: build-linux-stable-cumulus

cancel-pipeline-build-linux-substrate:
  extends: .cancel-pipeline-template
  needs:
    - job: build-linux-substrate

cancel-pipeline-test-node-metrics:
  extends: .cancel-pipeline-template
  needs:
    - job: test-node-metrics

cancel-pipeline-test-frame-ui:
  extends: .cancel-pipeline-template
  needs:
    - job: test-frame-ui

cancel-pipeline-quick-benchmarks:
  extends: .cancel-pipeline-template
  needs:
    - job: quick-benchmarks

cancel-pipeline-check-try-runtime:
  extends: .cancel-pipeline-template
  needs:
    - job: check-try-runtime

cancel-pipeline-test-frame-examples-compile-to-wasm:
  extends: .cancel-pipeline-template
  needs:
    - job: test-frame-examples-compile-to-wasm

cancel-pipeline-build-short-benchmark:
  extends: .cancel-pipeline-template
  needs:
    - job: build-short-benchmark

cancel-pipeline-check-runtime-migration-rococo:
  extends: .cancel-pipeline-template
  needs:
    - job: check-runtime-migration-rococo

cancel-pipeline-check-runtime-migration-westend:
  extends: .cancel-pipeline-template
  needs:
    - job: check-runtime-migration-westend