main-cron.yml

cargo-cache: &cargo-cache
  id: cache
  key: "v1-cache-{{ id }}-{{ runner.os }}-{{ checksum 'Cargo.lock' }}"
  restore-keys:
    - 'v1-cache-{{ id }}-{{ runner.os }}-'
    - 'v1-cache-{{ id }}-'
  backend: s3
  s3:
    bucket: ci-cache-bucket
  paths:
    - ".cargo/registry/index"
    - ".cargo/registry/cache"
    - ".cargo/git/db"

auto-retry: &auto-retry
  automatic:
    - exit_status: -1  # Agent was lost
      limit: 2
    - exit_status: 255 # Forced agent shutdown
      limit: 2

steps:
  - label: "build"
    command: "ci/scripts/build.sh -t ci-release -p ci-release"
    key: "build"
    plugins:
      - gencer/cache#v2.4.10: *cargo-cache
      - docker-compose#v4.9.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
    timeout_in_minutes: 20
    retry: *auto-retry

  - label: "build other components"
    command: "ci/scripts/build-other.sh"
    key: "build-other"
    plugins:
      - seek-oss/aws-sm#v2.3.1:
          env:
            GITHUB_TOKEN: github-token
      - docker-compose#v4.9.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
          environment:
            - GITHUB_TOKEN
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "build (deterministic simulation)"
    command: "ci/scripts/build-simulation.sh"
    key: "build-simulation"
    plugins:
      - gencer/cache#v2.4.10: *cargo-cache
      - docker-compose#v4.9.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
    timeout_in_minutes: 15
    retry: *auto-retry
  
  - label: "docslt"
    command: "ci/scripts/docslt.sh"
    key: "docslt"
    plugins:
      - gencer/cache#v2.4.10: *cargo-cache
      - docker-compose#v4.9.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "end-to-end test (release)"
    command: "ci/scripts/cron-e2e-test.sh -p ci-release"
    depends_on:
      - "build"
      - "docslt"
    plugins:
      - gencer/cache#v2.4.10: *cargo-cache
      - docker-compose#v4.9.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 40
    retry: *auto-retry

  - label: "end-to-end source test (release)"
    command: "ci/scripts/e2e-source-test.sh -p ci-release"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - gencer/cache#v2.4.10: *cargo-cache
      - docker-compose#v4.9.0:
          run: source-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 15
    retry: *auto-retry

  - label: "unit test"
    command: "ci/scripts/unit-test.sh"
    plugins:
      - ./ci/plugins/swapfile
      - gencer/cache#v2.4.10: *cargo-cache
      - seek-oss/aws-sm#v2.3.1:
          env:
            CODECOV_TOKEN: my-codecov-token
      - docker-compose#v4.9.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          environment:
            - CODECOV_TOKEN
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "fuzz test"
    command: "ci/scripts/cron-fuzz-test.sh -p ci-release"
    depends_on:
      - "build"
    plugins:
      - ./ci/plugins/swapfile
      - gencer/cache#v2.4.10: *cargo-cache
      - docker-compose#v4.9.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
          environment:
            - RW_RANDOM_SEED_SQLSMITH: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 20
    retry: *auto-retry

  - label: "unit test (deterministic simulation)"
    command: "ci/scripts/deterministic-unit-test.sh"
    plugins:
      - gencer/cache#v2.4.10: *cargo-cache
      - docker-compose#v4.9.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "scaling test (deterministic simulation)"
    command: "MADSIM_TEST_NUM=10 ci/scripts/deterministic-scale-test.sh"
    depends_on: "build-simulation"
    plugins:
      - gencer/cache#v2.4.10: *cargo-cache
      - docker-compose#v4.9.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
    timeout_in_minutes: 30
    retry: *auto-retry

  - label: "end-to-end test (deterministic simulation)"
    command: "timeout 14m ci/scripts/deterministic-e2e-test.sh"
    depends_on: "build-simulation"
    plugins:
      - gencer/cache#v2.4.10: *cargo-cache
      - docker-compose#v4.9.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 15
    retry: *auto-retry

  - label: "recovery test (deterministic simulation)"
    command: "KILL_RATE=1.0 ci/scripts/deterministic-recovery-test.sh"
    depends_on: "build-simulation"
    plugins:
      - gencer/cache#v2.4.10: *cargo-cache
      - docker-compose#v4.9.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 30
    retry: *auto-retry

  - label: "misc check"
    command: "ci/scripts/misc-check.sh"
    plugins:
      - gencer/cache#v2.4.10: *cargo-cache
      - docker-compose#v4.9.0:
          run: rw-build-env
          config: ci/docker-compose.yml
      - shellcheck#v1.2.0:
          files: ./**/*.sh
    timeout_in_minutes: 5
    retry: *auto-retry