reusable-prober.yml

name: Reusable Sigstore Prober Workflow

on:
  workflow_call:
    secrets:
      PAGERDUTY_INTEGRATION_KEY:
        description: 'Integration key for PagerDuty'
        required: true
    inputs:
      rekor_url:
        required: false
        type: string
        default: 'https://rekor.sigstore.dev'
        description: 'Rekor URL'
      fulcio_url:
        required: false
        type: string
        default: 'https://fulcio.sigstore.dev'
        description: 'Fulcio URL'
      oidc_url:
        required: false
        type: string
        default: 'https://oauth2.sigstore.dev/auth'
        description: 'OIDC URL'
      enable_staging:
        required: false
        type: boolean
      tuf_repo:
        required: false
        type: string
        default: 'https://tuf-repo-cdn.sigstore.dev'
        description: 'TUF Repo'
      tuf_preprod_repo:
        required: false
        type: string
        default: 'https://tuf-preprod-repo-cdn.sigstore.dev'
        description: 'TUF Repo'
      tuf_root_path:
        required: false
        type: string
        default: "root.json"
        description: "path to the tuf root"
      tuf_root_url:
        required: false
        type: string
      triggerPagerDutyTest:
        description: 'Trigger PagerDuty test message'
        required: false
        type: string
      github_issues_repo:
        required: false
        type: string
        default: 'sigstore/sigstore-probers'
        description: 'GitHub repo to file alert issues under'

permissions:
  contents: read
  id-token: write
jobs:

  sigstore-probe:
    runs-on: ubuntu-latest
    timeout-minutes: 10

    outputs:
      sigstore_probe: ${{ steps.msg.outputs.sigstore_probe }}
    steps:
      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
      - name: Set up Go
        id: setup-go
        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v3.3.0
        with:
          go-version-file: 'prober/hack/toolz/go.mod'
          check-latest: true
          cache: false

      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-${{ steps.setup-go.outputs.go-version }}-${{ github.job }}-${{ hashFiles('prober/hack/toolz/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-${{ steps.setup-go.outputs.go-version }}-${{ github.job }}-

      - name: Install 'prober' from sigstore/scaffolding
        run: |
          make -C prober/ prober
          echo "PATH=$PATH:$PWD/prober/hack/toolz/bin" >> $GITHUB_ENV

      # Make sure rekor is up and we can get root info
      - name: Run prober
        env:
          DEBUG: 1
        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
        with:
          timeout_minutes: 3
          max_attempts: 3
          retry_wait_seconds: 60
          retry_on: error
          command: prober --one-time --rekor-url ${{ inputs.rekor_url }} --fulcio-url ${{ inputs.fulcio_url }}

      - name: Set messages
        id: msg
        if: success() || failure()
        run: |
          echo "sigstore_probe=good" >> $GITHUB_OUTPUT
          if [ "${{ job.status }}" == 'failure' ]; then echo "sigstore_probe=failure" >> $GITHUB_OUTPUT; fi


  root-probe:
    timeout-minutes: 10
    runs-on: ubuntu-latest
    outputs:
      root_state: ${{ steps.msg.outputs.root_state }}
      dedup_key: ${{ steps.check.outputs.dedup_key }}
    steps:
      - name: Checkout sigstore-probers code
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
      - name: Checkout root-signing to get roots
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: sigstore/root-signing
          ref: main
          path: root-signing
      - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v3.3.0
        id: setup-go
        with:
          go-version-file: 'prober/hack/toolz/go.mod'
          check-latest: true
          cache: false

      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-${{ steps.setup-go.outputs.go-version }}-${{ github.job }}-${{ hashFiles('prober/hack/toolz/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-${{ steps.setup-go.outputs.go-version }}-${{ github.job }}-

      - name: Install 'verify' tool from sigstore/root-signing
        run: |
          make -C prober/ verify
          echo "PATH=$PATH:$PWD/prober/hack/toolz/bin" >> $GITHUB_ENV

      - name: Check expiration
        id: check
        run: |
          if [ "${{ inputs.tuf_root_url }}" != "" ]; then
            echo "::warning::fetching root.json from ${{ inputs.tuf_root_url }}"
            curl -Lo root.json --user-agent "sigstore-prober" ${{ inputs.tuf_root_url }}
            export ROOT_PATH=root.json
          else
            export ROOT_PATH=$GITHUB_WORKSPACE/root-signing/${{ inputs.tuf_root_path }}
          fi
          export PD_DEDUP_KEY_FILE=$(mktemp)

          echo "::group::tuf-repo-cdn 2 day check"
          export EXPIRY=$(date -d '+2 days' '+%s')
          verify repository --repository ${{ inputs.tuf_repo }} --root ${ROOT_PATH} --valid-until ${EXPIRY} --dedup-key-file ${PD_DEDUP_KEY_FILE} || (echo "dedup_key=$(cat ${PD_DEDUP_KEY_FILE})" >> $GITHUB_OUTPUT ; exit 1)
          echo "::endgroup::"

          echo "::group::tuf-repo-cdn 15 day check"
          export EXPIRY=$(date -d '+15 days' '+%s')
          echo "Verifying root valid within 15 days..."
          verify repository --repository ${{ inputs.tuf_repo }} --root ${ROOT_PATH} --valid-until ${EXPIRY} --role root.json --role targets.json --dedup-key-file ${PD_DEDUP_KEY_FILE} || (echo "dedup_key=$(cat ${PD_DEDUP_KEY_FILE})" >> $GITHUB_OUTPUT ; exit 1)
          echo "::endgroup::"

          # For preprod/staging TUF bucket
          echo "::group::tuf-preprod-repo-cdn 2 day check"
          export EXPIRY=$(date -d '+2 days' '+%s')
          verify repository --repository ${{ inputs.tuf_preprod_repo }} --root ${ROOT_PATH} --valid-until ${EXPIRY} --dedup-key-file ${PD_DEDUP_KEY_FILE} || (echo "dedup_key=$(cat ${PD_DEDUP_KEY_FILE})" >> $GITHUB_OUTPUT ; exit 1)
          echo "::endgroup::"

          echo "::group::tuf-preprod-repo-cdn 15 day check"
          export EXPIRY=$(date -d '+15 days' '+%s')
          echo "Verifying root valid within 15 days..."
          verify repository --repository ${{ inputs.tuf_preprod_repo }} --root ${ROOT_PATH} --valid-until ${EXPIRY} --role root.json --role targets.json --dedup-key-file ${PD_DEDUP_KEY_FILE} || (echo "dedup_key=$(cat ${PD_DEDUP_KEY_FILE})" >> $GITHUB_OUTPUT ; exit 1)
          echo "::endgroup::"

      - name: Set messages
        id: msg
        if: success() || failure()
        run: |
          echo "root_state=good" >> $GITHUB_OUTPUT
          if [ "${{ job.status }}" == 'failure' ]; then echo "root_state=failure" >> $GITHUB_OUTPUT; fi

  rekor-fulcio-e2e:
    timeout-minutes: 10
    permissions:
      id-token: write
      contents: read
    env:
      COSIGN_YES: "true"
      GIT_HASH: ${{ github.sha }}
      GIT_VERSION: unstable
      GITHUB_RUN_ID: ${{ github.run_id }}
      GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
      IMAGE: localhost:1338/image:${{ github.sha }}-${{ github.run_id }}
      REKOR_SERVER: ${{ inputs.rekor_url }}
    runs-on: ubuntu-latest
    outputs:
      rekor_fulcio_e2e: ${{ steps.msg.outputs.rekor_fulcio_e2e }}
      skip_pagerduty: ${{ steps.set-skip-pagerduty.outputs.skip_pagerduty }}
    steps:
      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2

      # This server is often down, resulting in a lot of flaky probers
      # If the server is down, and this step fails, we don't alert PagerDuty
      - name: Confirm Github OIDC Server is Available
        continue-on-error: true
        run: |
          curl -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" $ACTIONS_ID_TOKEN_REQUEST_URL&audience=sigstore

      # Since the server is down, we want to ignore the failure in this workflow
      # and skip paging PagerDuty
      - name: Set skip_pagerduty outputs
        id: set-skip-pagerduty
        if: failure()
        run: |
          echo "skip_pagerduty=true" >> $GITHUB_OUTPUT

      - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v3.3.0
        id: setup-go
        with:
          go-version-file: 'prober/hack/toolz/go.mod'
          check-latest: true
          cache: false

      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-${{ steps.setup-go.outputs.go-version }}-${{ github.job }}-${{ hashFiles('prober/hack/toolz/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-${{ steps.setup-go.outputs.go-version }}-${{ github.job }}-

      # Install crane / rekor-cli / cosign tools
      - name: Install (crane, rekor-cli, cosign) tools
        run: |
          make -C prober/ crane rekor-cli cosign
          echo "PATH=$PATH:$PWD/prober/hack/toolz/bin" >> $GITHUB_ENV

      # Setup the registry on port 1338
      - uses: chainguard-dev/actions/setup-registry@main

      - name: Build and copy a container image
        continue-on-error: true
        run: |
          for i in {1..5}
          do
            if crane cp busybox@sha256:d2b53584f580310186df7a2055ce3ff83cc0df6caacf1e3489bff8cf5d0af5d8 ${IMAGE}; then
              echo "Successfully copied image" && exit 0
            else
              echo "Failed to copy image ${IMAGE}" && sleep 10
            fi
          done
          exit 1

      # START: PREPRODUCTION VERIFICATION
      # TODO: Create a matrix (https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs)
      #       to reduce duplication
      # Test the preproduction bucket only for prod

      # Test that initialize works from prod to preprod
      - name: Initialize prod TUF root
        if: ${{ inputs.enable_staging == false }}
        run: |
          for i in {1..5}
          do
            if cosign initialize; then
              echo "Successfully initialized" && exit 0
            else
              echo "Failed to initialize" && sleep 10
            fi
          done

      - name: Initialize preprod TUF root
        if: ${{ inputs.enable_staging == false }}
        run: |
          curl -Lo root.json --user-agent "sigstore-prober" ${{ inputs.tuf_preprod_repo }}/root.json
          for i in {1..5}
          do
            if cosign initialize --mirror=${{ inputs.tuf_preprod_repo }} --root=root.json; then
              echo "Successfully initialized" && exit 0
            else
              echo "Failed to initialize" && sleep 10
            fi
          done

      # Test signing in preproduction
      - name: Sign and verify the image with preprod TUF
        if: ${{ inputs.enable_staging == false }}
        run: |
          cosign sign --yes ${IMAGE} --rekor-url  ${{ inputs.rekor_url }} --fulcio-url ${{ inputs.fulcio_url }} --oidc-issuer ${{ inputs.oidc_url }} --oidc-provider github-actions
          cosign verify ${IMAGE} --rekor-url  ${{ inputs.rekor_url }} --certificate-oidc-issuer=https://token.actions.githubusercontent.com --certificate-identity-regexp='https://github.com/sigstore/sigstore-probers/.github/workflows/reusable-prober.yml@refs/.*'

      - name: Remove preprod TUF
        if: ${{ inputs.enable_staging == false }}
        run: |
          rm -rf ~/.sigstore
      # END: PREPRODUCTION VERIFICATION

      - name: Initialize prod TUF root
        if: ${{ inputs.enable_staging == false }}
        run: |
          for i in {1..5}
          do
            if cosign initialize; then
              echo "Successfully initialized" && exit 0
            else
              echo "Failed to initialize" && sleep 10
            fi
          done

      - name: Initialize staging TUF root
        if: ${{ inputs.enable_staging }}
        run: |
          curl -Lo root.json ${{ inputs.tuf_root_url }}
          for i in {1..5}
          do
            if cosign initialize --mirror=${{ inputs.tuf_repo }} --root=root.json; then
              echo "Successfully initialized" && exit 0
            else
              echo "Failed to initialize" && sleep 10
            fi
          done
          exit 1

      - name: Sign and verify the image
        run: |
          cosign sign --yes ${IMAGE} --rekor-url  ${{ inputs.rekor_url }} --fulcio-url ${{ inputs.fulcio_url }} --oidc-issuer ${{ inputs.oidc_url }} --oidc-provider github-actions
          cosign verify ${IMAGE} --rekor-url  ${{ inputs.rekor_url }} --certificate-oidc-issuer=https://token.actions.githubusercontent.com --certificate-identity-regexp='https://github.com/sigstore/sigstore-probers/.github/workflows/reusable-prober.yml@refs/.*'

      - name: Generate and upload attestation
        run: |
          cosign attest --predicate ./prober/attestation.json --type slsaprovenance --rekor-url  ${{ inputs.rekor_url }} --fulcio-url ${{ inputs.fulcio_url }} --oidc-issuer ${{ inputs.oidc_url }} ${IMAGE}
          cosign verify-attestation --rekor-url  ${{ inputs.rekor_url }} --type=slsaprovenance ${IMAGE} --certificate-oidc-issuer=https://token.actions.githubusercontent.com --certificate-identity-regexp='https://github.com/sigstore/sigstore-probers/.github/workflows/reusable-prober.yml@refs/.*'

      - name: Set messages
        id: msg
        if: success() || failure()
        run: |
          echo "rekor_fulcio_e2e=good" >> $GITHUB_OUTPUT
          if [ "${{ job.status }}" == 'failure' ]; then echo "rekor_fulcio_e2e=failure" >> $GITHUB_OUTPUT; fi

  compute-summary-msg:
    runs-on: ubuntu-latest
    outputs:
      summary: ${{ steps.msg.outputs.summary }}
      group: ${{ steps.msg.outputs.group }}
    steps:
      - name: Set messages
        id: msg
        if: success() || failure()
        run: |
          if [ "${{ inputs.triggerPagerDutyTest }}" == "true" ]; then
            echo "summary=Test Notification" >> $GITHUB_OUTPUT
          else
            echo "summary=Prober Failed" >> $GITHUB_OUTPUT
          fi
          echo "group=production" >> $GITHUB_OUTPUT
          if [ ${{ inputs.enable_staging }} == 'true' ]; then
            echo "group=staging" >> $GITHUB_OUTPUT;
          fi

  pagerduty-notification:
    if: github.event.inputs.triggerPagerDutyTest=='true' || (failure() && needs.rekor-fulcio-e2e.outputs.skip_pagerduty != 'true')
    needs: [sigstore-probe, root-probe, rekor-fulcio-e2e, compute-summary-msg]
    uses: ./.github/workflows/reusable-pager.yml
    secrets:
      PAGERDUTY_INTEGRATION_KEY: ${{ secrets.PAGERDUTY_INTEGRATION_KEY }}
    with:
      summary: ${{ needs.compute-summary-msg.outputs.summary }}
      component: "health prober"
      group: ${{ needs.compute-summary-msg.outputs.group }}
      dedup_key: "${{ needs.root-probe.outputs.dedup_key }}"
      details: >
        {
          "Environment": "${{ needs.compute-summary-msg.outputs.group }}",
          "Failure URL": "https://github.com/sigstore/sigstore-probers/actions/runs/${{ github.run_id }}",
          "Commit": "${{ github.sha }}",
          "Prober": "${{ needs.sigstore-probe.outputs.sigstore_probe }}",
          "GCS Root": "${{ needs.root-probe.outputs.root_state }}",
          "Rekor Fulcio E2E Test": "${{ needs.rekor-fulcio-e2e.outputs.rekor_fulcio_e2e }}"
        }
      links: >
        [
          {
            "href": "https://github.com/sigstore/public-good-instance/blob/main/playbooks/alerting/alerts/k8s-api-endpoint-prober.md",
            "text": "Prober Failure Playbook"
          },
          {
            "href": "https://github.com/sigstore/public-good-instance/blob/main/playbooks/alerting/tuf.md",
            "text": "Expiring TUF Metadata Playbook"
          }
        ]

  github-issue:
    if: always() && (needs.sigstore-probe.result == 'failure' || needs.root-probe.result == 'failure' || needs.rekor-fulcio-e2e.result == 'failure')
    runs-on: ubuntu-latest
    needs: [sigstore-probe, root-probe, rekor-fulcio-e2e, compute-summary-msg]
    steps:
      - env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          cat << 'EOF' | gh issue create --repo ${{ inputs.github_issues_repo }} --title "[ALERT][${{ needs.compute-summary-msg.outputs.group }}] Prober Failed" --label "alert,needs-assignee,oncall,${{ inputs.enable_staging && 'env:staging' || 'env:production' }}" -F -
          Environment: ${{ needs.compute-summary-msg.outputs.group }}
          Failure URL: https://github.com/sigstore/sigstore-probers/actions/runs/${{ github.run_id }}
          Commit: ${{ github.sha }}
          Prober: ${{ needs.sigstore-probe.outputs.sigstore_probe }}
          GCS Root: ${{ needs.root-probe.outputs.root_state }}
          Rekor Fulcio E2E Test: ${{ needs.rekor-fulcio-e2e.outputs.rekor_fulcio_e2e }}
          EOF