142 lines (130 loc) · 6.76 KB
/
sync-main-to-preprod-and-prod.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
#
# Copyright 2021 The Sigstore Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This sync will execute when any combination of the following files,
# and no other files, are changed on the main branch:
# - timestamp.json
# - snapshot.json
# - [0-9]+.snapshot.json
# Under this condition, all files from the repository/repository directory
# on the main branch will sync to both preprod and prod.
name: Sync Repository Main Branch with both GCS Preprod and Prod Buckets
on:
push:
branches:
- main
paths:
# When timestamp or snapshot files are changed.
# Note: the sync job below uses a diff to ensure ONLY these files are changed
# prior to syncing.
- 'repository/repository/timestamp.json'
- 'repository/repository/snapshot.json'
- 'repository/repository/[0-9]+.snapshot.json'
workflow_dispatch:
jobs:
sync:
runs-on: ubuntu-latest
permissions:
id-token: 'write'
steps:
- uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
with:
fetch-depth: 0
- uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
with:
go-version-file: './go.mod'
check-latest: true
# Setup OIDC->SA auth
- uses: google-github-actions/auth@55bd3a7c6e2ae7cf1877fd1ccb9d54c0503c457c # v2.1.2
id: auth
with:
token_format: 'access_token'
workload_identity_provider: 'projects/306323169285/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider'
service_account: 'tuf-gha@project-rekor.iam.gserviceaccount.com'
create_credentials_file: true
- uses: google-github-actions/setup-gcloud@98ddc00a17442e89a24bbf282954a3b65ce6d200 # v2.1.0
with:
project_id: project-rekor
- name: Login
run: |
gcloud auth login --brief --cred-file="${{ steps.auth.outputs.credentials_file_path }}"
gcloud auth list
# Sync
- name: sync
run: |
check_expiration() {
expiry=$(jq -r '.signed.expires' $1)
expires=$(date -d $expiry +%s)
current=$(date +%s)
if (( expires < current )); then
echo "Detected expired metadata file $1 at $expiry!"
exit 1
fi;
}
# Checks whether a filename matches timestamp.json, snapshot.json, or [0-9]+.snapshot.json. If not,
# this workflow will exit as we only want to run it when ONLY these files are changed.
# TODO it may be good to check whether the [0-9]+.snapshot.json is the next one chronologically
check_filename() {
if [[ $1 != "timestamp.json" && $1 != "snapshot.json" && !($1 =~ ^[0-9]+\.snapshot.json$) ]]; then
echo "Sync main to preprod and prod workflow: Files other than timestamp and snapshot were updated in main branch, including file: $1. Not syncing, exiting."
exit 0
fi;
}
# Download bucket metadata
gcloud --quiet storage cp -r gs://sigstore-tuf-root/ .
# Diff main and prod to determine whether ONLY the timestamp and snapshot files have changed in main.
# If other files have also changed, exit - in this case, the sync should be to preprod only.
# NOTE other non-timestamp/snapshot changes should only occur during a ceremony, and
# will go through the sync-ceremony-* flow that hits main and preprod. This means there should never
# be changed files in prod that have not also hit the main branch.
# NOTE We deliberately diff only with prod to avoid a scenario in which the ceremony branch's sync to
# main and preprod kicks off this workflow, in which case the lack of diff between main and preprod
# could trigger this workflow to auto sync to prod.
#
# TODO this does not check whether the updates are in main or in prod, only that files differ. We could
# make this more exact later to check that the updates are in main (anything else is unexpected).
diff -qr repository/repository sigstore-tuf-root | grep -Po '([0-9\.]*\w+[\.\w+]*(?= differ))|((Only in \w+\: )\K(.*))' | while read l; do check_filename $l; done
# Upload all but TUF timestamp. Once timestamp is uploaded, all other files must have been uploaded.
for f in $(ls repository/repository/ -I *timestamp.json)
do
# Check for expiration if this is a non-versioned metadata file.
# Versioned metadata like 1.root.json may be expired.
# TODO(asraa): When consistent snapshots are enabled, this logic must be changed so that
# only old versioned metadata can be expired.
if [[ $f == [^0-9]*.json ]]; then
check_expiration repository/repository/$f
fi;
gcloud --quiet storage cp --cache-control=no-store -r repository/repository/$f gs://sigstore-preprod-tuf-root/
gcloud --quiet storage cp --cache-control=no-store -r repository/repository/$f gs://sigstore-tuf-root/
done
# Upload timestamp after checking latest timestamp expiration
check_expiration repository/repository/timestamp.json
gcloud --quiet storage cp --cache-control=no-store -r repository/repository/*timestamp.json gs://sigstore-preprod-tuf-root/
gcloud --quiet storage cp --cache-control=no-store -r repository/repository/*timestamp.json gs://sigstore-tuf-root/
# NOTE as this workflow runs only when timestamp or snapshot files are added or updated, there should not
# be a scenario where files that are removed from main must be synced to (removed from) preprod/prod.
gcloud compute url-maps invalidate-cdn-cache tuf-preprod-repo-cdn-lb --path "/*" --async
gcloud compute url-maps invalidate-cdn-cache tuf-repo-cdn-lb --path "/*" --async
if-failed:
runs-on: ubuntu-latest
needs: [sync]
permissions:
issues: 'write'
actions: 'read'
if: always() && needs.sync.result == 'failure'
steps:
- name: Open issue or add comment on failure
uses: sigstore/sigstore-probers/.github/actions/open-workflow-issue@main
with:
comment_for_each_failure: true