/
.one-pipeline.yaml
300 lines (250 loc) · 11.5 KB
/
.one-pipeline.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
version: '1'
setup:
image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
script: |
#!/usr/bin/env bash
SKIP_SETUP=$(get_env SKIP_SETUP)
if [[ "$SKIP_SETUP" != 1 ]]; then
GHE_TOKEN=$(cat "$WORKSPACE/git-token")
GHE_TOKEN=$(get_env git-token)
GH_TOKEN=$(get_env git-token-GH)
OWNER=$(jq -r '.services[] | select(.toolchain_binding.name=="app-repo") | .parameters.owner_id' /toolchain/toolchain.json)
REPO=$(jq -r '.services[] | select(.toolchain_binding.name=="app-repo") | .parameters.repo_name' /toolchain/toolchain.json)
REPO=${REPO%.git}
REPO=sample-getting-started
BRANCH=$(get_env branch)
# The 'owner' is really the org, not the user with admin access to repo
OWNER=OpenLiberty
echo "Owner: $OWNER"
echo "REPO: $REPO"
echo "BRANCH: $BRANCH"
curl -u :$GH_TOKEN https://api.github.com/repos/$OWNER/$REPO/branches/$BRANCH/protection -H "Accept: application/vnd.github.v3+json" -X PUT -d '{"required_pull_request_reviews":{"dismiss_stale_reviews":true,"required_approving_review_count":1},"enforce_admins":null,"restrictions":null,"required_status_checks":null}'
// Update repo with Whitesource enabled
WHITESOURCE_GHE_REPO=$(get_env WHITESOURCE_GHE_REPO)
WHITESOURCE_GHE_DOMAIN=$(get_env WHITESOURCE_GHE_DOMAIN)
echo "git push --prune https://$GHE_TOKEN@$WHITESOURCE_GHE_DOMAIN/$WHITESOURCE_GHE_REPO +refs/remotes/origin/$BRANCH:refs/heads/$BRANCH +refs/tags/*:refs/tags/*"
git push --prune https://$GHE_TOKEN@$WHITESOURCE_GHE_DOMAIN/$WHITESOURCE_GHE_REPO +refs/remotes/origin/$BRANCH:refs/heads/$BRANCH +refs/tags/*:refs/tags/*
fi
test:
abort_on_failure: true
image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
script: |
#!/usr/bin/env bash
SKIP_UNIT_TESTS=$(get_env SKIP_UNIT_TESTS)
if [[ "$SKIP_UNIT_TESTS" != 1 ]]; then
cd ../"$(load_repo app-repo path)"
mvn clean package liberty:create liberty:install-feature liberty:test-start liberty:deploy failsafe:integration-test failsafe:verify liberty:test-stop
fi
static-scan:
abort_on_failure: false
dind: true
image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
script: |
#!/usr/bin/env bash
SKIP_STATIC_SCAN=$(get_env SKIP_STATIC_SCAN)
if [[ "$SKIP_STATIC_SCAN" != 1 ]]; then
read -r SONAR_HOST_URL <<< "$(get_env sonarqube | jq -r '.parameters.dashboard_url' | sed 's:/*$::')"
read -r SONAR_USER <<< "$(get_env sonarqube | jq -r '.parameters.user_login')"
SONARQUBE_INSTANCE_ID=$(get_env sonarqube | jq -r '.instance_id')
read -r SONAR_PASS <<< "$(jq -r --arg sonar_instance "$SONARQUBE_INSTANCE_ID" '[.services[] | select(."service_id"=="sonarqube")][] | select(."instance_id"==$sonar_instance) | .parameters.user_password' /toolchain/toolchain.json)"
touch "$WORKSPACE"/liberty-getting-started/sonar-project.properties
cat << EOF > "$WORKSPACE"/liberty-getting-started/sonar-project.properties
sonar.projectKey=liberty-getting-started
sonar.host.url=$SONAR_HOST_URL
sonar.sources=.
sonar.login=$SONAR_USER
sonar.password=$SONAR_PASS
sonar.c.file.suffixes=-
sonar.cpp.file.suffixes=-
sonar.objc.file.suffixes=-
EOF
chmod -x "$WORKSPACE"/liberty-getting-started/sonar-project.properties
#echo "$SONAR_PASS" >> /tmp/sonarqube-token
"${ONE_PIPELINE_PATH}"/internal/sonarqube/sonarqube_run
fi
containerize:
abort_on_failure: true
dind: true
image: icr.io/continuous-delivery/pipeline/pipeline-base-image:2.12@sha256:ff4053b0bca784d6d105fee1d008cfb20db206011453071e86b69ca3fde706a4
script: |
#!/usr/bin/env bash
if [[ "$PIPELINE_DEBUG" == 1 ]]; then
trap env EXIT
env
set -x
fi
apt-get update
apt-get -qq -y install build-essential software-properties-common uidmap
# Download and install skopeo
if ! command -v skopeo &> /dev/null; then
if [ ! -f "/apt/sources.list.d/devel:kubic:libcontainers:stable.list" ]; then
sudo sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_18.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list"
wget --no-check-certificate -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_18.04/Release.key -O- | sudo apt-key add -
sudo apt-get update -qq
fi
sudo apt-get -y install skopeo
else
skopeo --version
fi
# Build images
export PIPELINE_USERNAME=$(get_env ibmcloud-api-user)
export PIPELINE_PASSWORD=$(get_env ibmcloud-api-key-staging)
export CONTAINER_REGISTRY=$(get_env CONTAINER_REGISTRY "stg.icr.io")
REGISTRY_REPO=$(get_env REGISTRY_REPO)
IMAGE_TAG=$(get_env IMAGE_TAG)
IMAGE=$CONTAINER_REGISTRY/$REGISTRY_REPO:$IMAGE_TAG
BRANCH=$(get_env branch)
W3_USERNAME=$(get_env w3_username)
W3_PASSWORD=$(get_env w3_password)
export arch=$(get_env architecture)
# build P and Z
if [[ "$arch" == "ZXP" ]]; then
echo "<ciorchestrator> Sending request to build P and Z"
./scripts/pipeline/request-ciorchestrator.sh --command "make build-app-pipeline IMAGE=${IMAGE}" --user "$W3_USERNAME" --password "$W3_PASSWORD" --branch "$BRANCH" --repository "sample-getting-started" --org "OpenLiberty" --trigger "sample-liberty" --configFile ".ci-orchestrator/sample-liberty-build.yml"
pipelineid=$(cat ciorchestrator-submit.id)
fi
# Build X
make build-app-pipeline IMAGE=${IMAGE}
#wait on build for P and Z
if [[ "$arch" == "ZXP" ]]; then
# wait for build ppc64le and s390x images
echo "<ciorchestrator> waiting on request to build P and Z"
./scripts/pipeline/await-ciorchestrator.sh --user "$W3_USERNAME" --password "$W3_PASSWORD" --pipelineId "$pipelineid"
fi
# Build manifest
make build-manifest-pipeline IMAGE=${IMAGE}
# Save artifacts
echo "**** Saving Artifacts ****"
if [[ "$arch" == "ZXP" ]]; then
declare -a tags=("${IMAGE_TAG}" "${IMAGE_TAG}-amd64" "${IMAGE_TAG}-ppc64le" "${IMAGE_TAG}-s390x")
else
declare -a tags=("${IMAGE_TAG}" "${IMAGE_TAG}-amd64")
fi
for i in "${tags[@]}"
do
IMAGE=$CONTAINER_REGISTRY/$REGISTRY_REPO:$i
DIGEST="$(skopeo inspect docker://$IMAGE | grep Digest | grep -o 'sha[^\"]*')"
{ ARCH="$(echo $i | grep -o '\(amd64\|s390x\|ppc64le\)$')" && TYPE="image"; } || { TYPE="manifest"; }
if [[ "$TYPE" == "manifest" ]]; then
echo "Saving artifact $i type=$TYPE name=$IMAGE digest=$DIGEST"
save_artifact $i type=$TYPE name="$IMAGE" "digest=$DIGEST"
else
echo "Saving artifact $i type=$TYPE name=$IMAGE digest=$DIGEST arch=$ARCH"
save_artifact $i type=$TYPE name="$IMAGE" "digest=$DIGEST" "arch=$ARCH"
fi
done
sign-artifact:
abort_on_failure: false
image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
script: |
#!/usr/bin/env bash
echo "Skipping step because it doesn't make sense to sign an image at this step in the flow"
deploy:
image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
script: |
#!/usr/bin/env bash
if [[ "$PIPELINE_DEBUG" == 1 ]]; then
trap env EXIT
env
set -x
fi
echo "Skipping Deploy"
dynamic-scan:
abort_on_failure: false
image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
script: |
#!/usr/bin/env bash
echo "Skipping dynamic-scan"
acceptance-test:
abort_on_failure: true
dind: true
image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
script: |
#!/usr/bin/env bash
if [[ "$PIPELINE_DEBUG" == 1 ]]; then
trap env EXIT
env
set -x
fi
export PIPELINE_USERNAME=$(get_env ibmcloud-api-user)
export PIPELINE_PASSWORD=$(get_env ibmcloud-api-key-staging)
export CONTAINER_REGISTRY=$(get_env CONTAINER_REGISTRY "stg.icr.io")
REGISTRY_REPO=$(get_env REGISTRY_REPO)
IMAGE_TAG=$(get_env IMAGE_TAG)
IMAGE=$CONTAINER_REGISTRY/$REGISTRY_REPO:$IMAGE_TAG
BRANCH=$(get_env branch)
W3_USERNAME=$(get_env w3_username)
W3_PASSWORD=$(get_env w3_password)
export arch=$(get_env architecture)
# check P and Z
if [[ "$arch" == "ZXP" ]]; then
echo "<ciorchestrator> Sending request to build P and Z"
./scripts/pipeline/request-ciorchestrator.sh --command "make check-build IMAGE=${IMAGE}" --user "$W3_USERNAME" --password "$W3_PASSWORD" --branch "$BRANCH" --repository "sample-getting-started" --org "OpenLiberty" --trigger "sample-liberty" --configFile ".ci-orchestrator/sample-liberty-build.yml"
pipelineid=$(cat ciorchestrator-submit.id)
fi
# Check X
make check-build IMAGE=${IMAGE}
#wait on build for P and Z
if [[ "$arch" == "ZXP" ]]; then
# wait for build ppc64le and s390x images
echo "<ciorchestrator> waiting on request to build P and Z"
./scripts/pipeline/await-ciorchestrator.sh --user "$W3_USERNAME" --password "$W3_PASSWORD" --pipelineId "$pipelineid"
fi
scan-artifact:
abort_on_failure: true
image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
script: |
#!/usr/bin/env bash
echo $STAGE
# ========== Security Scanner ==========
./scripts/pipeline/ci_to_secure_pipeline_scan.sh
release:
abort_on_failure: false
dind: true
image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
script: |
#!/usr/bin/env bash
if [[ "$PIPELINE_DEBUG" == 1 ]]; then
trap env EXIT
env
set -x
fi
PUBLISH_IMAGES=$(get_env PUBLISH_IMAGES)
if [[ "$PUBLISH_IMAGES" == 1 ]]; then
skopeo --version
IMAGE_TAG=$(get_env IMAGE_TAG)
export PIPELINE_USERNAME=$(get_env ibmcloud-api-user)
PIPELINE_STAGING_PASSWORD=$(get_env ibmcloud-api-key-staging)
CONTAINER_REGISTRY=$(get_env CONTAINER_REGISTRY "stg.icr.io")
REGISTRY_REPO=$(get_env REGISTRY_REPO)
STAGING_IMAGE=$CONTAINER_REGISTRY/$REGISTRY_REPO:$IMAGE_TAG
PIPELINE_PROD_PASSWORD=$(get_env ibmcloud-api-key)
PROD_CONTAINER_REGISTRY=$(get_env PROD_CONTAINER_REGISTRY)
PROD_REGISTRY_REPO=$(get_env PROD_REGISTRY_REPO)
PROD_IMAGE=$PROD_CONTAINER_REGISTRY/$PROD_REGISTRY_REPO:$IMAGE_TAG
# Copy image digests and manifest list to prod
echo $PIPELINE_STAGING_PASSWORD | docker login $CONTAINER_REGISTRY -u "$PIPELINE_USERNAME" --password-stdin
echo $PIPELINE_PROD_PASSWORD | docker login $PROD_CONTAINER_REGISTRY -u "$PIPELINE_USERNAME" --password-stdin
echo "Copying staged image $STAGING_IMAGE to prod at $PROD_IMAGE as user $PIPELINE_USERNAME"
skopeo copy --src-creds $PIPELINE_USERNAME:$PIPELINE_STAGING_PASSWORD --dest-creds $PIPELINE_USERNAME:$PIPELINE_PROD_PASSWORD --all docker://$STAGING_IMAGE docker://$PROD_IMAGE
echo "Manifest list of $PROD_IMAGE"
docker manifest inspect "$PROD_IMAGE"
digest="$(skopeo inspect docker://$PROD_IMAGE | grep Digest | grep -o 'sha[^\"]*')"
echo "$digest"
export CONTAINER_REGISTRY=$PROD_CONTAINER_REGISTRY
export PIPELINE_PASSWORD=$PIPELINE_PROD_PASSWORD
# Check X image from prod registry
make check-build IMAGE=${PROD_IMAGE}
fi
owasp-zap-api:
dind: true
abort_on_failure: false
image: icr.io/continuous-delivery/pipeline/pipeline-base-ubi:3.12
script: |
#!/usr/bin/env bash
if [[ "$PIPELINE_DEBUG" == 1 ]]; then
trap env EXIT
env
set -x
fi
echo "Skipping OWASP ZAP API"