forked from googleapis/google-cloud-go
/
proftest.go
639 lines (571 loc) · 22.5 KB
/
proftest.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Package proftest contains test helpers for profiler agent integration tests.
// This package is experimental.
package proftest
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"regexp"
"strconv"
"strings"
"text/template"
"time"
"cloud.google.com/go/storage"
gax "github.com/googleapis/gax-go/v2"
compute "google.golang.org/api/compute/v1"
container "google.golang.org/api/container/v1"
"google.golang.org/api/googleapi"
)
const (
monitorWriteScope = "https://www.googleapis.com/auth/monitoring.write"
)
var (
logtimeRE = regexp.MustCompile(`[A-Z][a-z]{2} [A-Z][a-z]{2} ?\d+ \d\d:\d\d:\d\d [A-Z]{3} \d{4}`)
// "ms" must be specified before "m" in the regexp, to ensure "ms" is fully
// matched.
backoffTimeRE = regexp.MustCompile(`(?:^|\W)((\d+(\.\d+)?(ms|h|m|s|us))+)(?:$|\W)`)
)
// BaseStartupTmpl is the common part of the startup script that
// can be shared by multiple tests.
var BaseStartupTmpl = template.Must(template.New("startupScript").Parse(`
{{ define "prologue" -}}
#! /bin/bash
(
# Signal any unexpected error.
trap 'echo "{{.ErrorString}}"' ERR
# Shut down the VM in 5 minutes after this script exits
# to stop accounting the VM for billing and cores quota.
trap "sleep 300 && poweroff" EXIT
retry() {
for i in {1..3}; do
[ $i == 1 ] || sleep 10 # Backing off after a failed attempt.
"${@}" && return 0
done
return 1
}
# Fail on any error.
set -eo pipefail
# Display commands being run
set -x
{{- end }}
{{ define "epilogue" -}}
# Write output to serial port 2 with timestamp.
) 2>&1 | while read line; do echo "$(date): ${line}"; done >/dev/ttyS1
{{- end }}
`))
// TestRunner has common elements used for testing profiling agents on a range
// of environments.
type TestRunner struct {
Client *http.Client
}
// GCETestRunner supports testing a profiling agent on GCE.
type GCETestRunner struct {
TestRunner
ComputeService *compute.Service
}
// GKETestRunner supports testing a profiling agent on GKE.
type GKETestRunner struct {
TestRunner
ContainerService *container.Service
StorageClient *storage.Client
Dockerfile string
}
// ProfileResponse contains the response produced when querying profile server.
type ProfileResponse struct {
Profile ProfileData `json:"profile"`
NumProfiles int32 `json:"numProfiles"`
Deployments []interface{} `json:"deployments"`
}
// ProfileData has data of a single profile.
type ProfileData struct {
Samples []int32 `json:"samples"`
SampleMetrics interface{} `json:"sampleMetrics"`
DefaultMetricType string `json:"defaultMetricType"`
TreeNodes interface{} `json:"treeNodes"`
Functions functionArray `json:"functions"`
SourceFiles sourceFileArray `json:"sourceFiles"`
}
type functionArray struct {
Name []string `json:"name"`
Sourcefile []int32 `json:"sourceFile"`
}
type sourceFileArray struct {
Name []string `json:"name"`
}
// InstanceConfig is configuration for starting single GCE instance for
// profiling agent test case.
type InstanceConfig struct {
ProjectID string
Zone string
Name string
StartupScript string
MachineType string
ImageProject string
ImageFamily string
Scopes []string
}
// ClusterConfig is configuration for starting single GKE cluster for profiling
// agent test case.
type ClusterConfig struct {
ProjectID string
Zone string
ClusterName string
PodName string
ImageSourceName string
ImageName string
Bucket string
Dockerfile string
}
// queryProfileRequest is the request format for querying the profile server.
type queryProfileRequest struct {
StartTime string `json:"startTime"`
EndTime string `json:"endTime"`
ProfileType string `json:"profileType"`
Target string `json:"target"`
DeploymentLabels map[string]string `json:"deploymentLabels,omitempty"`
}
// CheckNonEmpty returns nil if the profile has a profiles and deployments
// associated. Otherwise, returns a desciptive error.
func (pr *ProfileResponse) CheckNonEmpty() error {
if pr.NumProfiles == 0 {
return fmt.Errorf("profile response contains zero profiles: %v", pr)
}
if len(pr.Deployments) == 0 {
return fmt.Errorf("profile response contains zero deployments: %v", pr)
}
return nil
}
// HasFunction returns nil if the function is present, or, if the function is
// not present, and error providing more details why the function is not
// present.
func (pr *ProfileResponse) HasFunction(functionName string) error {
if err := pr.CheckNonEmpty(); err != nil {
return fmt.Errorf("failed to find function name %s in profile: %v", functionName, err)
}
for _, name := range pr.Profile.Functions.Name {
if strings.Contains(name, functionName) {
return nil
}
}
return fmt.Errorf("failed to find function name %s in profile", functionName)
}
// HasFunctionInFile returns nil if function is present in the specified file, and an
// error if the function/file combination is not present in the profile.
func (pr *ProfileResponse) HasFunctionInFile(functionName string, filename string) error {
if err := pr.CheckNonEmpty(); err != nil {
return fmt.Errorf("failed to find function name %s in file %s in profile: %v", functionName, filename, err)
}
for i, name := range pr.Profile.Functions.Name {
file := pr.Profile.SourceFiles.Name[pr.Profile.Functions.Sourcefile[i]]
if strings.Contains(name, functionName) && strings.HasSuffix(file, filename) {
return nil
}
}
return fmt.Errorf("failed to find function name %s in file %s in profile", functionName, filename)
}
// HasSourceFile returns nil if the file (or file where the end of the file path
// matches the filename) is present in the profile. Or, if the filename is not
// present, an error is returned.
func (pr *ProfileResponse) HasSourceFile(filename string) error {
if err := pr.CheckNonEmpty(); err != nil {
return fmt.Errorf("failed to find filename %s in profile: %v", filename, err)
}
for _, name := range pr.Profile.SourceFiles.Name {
if strings.HasSuffix(name, filename) {
return nil
}
}
return fmt.Errorf("failed to find filename %s in profile", filename)
}
// StartInstance starts a GCE Instance with configs specified by inst,
// and which runs the startup script specified in inst. If image project
// is not specified, it defaults to "debian-cloud". If image family is
// not specified, it defaults to "debian-9".
func (tr *GCETestRunner) StartInstance(ctx context.Context, inst *InstanceConfig) error {
imageProject, imageFamily := inst.ImageProject, inst.ImageFamily
if imageProject == "" {
imageProject = "debian-cloud"
}
if imageFamily == "" {
imageFamily = "debian-9"
}
img, err := tr.ComputeService.Images.GetFromFamily(imageProject, imageFamily).Context(ctx).Do()
if err != nil {
return fmt.Errorf("failed to get image from family %q in project %q: %v", imageFamily, imageProject, err)
}
op, err := tr.ComputeService.Instances.Insert(inst.ProjectID, inst.Zone, &compute.Instance{
MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", inst.Zone, inst.MachineType),
Name: inst.Name,
Disks: []*compute.AttachedDisk{{
AutoDelete: true, // delete the disk when the VM is deleted.
Boot: true,
Type: "PERSISTENT",
Mode: "READ_WRITE",
InitializeParams: &compute.AttachedDiskInitializeParams{
SourceImage: img.SelfLink,
DiskType: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/zones/%s/diskTypes/pd-standard", inst.ProjectID, inst.Zone),
},
}},
NetworkInterfaces: []*compute.NetworkInterface{{
Network: fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/global/networks/default", inst.ProjectID),
AccessConfigs: []*compute.AccessConfig{{
Name: "External NAT",
}},
}},
Metadata: &compute.Metadata{
Items: []*compute.MetadataItems{{
Key: "startup-script",
Value: googleapi.String(inst.StartupScript),
}},
},
ServiceAccounts: []*compute.ServiceAccount{{
Email: "default",
Scopes: append(inst.Scopes, monitorWriteScope),
}},
}).Do()
if err != nil {
return fmt.Errorf("failed to create instance: %v", err)
}
// Poll status of the operation to create the instance.
getOpCall := tr.ComputeService.ZoneOperations.Get(inst.ProjectID, inst.Zone, op.Name)
for {
if err := checkOpErrors(op); err != nil {
return fmt.Errorf("failed to create instance: %v", err)
}
if op.Status == "DONE" {
return nil
}
if err := gax.Sleep(ctx, 5*time.Second); err != nil {
return err
}
op, err = getOpCall.Do()
if err != nil {
return fmt.Errorf("failed to get operation: %v", err)
}
}
}
// checkOpErrors returns nil if the operation does not have any errors and an
// error summarizing all errors encountered if the operation has errored.
func checkOpErrors(op *compute.Operation) error {
if op.Error == nil || len(op.Error.Errors) == 0 {
return nil
}
var errs []string
for _, e := range op.Error.Errors {
if e.Message != "" {
errs = append(errs, e.Message)
} else {
errs = append(errs, e.Code)
}
}
return errors.New(strings.Join(errs, ","))
}
// DeleteInstance deletes an instance with project id, name, and zone matched
// by inst.
func (tr *GCETestRunner) DeleteInstance(ctx context.Context, inst *InstanceConfig) error {
if _, err := tr.ComputeService.Instances.Delete(inst.ProjectID, inst.Zone, inst.Name).Context(ctx).Do(); err != nil {
return fmt.Errorf("Instances.Delete(%s) got error: %v", inst.Name, err)
}
return nil
}
// CheckSerialOutputForBackoffs parses serial port output, and validates that
// server-specified backoffs were respected by the agent.
func CheckSerialOutputForBackoffs(output string, numBenchmarks int, serverBackoffSubstring, createProfileSubstring, benchmarkNumPrefix string) error {
benchmarkNumRE, err := regexp.Compile(fmt.Sprintf("%s (\\d+):", benchmarkNumPrefix))
if err != nil {
return fmt.Errorf("could not compile regexp to determine benchmark number: %v", err)
}
// Each CreateProfile request after a backoff should occur within
// createTimeBuffer of the time at which the request is expected.
createTimeBuffer := time.Minute
nextCreateTime := make([]time.Time, numBenchmarks)
benchmarkLogFounds := make([]bool, numBenchmarks)
var serverBackoff, createAfterBackoff bool
var logTime time.Time
for _, line := range strings.Split(output, "\n") {
if line == "" {
continue
}
// Find the benchmark number associated with line. Ignore cases when
// the benchmark number cannot be determined; the log line might just
// not be associated with a benchmark.
benchNum, err := parseBenchmarkNumber(line, numBenchmarks, benchmarkNumRE)
if err != nil {
continue
}
benchmarkLogFounds[benchNum] = true
// Find the time at which log statement was logged.
logTime, err = parseLogTime(line)
if err != nil {
return fmt.Errorf("failed to parse timestamp for log statement: %v", err)
}
switch {
// For a log indicating that the agent has received a server-specified
// backoff, record when the next CreateProfile request should happen
// if this backoff is respected.
case strings.Contains(line, serverBackoffSubstring):
serverBackoff = true
backoffDur, err := parseBackoffDuration(line)
if err != nil {
return fmt.Errorf("failed to parse backoff duration for line %q: %v", line, err)
}
nextCreateTime[benchNum] = logTime.Add(backoffDur)
// For a log indicating a CreateProfile request has been sent, confirm
// that if the agent has previously received a server-specified backoff,
// then this CreateProfile request is sent when the agent should sent
// its next CreateProfile request.
case strings.Contains(line, createProfileSubstring):
if time.Time.IsZero(nextCreateTime[benchNum]) {
continue
}
if logTime.Before(nextCreateTime[benchNum].Add(-1*createTimeBuffer)) || logTime.After(nextCreateTime[benchNum].Add(createTimeBuffer)) {
return fmt.Errorf("got next CreateProfile request for benchmark %d at %v, want next CreateProfile request within %v of %v", benchNum, logTime, createTimeBuffer, nextCreateTime[benchNum])
}
nextCreateTime[benchNum] = time.Time{}
createAfterBackoff = true
}
}
// Return an error if there is not at least one server-specified backoff
// in the logs. We can't validated that server-specified backoffs are
// respected if no server-specified backoffs appearred.
if !serverBackoff {
return errors.New("no server-specified backoff encountered")
}
// Return an error if there is not at least one CreateProfile request
// from an agent which had previously received a server-specified
// backoff. We cannot validate that agents send another CreateProfile
// request after the backoff duration has elapsed unless some agent
// has sent a CreateProfile request after having received a backoff.
if !createAfterBackoff {
return errors.New("did not encounter a CreateProfile request which occurred after a server-specified backoff")
}
// Confirm that no agent which was backed off at the time at which
// benchmark applications exited should have sent a CreateProfile request
// prior to the time at which benchmark applications exited.
for i, nct := range nextCreateTime {
// logTime will contain the timestamp for the last benchmark log
// statment, so is a proxy for when benchmarks have exitted.
// After benchmarks exit, any expected CreateProfile requests should
// either have been made (and so not be in nextCreateTime) or be
// expected to occur after the benchmark exits. Since not all
// benchmarks will exit at the same time and CreateProfile requests may
// not occur exactly when expected, fail only if CreateProfile requests
// occur more than 1 minute + createTimeBuffer before logTime.
if !time.Time.IsZero(nct) && nct.Add(createTimeBuffer+time.Minute).Before(logTime) {
return fmt.Errorf("got no CreateProfile request for benchmark %d before %v, want CreateProfileRequest at %v", i, logTime, nct)
}
}
return nil
}
// parseBenchmarkNumber returns the benchmark number associated with a line or
// an error if the line is missing or has an invalid benchmark number.
// Each instance of a benchmark run as part of a backoff integration test is
// associated with a number to allow logs for each instance of each benchmark
// to be considered independently.
func parseBenchmarkNumber(line string, numBenchmarks int, benchmarkNumRE *regexp.Regexp) (int, error) {
m := benchmarkNumRE.FindStringSubmatch(line)
if m == nil || len(m) < 2 {
return 0, fmt.Errorf("line %q does not have benchmark number", line)
}
benchNum, err := strconv.Atoi(m[1])
if err != nil {
return 0, fmt.Errorf("line %q has invalid benchmark number %q: %v", line, benchNum, err)
}
if benchNum < 0 || benchNum >= numBenchmarks {
return 0, fmt.Errorf("line %q had invalid benchmark number %d: benchmark number must be between 0 and %d", line, benchNum, numBenchmarks-1)
}
return benchNum, nil
}
// parseLogTime returns the timestamp associated with a logged line, or an
// error if the log's timestamp cannot be determined.
func parseLogTime(line string) (time.Time, error) {
logTimeStr := logtimeRE.FindString(line)
if logTimeStr == "" {
return time.Time{}, fmt.Errorf("logged line %q does not include a timestamp", line)
}
return time.Parse("Mon Jan 2 15:04:05 UTC 2006", logTimeStr)
}
// parseBackoffDuration returns the backoff duration associated with a logged
// line, or an error if the line does not contain a valid backoff duration.
func parseBackoffDuration(line string) (time.Duration, error) {
backoffTimeMatch := backoffTimeRE.FindStringSubmatch(line)
if len(backoffTimeMatch) < 2 || backoffTimeMatch[1] == "" {
return 0, fmt.Errorf("log for server-specified backoff %q does not include a backoff time", line)
}
backoffTimeStr := backoffTimeMatch[1]
backoff, err := time.ParseDuration(backoffTimeStr)
if err != nil {
return 0, fmt.Errorf("failed to parse backoff duration %q", backoffTimeStr)
}
return backoff, nil
}
// PollAndLogSerialPort polls serial port 2 of the given GCE instance and
// returns when the finishString appears in the serial output of the instance,
// or when the context times out. It logs the serial output of the instance
// using the specified log function and returns the serial output in the first
// return value.
func (tr *GCETestRunner) PollAndLogSerialPort(ctx context.Context, inst *InstanceConfig, finishString, errorString string, logf func(string, ...interface{})) (string, error) {
var output string
defer func() {
// Avoid escaping and double newlines in the rendered output (b/175999077).
// TODO: Use strings.ReplaceAll once support for Go 1.11 is dropped.
output = strings.Replace(output, "\r\n", "\n", -1)
output = strings.Replace(output, "\033", "\\033", -1)
logf("Serial port output for %s:\n%s", inst.Name, output)
}()
for {
select {
case <-ctx.Done():
return "", ctx.Err()
case <-time.After(20 * time.Second):
resp, err := tr.ComputeService.Instances.GetSerialPortOutput(inst.ProjectID, inst.Zone, inst.Name).Port(2).Context(ctx).Do()
if err != nil {
// Transient failure.
logf("Transient error getting serial port output from instance %s (will retry): %v", inst.Name, err)
continue
}
if resp.Contents == "" {
logf("Ignoring empty serial port output from instance %s (will retry)", inst.Name)
continue
}
if output = resp.Contents; strings.Contains(output, finishString) {
return output, nil
}
if strings.Contains(output, errorString) {
return output, fmt.Errorf("failed to execute the prober benchmark script")
}
}
}
}
// QueryProfiles retrieves profiles of a specific type, from a specific time
// range, associated with a particular service and project.
func (tr *TestRunner) QueryProfiles(projectID, service, startTime, endTime, profileType string) (ProfileResponse, error) {
return tr.QueryProfilesWithZone(projectID, service, startTime, endTime, profileType, "")
}
// QueryProfilesWithZone retrieves profiles of a specific type, from a specific
// time range, in a specified zone, associated with a particular service
// and project.
func (tr *TestRunner) QueryProfilesWithZone(projectID, service, startTime, endTime, profileType, zone string) (ProfileResponse, error) {
queryURL := fmt.Sprintf("https://cloudprofiler.googleapis.com/v2/projects/%s/profiles:query", projectID)
deploymentLabels := map[string]string{}
if zone != "" {
deploymentLabels["zone"] = zone
}
qpr := queryProfileRequest{
StartTime: startTime,
EndTime: endTime,
ProfileType: profileType,
Target: service,
DeploymentLabels: deploymentLabels,
}
queryJSON, err := json.Marshal(qpr)
if err != nil {
return ProfileResponse{}, fmt.Errorf("failed to marshall request to JSON: %v", err)
}
req, err := http.NewRequest("POST", queryURL, bytes.NewReader(queryJSON))
if err != nil {
return ProfileResponse{}, fmt.Errorf("failed to create an API request: %v", err)
}
req.Header = map[string][]string{
"X-Goog-User-Project": {projectID},
}
resp, err := tr.Client.Do(req)
if err != nil {
return ProfileResponse{}, fmt.Errorf("failed to query API: %v", err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return ProfileResponse{}, fmt.Errorf("failed to read response body: %v", err)
}
if resp.StatusCode != 200 {
return ProfileResponse{}, fmt.Errorf("failed to query API: status: %s, response body: %s", resp.Status, string(body))
}
var pr ProfileResponse
if err := json.Unmarshal(body, &pr); err != nil {
return ProfileResponse{}, err
}
return pr, nil
}
type imageResponse struct {
Manifest map[string]interface{} `json:"manifest"`
Name string `json:"name"`
Tags []string `json:"tags"`
}
// deleteDockerImage deletes a docker image from Google Container Registry.
func (tr *GKETestRunner) deleteDockerImage(ctx context.Context, ImageName string) []error {
queryImageURL := fmt.Sprintf("https://gcr.io/v2/%s/tags/list", ImageName)
resp, err := tr.Client.Get(queryImageURL)
if err != nil {
return []error{fmt.Errorf("failed to list tags: %v", err)}
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return []error{err}
}
var ir imageResponse
if err := json.Unmarshal(body, &ir); err != nil {
return []error{err}
}
const deleteImageURLFmt = "https://gcr.io/v2/%s/manifests/%s"
var errs []error
for _, tag := range ir.Tags {
if err := deleteDockerImageResource(tr.Client, fmt.Sprintf(deleteImageURLFmt, ImageName, tag)); err != nil {
errs = append(errs, fmt.Errorf("failed to delete tag %s: %v", tag, err))
}
}
for manifest := range ir.Manifest {
if err := deleteDockerImageResource(tr.Client, fmt.Sprintf(deleteImageURLFmt, ImageName, manifest)); err != nil {
errs = append(errs, fmt.Errorf("failed to delete manifest %s: %v", manifest, err))
}
}
return errs
}
func deleteDockerImageResource(client *http.Client, url string) error {
req, err := http.NewRequest("DELETE", url, nil)
if err != nil {
return fmt.Errorf("failed to get request: %v", err)
}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("failed to delete resource: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
return fmt.Errorf("failed to delete resource: status code = %d", resp.StatusCode)
}
return nil
}
// DeleteClusterAndImage deletes cluster and images used to create cluster.
func (tr *GKETestRunner) DeleteClusterAndImage(ctx context.Context, cfg *ClusterConfig) []error {
var errs []error
if err := tr.StorageClient.Bucket(cfg.Bucket).Object(cfg.ImageSourceName).Delete(ctx); err != nil {
errs = append(errs, fmt.Errorf("failed to delete storage client: %v", err))
}
for _, err := range tr.deleteDockerImage(ctx, cfg.ImageName) {
errs = append(errs, fmt.Errorf("failed to delete docker image: %v", err))
}
if _, err := tr.ContainerService.Projects.Zones.Clusters.Delete(cfg.ProjectID, cfg.Zone, cfg.ClusterName).Context(ctx).Do(); err != nil {
errs = append(errs, fmt.Errorf("failed to delete cluster %s: %v", cfg.ClusterName, err))
}
return errs
}