fix(profiler): Force gax to retry in case of certificate errors

profiler/profiler.go

@@ -44,6 +44,7 @@ import (
 	"regexp"
 	"runtime"
 	"runtime/pprof"
+	"strings"
 	"sync"
 	"time"
 
@@ -67,6 +68,7 @@ var (
 	config       Config
 	startOnce    allowUntilSuccess
 	mutexEnabled bool
+	logger       *log.Logger
 	// The functions below are stubbed to be overrideable for testing.
 	getProjectID     = gcemd.ProjectID
 	getInstanceName  = gcemd.InstanceName
@@ -215,6 +217,7 @@ func (o *allowUntilSuccess) do(f func() error) (err error) {
 // Config for details. Start should only be called once. Any
 // additional calls will be ignored.
 func Start(cfg Config, options ...option.ClientOption) error {
+	logger = log.New(os.Stderr, "Cloud Profiler: ", log.LstdFlags|log.Lmsgprefix)
 	startError := startOnce.do(func() error {
 		return start(cfg, options...)
 	})
@@ -261,7 +264,7 @@ func start(cfg Config, options ...option.ClientOption) error {
 
 func debugLog(format string, e ...interface{}) {
 	if config.DebugLogging {
-		log.Printf(format, e...)
+		logger.Printf(format, e...)
 	}
 }
 
@@ -315,7 +318,8 @@ func (r *retryer) Retry(err error) (time.Duration, bool) {
 // createProfile talks to the profiler server to create profile. In
 // case of error, the goroutine will sleep and retry. Sleep duration may
 // be specified by the server. Otherwise it will be an exponentially
-// increasing value, bounded by maxBackoff.
+// increasing value, bounded by maxBackoff. Special handling for
+// certificate errors is described below.
 func (a *agent) createProfile(ctx context.Context) *pb.Profile {
 	req := pb.CreateProfileRequest{
 		Parent:      "projects/" + a.deployment.ProjectId,
@@ -332,6 +336,11 @@ func (a *agent) createProfile(ctx context.Context) *pb.Profile {
 		p, err = a.client.CreateProfile(ctx, &req, grpc.Trailer(&md))
 		if err != nil {
 			debugLog("failed to create profile, will retry: %v", err)
+			if strings.Contains(err.Error(), "x509: certificate signed by unknown authority") {
+				// gax.Invoke does not retry missing certificate error. Force a retry by returning
+				// a different error. See https://github.com/googleapis/google-cloud-go/issues/3158.
+				err = errors.New("retry the certificate error")
+			}
 		}
 		return err
 	}, gax.WithRetry(func() gax.Retryer {