From b824c897e6941270747b612f6d36a8d6ae081315 Mon Sep 17 00:00:00 2001 From: Brenna N Epp Date: Fri, 8 Oct 2021 11:44:52 -0700 Subject: [PATCH] feat(storage): SignedUrl can use existing creds to authenticate (#4604) Co-authored-by: Cody Oss <6331106+codyoss@users.noreply.github.com> --- storage/bucket.go | 115 +++++++++++++++++++++++++++++++++ storage/integration_test.go | 125 +++++++++++++++++++++++++++++++++--- storage/storage.go | 50 ++++++++++++--- 3 files changed, 273 insertions(+), 17 deletions(-) diff --git a/storage/bucket.go b/storage/bucket.go index bfcddd5aa3f..d8747e58c95 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -16,16 +16,22 @@ package storage import ( "context" + "encoding/base64" + "encoding/json" + "errors" "fmt" "net/http" "reflect" "time" + "cloud.google.com/go/compute/metadata" "cloud.google.com/go/internal/optional" "cloud.google.com/go/internal/trace" "golang.org/x/xerrors" "google.golang.org/api/googleapi" + "google.golang.org/api/iamcredentials/v1" "google.golang.org/api/iterator" + "google.golang.org/api/option" raw "google.golang.org/api/storage/v1" ) @@ -225,6 +231,115 @@ func (b *BucketHandle) newPatchCall(uattrs *BucketAttrsToUpdate) (*raw.BucketsPa return req, nil } +// SignedURL returns a URL for the specified object. Signed URLs allow anyone +// access to a restricted resource for a limited time without needing a +// Google account or signing in. For more information about signed URLs, see +// https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication +// +// This method only requires the Method and Expires fields in the specified +// SignedURLOptions opts to be non-nil. If not provided, it attempts to fill the +// GoogleAccessID and PrivateKey from the GOOGLE_APPLICATION_CREDENTIALS environment variable. +// If no private key is found, it attempts to use the GoogleAccessID to sign the URL. +// This requires the IAM Service Account Credentials API to be enabled +// (https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview) +// and iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account. +// If you do not want these fields set for you, you may pass them in through opts or use +// SignedURL(bucket, name string, opts *SignedURLOptions) instead. +func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string, error) { + if opts.GoogleAccessID != "" && (opts.SignBytes != nil || len(opts.PrivateKey) > 0) { + return SignedURL(b.name, object, opts) + } + // Make a copy of opts so we don't modify the pointer parameter. + newopts := opts.clone() + + if newopts.GoogleAccessID == "" { + id, err := b.detectDefaultGoogleAccessID() + if err != nil { + return "", err + } + newopts.GoogleAccessID = id + } + if newopts.SignBytes == nil && len(newopts.PrivateKey) == 0 { + if len(b.c.creds.JSON) > 0 { + var sa struct { + PrivateKey string `json:"private_key"` + } + err := json.Unmarshal(b.c.creds.JSON, &sa) + if err == nil && sa.PrivateKey != "" { + newopts.PrivateKey = []byte(sa.PrivateKey) + } + } + + // Don't error out if we can't unmarshal the private key from the client, + // fallback to the default sign function for the service account. + if len(newopts.PrivateKey) == 0 { + newopts.SignBytes = b.defaultSignBytesFunc(newopts.GoogleAccessID) + } + } + return SignedURL(b.name, object, newopts) +} + +// TODO: Add a similar wrapper for GenerateSignedPostPolicyV4 allowing users to +// omit PrivateKey/SignBytes + +func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) { + returnErr := errors.New("no credentials found on client and not on GCE (Google Compute Engine)") + + if len(b.c.creds.JSON) > 0 { + var sa struct { + ClientEmail string `json:"client_email"` + } + err := json.Unmarshal(b.c.creds.JSON, &sa) + if err == nil && sa.ClientEmail != "" { + return sa.ClientEmail, nil + } else if err != nil { + returnErr = err + } else { + returnErr = errors.New("storage: empty client email in credentials") + } + + } + + // Don't error out if we can't unmarshal, fallback to GCE check. + if metadata.OnGCE() { + email, err := metadata.Email("default") + if err == nil && email != "" { + return email, nil + } else if err != nil { + returnErr = err + } else { + returnErr = errors.New("got empty email from GCE metadata service") + } + + } + return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %v", returnErr) +} + +func (b *BucketHandle) defaultSignBytesFunc(email string) func([]byte) ([]byte, error) { + return func(in []byte) ([]byte, error) { + ctx := context.Background() + + // It's ok to recreate this service per call since we pass in the http client, + // circumventing the cost of recreating the auth/transport layer + svc, err := iamcredentials.NewService(ctx, option.WithHTTPClient(b.c.hc)) + if err != nil { + return nil, fmt.Errorf("unable to create iamcredentials client: %v", err) + } + + resp, err := svc.Projects.ServiceAccounts.SignBlob(fmt.Sprintf("projects/-/serviceAccounts/%s", email), &iamcredentials.SignBlobRequest{ + Payload: base64.StdEncoding.EncodeToString(in), + }).Do() + if err != nil { + return nil, fmt.Errorf("unable to sign bytes: %v", err) + } + out, err := base64.StdEncoding.DecodeString(resp.SignedBlob) + if err != nil { + return nil, fmt.Errorf("unable to base64 decode response: %v", err) + } + return out, nil + } +} + // BucketAttrs represents the metadata for a Google Cloud Storage bucket. // Read-only fields are ignored by BucketHandle.Create. type BucketAttrs struct { diff --git a/storage/integration_test.go b/storage/integration_test.go index 36ade9436be..1b2457f2166 100644 --- a/storage/integration_test.go +++ b/storage/integration_test.go @@ -54,6 +54,7 @@ import ( "google.golang.org/api/iterator" itesting "google.golang.org/api/iterator/testing" "google.golang.org/api/option" + "google.golang.org/api/transport" iampb "google.golang.org/genproto/googleapis/iam/v1" ) @@ -202,11 +203,11 @@ func initUIDsAndRand(t time.Time) { // testConfig returns the Client used to access GCS. testConfig skips // the current test if credentials are not available or when being run // in Short mode. -func testConfig(ctx context.Context, t *testing.T) *Client { +func testConfig(ctx context.Context, t *testing.T, scopes ...string) *Client { if testing.Short() && !replaying { t.Skip("Integration tests skipped in short mode") } - client := config(ctx) + client := config(ctx, scopes...) if client == nil { t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") } @@ -229,8 +230,9 @@ func testConfigGRPC(ctx context.Context, t *testing.T) (gc *Client) { } // config is like testConfig, but it doesn't need a *testing.T. -func config(ctx context.Context) *Client { - ts := testutil.TokenSource(ctx, ScopeFullControl) +func config(ctx context.Context, scopes ...string) *Client { + scopes = append(scopes, ScopeFullControl) + ts := testutil.TokenSource(ctx, scopes...) if ts == nil { return nil } @@ -2017,7 +2019,8 @@ func TestIntegration_SignedURL(t *testing.T) { opts.PrivateKey = jwtConf.PrivateKey opts.Method = "GET" opts.Expires = time.Now().Add(time.Hour) - u, err := SignedURL(bucketName, obj, &opts) + + u, err := bkt.SignedURL(obj, &opts) if err != nil { t.Errorf("%s: SignedURL: %v", test.desc, err) continue @@ -2049,6 +2052,8 @@ func TestIntegration_SignedURL_WithEncryptionKeys(t *testing.T) { client := testConfig(ctx, t) defer client.Close() + bkt := client.Bucket(bucketName) + // TODO(deklerk): document how these were generated and their significance encryptionKey := "AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI=" encryptionKeySha256 := "QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k=" @@ -2089,7 +2094,6 @@ func TestIntegration_SignedURL_WithEncryptionKeys(t *testing.T) { } defer func() { // Delete encrypted object. - bkt := client.Bucket(bucketName) err := bkt.Object("csek.json").Delete(ctx) if err != nil { log.Printf("failed to deleted encrypted file: %v", err) @@ -2102,7 +2106,7 @@ func TestIntegration_SignedURL_WithEncryptionKeys(t *testing.T) { opts.PrivateKey = jwtConf.PrivateKey opts.Expires = time.Now().Add(time.Hour) - u, err := SignedURL(bucketName, "csek.json", test.opts) + u, err := bkt.SignedURL("csek.json", test.opts) if err != nil { t.Fatalf("%s: %v", test.desc, err) } @@ -2152,7 +2156,8 @@ func TestIntegration_SignedURL_EmptyStringObjectName(t *testing.T) { Expires: time.Now().Add(time.Hour), } - u, err := SignedURL(bucketName, "", opts) + bkt := client.Bucket(bucketName) + u, err := bkt.SignedURL("", opts) if err != nil { t.Fatal(err) } @@ -4241,6 +4246,110 @@ func TestIntegration_Scopes(t *testing.T) { } +func TestBucketSignURL(t *testing.T) { + ctx := context.Background() + + if testing.Short() && !replaying { + t.Skip("Integration tests skipped in short mode") + } + + // We explictly send the key to the client to sign with the private key + clientWithCredentials := newTestClientWithExplicitCredentials(ctx, t) + defer clientWithCredentials.Close() + + // Create another client to test the sign byte function as well + clientWithoutPrivateKey := testConfig(ctx, t, ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform") + defer clientWithoutPrivateKey.Close() + + jwt, err := testutil.JWTConfig() + if err != nil { + t.Fatalf("unable to find test credentials: %v", err) + } + + // We can use any client to create the object + obj := "testBucketSignedURL" + contents := []byte("test") + if err := writeObject(ctx, clientWithoutPrivateKey.Bucket(bucketName).Object(obj), "text/plain", contents); err != nil { + t.Fatalf("writing: %v", err) + } + + for _, test := range []struct { + desc string + opts SignedURLOptions + client *Client + }{ + { + desc: "signing with the private key", + opts: SignedURLOptions{ + Method: "GET", + Expires: time.Now().Add(30 * time.Second), + }, + client: clientWithCredentials, + }, + { + desc: "signing with the default sign bytes func", + opts: SignedURLOptions{ + Method: "GET", + Expires: time.Now().Add(30 * time.Second), + GoogleAccessID: jwt.Email, + }, + client: clientWithoutPrivateKey, + }, + } { + bkt := test.client.Bucket(bucketName) + url, err := bkt.SignedURL(obj, &test.opts) + if err != nil { + t.Fatalf("unable to create signed URL: %v", err) + } + resp, err := http.Get(url) + if err != nil { + t.Fatalf("http.Get(%q) errored: %q", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatalf("resp.StatusCode = %v, want 200: %v", resp.StatusCode, err) + } + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatalf("unable to read resp.Body: %v", err) + } + if !bytes.Equal(b, contents) { + t.Fatalf("got %q, want %q", b, contents) + } + } +} + +func newTestClientWithExplicitCredentials(ctx context.Context, t *testing.T) *Client { + // By default we are authed with a token source, so don't have the context to + // read some of the fields from the keyfile + // Here we explictly send the key to the client + creds, err := findTestCredentials(ctx, "GCLOUD_TESTS_GOLANG_KEY", ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform") + if err != nil { + t.Fatalf("unable to find test credentials: %v", err) + } + + clientWithCredentials, err := newTestClient(ctx, option.WithCredentials(creds)) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + if clientWithCredentials == nil { + t.Skip("Integration tests skipped. See CONTRIBUTING.md for details") + } + return clientWithCredentials +} + +func findTestCredentials(ctx context.Context, envVar string, scopes ...string) (*google.Credentials, error) { + key := os.Getenv(envVar) + var opts []option.ClientOption + if len(scopes) > 0 { + opts = append(opts, option.WithScopes(scopes...)) + } + if key != "" { + opts = append(opts, option.WithCredentialsFile(key)) + } + return transport.Creds(ctx, opts...) +} + type testHelper struct { t *testing.T } diff --git a/storage/storage.go b/storage/storage.go index ab5b8133452..98a0389d0fc 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -41,11 +41,13 @@ import ( "cloud.google.com/go/internal/trace" "cloud.google.com/go/internal/version" gapic "cloud.google.com/go/storage/internal/apiv2" + "golang.org/x/oauth2/google" "golang.org/x/xerrors" "google.golang.org/api/googleapi" "google.golang.org/api/option" "google.golang.org/api/option/internaloption" raw "google.golang.org/api/storage/v1" + "google.golang.org/api/transport" htransport "google.golang.org/api/transport/http" storagepb "google.golang.org/genproto/googleapis/storage/v2" "google.golang.org/protobuf/proto" @@ -98,6 +100,7 @@ type Client struct { scheme string // ReadHost is the default host used on the reader. readHost string + creds *google.Credentials // gc is an optional gRPC-based, GAPIC client. // @@ -112,6 +115,7 @@ type Client struct { // Clients should be reused instead of created as needed. The methods of Client // are safe for concurrent use by multiple goroutines. func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error) { + var creds *google.Credentials // In general, it is recommended to use raw.NewService instead of htransport.NewClient // since raw.NewService configures the correct default endpoints when initializing the @@ -122,10 +126,19 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error // need to account for STORAGE_EMULATOR_HOST override when setting the default endpoints. if host := os.Getenv("STORAGE_EMULATOR_HOST"); host == "" { // Prepend default options to avoid overriding options passed by the user. - opts = append([]option.ClientOption{option.WithScopes(ScopeFullControl), option.WithUserAgent(userAgent)}, opts...) + opts = append([]option.ClientOption{option.WithScopes(ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform"), option.WithUserAgent(userAgent)}, opts...) opts = append(opts, internaloption.WithDefaultEndpoint("https://storage.googleapis.com/storage/v1/")) opts = append(opts, internaloption.WithDefaultMTLSEndpoint("https://storage.mtls.googleapis.com/storage/v1/")) + + c, err := transport.Creds(ctx, opts...) + if err != nil { + return nil, err + } + creds = c + + opts = append(opts, internaloption.WithCredentials(creds)) + } else { var hostURL *url.URL @@ -172,6 +185,7 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error raw: rawService, scheme: u.Scheme, readHost: u.Host, + creds: creds, }, nil } @@ -210,6 +224,7 @@ func (c *Client) Close() error { // Set fields to nil so that subsequent uses will panic. c.hc = nil c.raw = nil + c.creds = nil if c.gc != nil { return c.gc.Close() } @@ -396,6 +411,23 @@ type SignedURLOptions struct { Scheme SigningScheme } +func (opts *SignedURLOptions) clone() *SignedURLOptions { + return &SignedURLOptions{ + GoogleAccessID: opts.GoogleAccessID, + SignBytes: opts.SignBytes, + PrivateKey: opts.PrivateKey, + Method: opts.Method, + Expires: opts.Expires, + ContentType: opts.ContentType, + Headers: opts.Headers, + QueryParameters: opts.QueryParameters, + MD5: opts.MD5, + Style: opts.Style, + Insecure: opts.Insecure, + Scheme: opts.Scheme, + } +} + var ( tabRegex = regexp.MustCompile(`[\t]+`) // I was tempted to call this spacex. :) @@ -509,11 +541,11 @@ func v4SanitizeHeaders(hdrs []string) []string { return sanitizedHeaders } -// SignedURL returns a URL for the specified object. Signed URLs allow -// the users access to a restricted resource for a limited time without having a -// Google account or signing in. For more information about the signed -// URLs, see https://cloud.google.com/storage/docs/accesscontrol#Signed-URLs. -func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) { +// SignedURL returns a URL for the specified object. Signed URLs allow anyone +// access to a restricted resource for a limited time without needing a +// Google account or signing in. For more information about signed URLs, see +// https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication +func SignedURL(bucket, object string, opts *SignedURLOptions) (string, error) { now := utcNow() if err := validateOptions(opts, now); err != nil { return "", err @@ -522,13 +554,13 @@ func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) { switch opts.Scheme { case SigningSchemeV2: opts.Headers = v2SanitizeHeaders(opts.Headers) - return signedURLV2(bucket, name, opts) + return signedURLV2(bucket, object, opts) case SigningSchemeV4: opts.Headers = v4SanitizeHeaders(opts.Headers) - return signedURLV4(bucket, name, opts, now) + return signedURLV4(bucket, object, opts, now) default: // SigningSchemeDefault opts.Headers = v2SanitizeHeaders(opts.Headers) - return signedURLV2(bucket, name, opts) + return signedURLV2(bucket, object, opts) } }