feat(storage): SignedUrl can use existing creds to authenticate

storage/bucket.go

@@ -16,16 +16,22 @@ package storage
 
 import (
 	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"reflect"
 	"time"
 
+	"cloud.google.com/go/compute/metadata"
 	"cloud.google.com/go/internal/optional"
 	"cloud.google.com/go/internal/trace"
 	"golang.org/x/xerrors"
 	"google.golang.org/api/googleapi"
+	"google.golang.org/api/iamcredentials/v1"
 	"google.golang.org/api/iterator"
+	"google.golang.org/api/option"
 	raw "google.golang.org/api/storage/v1"
 )
 
@@ -225,6 +231,115 @@ func (b *BucketHandle) newPatchCall(uattrs *BucketAttrsToUpdate) (*raw.BucketsPa
 	return req, nil
 }
 
+// SignedURL returns a URL for the specified object. Signed URLs allow anyone
+// access to a restricted resource for a limited time without needing a
+// Google account or signing in. For more information about signed URLs, see
+// https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication
+//
+// This method only requires the Method and Expires fields in the specified
+// SignedURLOptions opts to be non-nil. If not provided, it attempts to fill the
+// GoogleAccessID and PrivateKey from the GOOGLE_APPLICATION_CREDENTIALS environment variable.
+// If no private key is found, it attempts to use the GoogleAccessID to sign the URL.
+// This requires the IAM Service Account Credentials API to be enabled
+// (https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview)
+// and iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account.
+// If you do not want these fields set for you, you may pass them in through opts or use
+// SignedURL(bucket, name string, opts *SignedURLOptions) instead.
+func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string, error) {
+	if opts.GoogleAccessID != "" && (opts.SignBytes != nil || len(opts.PrivateKey) > 0) {
+		return SignedURL(b.name, object, opts)
+	}
+	// Make a copy of opts so we don't modify the pointer parameter.
+	newopts := opts.clone()
+
+	if newopts.GoogleAccessID == "" {
+		id, err := b.detectDefaultGoogleAccessID()
+		if err != nil {
+			return "", err
+		}
+		newopts.GoogleAccessID = id
+	}
+	if newopts.SignBytes == nil && len(newopts.PrivateKey) == 0 {
+		if len(b.c.creds.JSON) > 0 {
+			var sa struct {
+				PrivateKey string `json:"private_key"`
+			}
+			err := json.Unmarshal(b.c.creds.JSON, &sa)
+			if err == nil && sa.PrivateKey != "" {
+				newopts.PrivateKey = []byte(sa.PrivateKey)
+			}
+		}
+
+		// Don't error out if we can't unmarshal the private key from the client,
+		// fallback to the default sign function for the service account.
+		if len(newopts.PrivateKey) == 0 {
+			newopts.SignBytes = b.defaultSignBytesFunc(newopts.GoogleAccessID)
+		}
+	}
+	return SignedURL(b.name, object, newopts)
+}
+
+// TODO: Add a similar wrapper for GenerateSignedPostPolicyV4 allowing users to
+// omit PrivateKey/SignBytes
+
+func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) {
+	returnErr := errors.New("no credentials found on client and not on GCE (Google Compute Engine)")
+
+	if len(b.c.creds.JSON) > 0 {
+		var sa struct {
+			ClientEmail string `json:"client_email"`
+		}
+		err := json.Unmarshal(b.c.creds.JSON, &sa)
+		if err == nil && sa.ClientEmail != "" {
+			return sa.ClientEmail, nil
+		} else if err != nil {
+			returnErr = err
+		} else {
+			returnErr = errors.New("storage: empty client email in credentials")
+		}
+
+	}
+
+	// Don't error out if we can't unmarshal, fallback to GCE check.
+	if metadata.OnGCE() {
+		email, err := metadata.Email("default")
+		if err == nil && email != "" {
+			return email, nil
+		} else if err != nil {
+			returnErr = err
+		} else {
+			returnErr = errors.New("got empty email from GCE metadata service")
+		}
+
+	}
+	return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %v", returnErr)
+}
+
+func (b *BucketHandle) defaultSignBytesFunc(email string) func([]byte) ([]byte, error) {
+	return func(in []byte) ([]byte, error) {
+		ctx := context.Background()
+
+		// It's ok to recreate this service per call since we pass in the http client,
+		// circumventing the cost of recreating the auth/transport layer
+		svc, err := iamcredentials.NewService(ctx, option.WithHTTPClient(b.c.hc))
+		if err != nil {
+			return nil, fmt.Errorf("unable to create iamcredentials client: %v", err)
+		}
+
+		resp, err := svc.Projects.ServiceAccounts.SignBlob(fmt.Sprintf("projects/-/serviceAccounts/%s", email), &iamcredentials.SignBlobRequest{
+			Payload: base64.StdEncoding.EncodeToString(in),
+		}).Do()
+		if err != nil {
+			return nil, fmt.Errorf("unable to sign bytes: %v", err)
+		}
+		out, err := base64.StdEncoding.DecodeString(resp.SignedBlob)
+		if err != nil {
+			return nil, fmt.Errorf("unable to base64 decode response: %v", err)
+		}
+		return out, nil
+	}
+}
+
 // BucketAttrs represents the metadata for a Google Cloud Storage bucket.
 // Read-only fields are ignored by BucketHandle.Create.
 type BucketAttrs struct {

storage/integration_test.go

@@ -54,6 +54,7 @@ import (
 	"google.golang.org/api/iterator"
 	itesting "google.golang.org/api/iterator/testing"
 	"google.golang.org/api/option"
+	"google.golang.org/api/transport"
 	iampb "google.golang.org/genproto/googleapis/iam/v1"
 )
 
@@ -202,11 +203,11 @@ func initUIDsAndRand(t time.Time) {
 // testConfig returns the Client used to access GCS. testConfig skips
 // the current test if credentials are not available or when being run
 // in Short mode.
-func testConfig(ctx context.Context, t *testing.T) *Client {
+func testConfig(ctx context.Context, t *testing.T, scopes ...string) *Client {
 	if testing.Short() && !replaying {
 		t.Skip("Integration tests skipped in short mode")
 	}
-	client := config(ctx)
+	client := config(ctx, scopes...)
 	if client == nil {
 		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
 	}
@@ -229,8 +230,9 @@ func testConfigGRPC(ctx context.Context, t *testing.T) (gc *Client) {
 }
 
 // config is like testConfig, but it doesn't need a *testing.T.
-func config(ctx context.Context) *Client {
-	ts := testutil.TokenSource(ctx, ScopeFullControl)
+func config(ctx context.Context, scopes ...string) *Client {
+	scopes = append(scopes, ScopeFullControl)
+	ts := testutil.TokenSource(ctx, scopes...)
 	if ts == nil {
 		return nil
 	}
@@ -645,7 +647,6 @@ func TestIntegration_PublicAccessPrevention(t *testing.T) {
 	if err := a.Set(ctx, AllUsers, RoleReader); err == nil {
 		t.Error("ACL.Set: expected adding AllUsers ACL to object should fail")
 	}
-	t.Skip("https://github.com/googleapis/google-cloud-go/issues/4890")
 
 	// Update PAP setting to unspecified should work and not affect UBLA setting.
 	attrs, err := bkt.Update(ctx, BucketAttrsToUpdate{PublicAccessPrevention: PublicAccessPreventionUnspecified})
@@ -2010,7 +2011,8 @@ func TestIntegration_SignedURL(t *testing.T) {
 		opts.PrivateKey = jwtConf.PrivateKey
 		opts.Method = "GET"
 		opts.Expires = time.Now().Add(time.Hour)
-		u, err := SignedURL(bucketName, obj, &opts)
+
+		u, err := bkt.SignedURL(obj, &opts)
 		if err != nil {
 			t.Errorf("%s: SignedURL: %v", test.desc, err)
 			continue
@@ -2042,6 +2044,8 @@ func TestIntegration_SignedURL_WithEncryptionKeys(t *testing.T) {
 	client := testConfig(ctx, t)
 	defer client.Close()
 
+	bkt := client.Bucket(bucketName)
+
 	// TODO(deklerk): document how these were generated and their significance
 	encryptionKey := "AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI="
 	encryptionKeySha256 := "QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k="
@@ -2082,7 +2086,6 @@ func TestIntegration_SignedURL_WithEncryptionKeys(t *testing.T) {
 	}
 	defer func() {
 		// Delete encrypted object.
-		bkt := client.Bucket(bucketName)
 		err := bkt.Object("csek.json").Delete(ctx)
 		if err != nil {
 			log.Printf("failed to deleted encrypted file: %v", err)
@@ -2095,7 +2098,7 @@ func TestIntegration_SignedURL_WithEncryptionKeys(t *testing.T) {
 		opts.PrivateKey = jwtConf.PrivateKey
 		opts.Expires = time.Now().Add(time.Hour)
 
-		u, err := SignedURL(bucketName, "csek.json", test.opts)
+		u, err := bkt.SignedURL("csek.json", test.opts)
 		if err != nil {
 			t.Fatalf("%s: %v", test.desc, err)
 		}
@@ -2145,7 +2148,8 @@ func TestIntegration_SignedURL_EmptyStringObjectName(t *testing.T) {
 		Expires:        time.Now().Add(time.Hour),
 	}
 
-	u, err := SignedURL(bucketName, "", opts)
+	bkt := client.Bucket(bucketName)
+	u, err := bkt.SignedURL("", opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -4234,6 +4238,110 @@ func TestIntegration_Scopes(t *testing.T) {
 
 }
 
+func TestBucketSignURL(t *testing.T) {
+	ctx := context.Background()
+
+	if testing.Short() && !replaying {
+		t.Skip("Integration tests skipped in short mode")
+	}
+
+	// We explictly send the key to the client to sign with the private key
+	clientWithCredentials := newTestClientWithExplicitCredentials(ctx, t)
+	defer clientWithCredentials.Close()
+
+	// Create another client to test the sign byte function as well
+	clientWithoutPrivateKey := testConfig(ctx, t, ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform")
+	defer clientWithoutPrivateKey.Close()
+
+	jwt, err := testutil.JWTConfig()
+	if err != nil {
+		t.Fatalf("unable to find test credentials: %v", err)
+	}
+
+	// We can use any client to create the object
+	obj := "testBucketSignedURL"
+	contents := []byte("test")
+	if err := writeObject(ctx, clientWithoutPrivateKey.Bucket(bucketName).Object(obj), "text/plain", contents); err != nil {
+		t.Fatalf("writing: %v", err)
+	}
+
+	for _, test := range []struct {
+		desc   string
+		opts   SignedURLOptions
+		client *Client
+	}{
+		{
+			desc: "signing with the private key",
+			opts: SignedURLOptions{
+				Method:  "GET",
+				Expires: time.Now().Add(30 * time.Second),
+			},
+			client: clientWithCredentials,
+		},
+		{
+			desc: "signing with the default sign bytes func",
+			opts: SignedURLOptions{
+				Method:         "GET",
+				Expires:        time.Now().Add(30 * time.Second),
+				GoogleAccessID: jwt.Email,
+			},
+			client: clientWithoutPrivateKey,
+		},
+	} {
+		bkt := test.client.Bucket(bucketName)
+		url, err := bkt.SignedURL(obj, &test.opts)
+		if err != nil {
+			t.Fatalf("unable to create signed URL: %v", err)
+		}
+		resp, err := http.Get(url)
+		if err != nil {
+			t.Fatalf("http.Get(%q) errored: %q", url, err)
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != 200 {
+			t.Fatalf("resp.StatusCode = %v, want 200: %v", resp.StatusCode, err)
+		}
+		b, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			t.Fatalf("unable to read resp.Body: %v", err)
+		}
+		if !bytes.Equal(b, contents) {
+			t.Fatalf("got %q, want %q", b, contents)
+		}
+	}
+}
+
+func newTestClientWithExplicitCredentials(ctx context.Context, t *testing.T) *Client {
+	// By default we are authed with a token source, so don't have the context to
+	// read some of the fields from the keyfile
+	// Here we explictly send the key to the client
+	creds, err := findTestCredentials(ctx, "GCLOUD_TESTS_GOLANG_KEY", ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform")
+	if err != nil {
+		t.Fatalf("unable to find test credentials: %v", err)
+	}
+
+	clientWithCredentials, err := newTestClient(ctx, option.WithCredentials(creds))
+	if err != nil {
+		t.Fatalf("NewClient: %v", err)
+	}
+	if clientWithCredentials == nil {
+		t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
+	}
+	return clientWithCredentials
+}
+
+func findTestCredentials(ctx context.Context, envVar string, scopes ...string) (*google.Credentials, error) {
+	key := os.Getenv(envVar)
+	var opts []option.ClientOption
+	if len(scopes) > 0 {
+		opts = append(opts, option.WithScopes(scopes...))
+	}
+	if key != "" {
+		opts = append(opts, option.WithCredentialsFile(key))
+	}
+	return transport.Creds(ctx, opts...)
+}
+
 type testHelper struct {
 	t *testing.T
 }