Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
feat(storage): SignedUrl can use existing creds to authenticate (#4604)
Co-authored-by: Cody Oss <6331106+codyoss@users.noreply.github.com>
  • Loading branch information
BrennaEpp and codyoss committed Oct 8, 2021
1 parent 7ac8489 commit b824c89
Show file tree
Hide file tree
Showing 3 changed files with 273 additions and 17 deletions.
115 changes: 115 additions & 0 deletions storage/bucket.go
Expand Up @@ -16,16 +16,22 @@ package storage

import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"net/http"
"reflect"
"time"

"cloud.google.com/go/compute/metadata"
"cloud.google.com/go/internal/optional"
"cloud.google.com/go/internal/trace"
"golang.org/x/xerrors"
"google.golang.org/api/googleapi"
"google.golang.org/api/iamcredentials/v1"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
raw "google.golang.org/api/storage/v1"
)

Expand Down Expand Up @@ -225,6 +231,115 @@ func (b *BucketHandle) newPatchCall(uattrs *BucketAttrsToUpdate) (*raw.BucketsPa
return req, nil
}

// SignedURL returns a URL for the specified object. Signed URLs allow anyone
// access to a restricted resource for a limited time without needing a
// Google account or signing in. For more information about signed URLs, see
// https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication
//
// This method only requires the Method and Expires fields in the specified
// SignedURLOptions opts to be non-nil. If not provided, it attempts to fill the
// GoogleAccessID and PrivateKey from the GOOGLE_APPLICATION_CREDENTIALS environment variable.
// If no private key is found, it attempts to use the GoogleAccessID to sign the URL.
// This requires the IAM Service Account Credentials API to be enabled
// (https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview)
// and iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account.
// If you do not want these fields set for you, you may pass them in through opts or use
// SignedURL(bucket, name string, opts *SignedURLOptions) instead.
func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string, error) {
if opts.GoogleAccessID != "" && (opts.SignBytes != nil || len(opts.PrivateKey) > 0) {
return SignedURL(b.name, object, opts)
}
// Make a copy of opts so we don't modify the pointer parameter.
newopts := opts.clone()

if newopts.GoogleAccessID == "" {
id, err := b.detectDefaultGoogleAccessID()
if err != nil {
return "", err
}
newopts.GoogleAccessID = id
}
if newopts.SignBytes == nil && len(newopts.PrivateKey) == 0 {
if len(b.c.creds.JSON) > 0 {
var sa struct {
PrivateKey string `json:"private_key"`
}
err := json.Unmarshal(b.c.creds.JSON, &sa)
if err == nil && sa.PrivateKey != "" {
newopts.PrivateKey = []byte(sa.PrivateKey)
}
}

// Don't error out if we can't unmarshal the private key from the client,
// fallback to the default sign function for the service account.
if len(newopts.PrivateKey) == 0 {
newopts.SignBytes = b.defaultSignBytesFunc(newopts.GoogleAccessID)
}
}
return SignedURL(b.name, object, newopts)
}

// TODO: Add a similar wrapper for GenerateSignedPostPolicyV4 allowing users to
// omit PrivateKey/SignBytes

func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) {
returnErr := errors.New("no credentials found on client and not on GCE (Google Compute Engine)")

if len(b.c.creds.JSON) > 0 {
var sa struct {
ClientEmail string `json:"client_email"`
}
err := json.Unmarshal(b.c.creds.JSON, &sa)
if err == nil && sa.ClientEmail != "" {
return sa.ClientEmail, nil
} else if err != nil {
returnErr = err
} else {
returnErr = errors.New("storage: empty client email in credentials")
}

}

// Don't error out if we can't unmarshal, fallback to GCE check.
if metadata.OnGCE() {
email, err := metadata.Email("default")
if err == nil && email != "" {
return email, nil
} else if err != nil {
returnErr = err
} else {
returnErr = errors.New("got empty email from GCE metadata service")
}

}
return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %v", returnErr)
}

func (b *BucketHandle) defaultSignBytesFunc(email string) func([]byte) ([]byte, error) {
return func(in []byte) ([]byte, error) {
ctx := context.Background()

// It's ok to recreate this service per call since we pass in the http client,
// circumventing the cost of recreating the auth/transport layer
svc, err := iamcredentials.NewService(ctx, option.WithHTTPClient(b.c.hc))
if err != nil {
return nil, fmt.Errorf("unable to create iamcredentials client: %v", err)
}

resp, err := svc.Projects.ServiceAccounts.SignBlob(fmt.Sprintf("projects/-/serviceAccounts/%s", email), &iamcredentials.SignBlobRequest{
Payload: base64.StdEncoding.EncodeToString(in),
}).Do()
if err != nil {
return nil, fmt.Errorf("unable to sign bytes: %v", err)
}
out, err := base64.StdEncoding.DecodeString(resp.SignedBlob)
if err != nil {
return nil, fmt.Errorf("unable to base64 decode response: %v", err)
}
return out, nil
}
}

// BucketAttrs represents the metadata for a Google Cloud Storage bucket.
// Read-only fields are ignored by BucketHandle.Create.
type BucketAttrs struct {
Expand Down
125 changes: 117 additions & 8 deletions storage/integration_test.go
Expand Up @@ -54,6 +54,7 @@ import (
"google.golang.org/api/iterator"
itesting "google.golang.org/api/iterator/testing"
"google.golang.org/api/option"
"google.golang.org/api/transport"
iampb "google.golang.org/genproto/googleapis/iam/v1"
)

Expand Down Expand Up @@ -202,11 +203,11 @@ func initUIDsAndRand(t time.Time) {
// testConfig returns the Client used to access GCS. testConfig skips
// the current test if credentials are not available or when being run
// in Short mode.
func testConfig(ctx context.Context, t *testing.T) *Client {
func testConfig(ctx context.Context, t *testing.T, scopes ...string) *Client {
if testing.Short() && !replaying {
t.Skip("Integration tests skipped in short mode")
}
client := config(ctx)
client := config(ctx, scopes...)
if client == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
Expand All @@ -229,8 +230,9 @@ func testConfigGRPC(ctx context.Context, t *testing.T) (gc *Client) {
}

// config is like testConfig, but it doesn't need a *testing.T.
func config(ctx context.Context) *Client {
ts := testutil.TokenSource(ctx, ScopeFullControl)
func config(ctx context.Context, scopes ...string) *Client {
scopes = append(scopes, ScopeFullControl)
ts := testutil.TokenSource(ctx, scopes...)
if ts == nil {
return nil
}
Expand Down Expand Up @@ -2017,7 +2019,8 @@ func TestIntegration_SignedURL(t *testing.T) {
opts.PrivateKey = jwtConf.PrivateKey
opts.Method = "GET"
opts.Expires = time.Now().Add(time.Hour)
u, err := SignedURL(bucketName, obj, &opts)

u, err := bkt.SignedURL(obj, &opts)
if err != nil {
t.Errorf("%s: SignedURL: %v", test.desc, err)
continue
Expand Down Expand Up @@ -2049,6 +2052,8 @@ func TestIntegration_SignedURL_WithEncryptionKeys(t *testing.T) {
client := testConfig(ctx, t)
defer client.Close()

bkt := client.Bucket(bucketName)

// TODO(deklerk): document how these were generated and their significance
encryptionKey := "AAryxNglNkXQY0Wa+h9+7BLSFMhCzPo22MtXUWjOBbI="
encryptionKeySha256 := "QlCdVONb17U1aCTAjrFvMbnxW/Oul8VAvnG1875WJ3k="
Expand Down Expand Up @@ -2089,7 +2094,6 @@ func TestIntegration_SignedURL_WithEncryptionKeys(t *testing.T) {
}
defer func() {
// Delete encrypted object.
bkt := client.Bucket(bucketName)
err := bkt.Object("csek.json").Delete(ctx)
if err != nil {
log.Printf("failed to deleted encrypted file: %v", err)
Expand All @@ -2102,7 +2106,7 @@ func TestIntegration_SignedURL_WithEncryptionKeys(t *testing.T) {
opts.PrivateKey = jwtConf.PrivateKey
opts.Expires = time.Now().Add(time.Hour)

u, err := SignedURL(bucketName, "csek.json", test.opts)
u, err := bkt.SignedURL("csek.json", test.opts)
if err != nil {
t.Fatalf("%s: %v", test.desc, err)
}
Expand Down Expand Up @@ -2152,7 +2156,8 @@ func TestIntegration_SignedURL_EmptyStringObjectName(t *testing.T) {
Expires: time.Now().Add(time.Hour),
}

u, err := SignedURL(bucketName, "", opts)
bkt := client.Bucket(bucketName)
u, err := bkt.SignedURL("", opts)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -4241,6 +4246,110 @@ func TestIntegration_Scopes(t *testing.T) {

}

func TestBucketSignURL(t *testing.T) {
ctx := context.Background()

if testing.Short() && !replaying {
t.Skip("Integration tests skipped in short mode")
}

// We explictly send the key to the client to sign with the private key
clientWithCredentials := newTestClientWithExplicitCredentials(ctx, t)
defer clientWithCredentials.Close()

// Create another client to test the sign byte function as well
clientWithoutPrivateKey := testConfig(ctx, t, ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform")
defer clientWithoutPrivateKey.Close()

jwt, err := testutil.JWTConfig()
if err != nil {
t.Fatalf("unable to find test credentials: %v", err)
}

// We can use any client to create the object
obj := "testBucketSignedURL"
contents := []byte("test")
if err := writeObject(ctx, clientWithoutPrivateKey.Bucket(bucketName).Object(obj), "text/plain", contents); err != nil {
t.Fatalf("writing: %v", err)
}

for _, test := range []struct {
desc string
opts SignedURLOptions
client *Client
}{
{
desc: "signing with the private key",
opts: SignedURLOptions{
Method: "GET",
Expires: time.Now().Add(30 * time.Second),
},
client: clientWithCredentials,
},
{
desc: "signing with the default sign bytes func",
opts: SignedURLOptions{
Method: "GET",
Expires: time.Now().Add(30 * time.Second),
GoogleAccessID: jwt.Email,
},
client: clientWithoutPrivateKey,
},
} {
bkt := test.client.Bucket(bucketName)
url, err := bkt.SignedURL(obj, &test.opts)
if err != nil {
t.Fatalf("unable to create signed URL: %v", err)
}
resp, err := http.Get(url)
if err != nil {
t.Fatalf("http.Get(%q) errored: %q", url, err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Fatalf("resp.StatusCode = %v, want 200: %v", resp.StatusCode, err)
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatalf("unable to read resp.Body: %v", err)
}
if !bytes.Equal(b, contents) {
t.Fatalf("got %q, want %q", b, contents)
}
}
}

func newTestClientWithExplicitCredentials(ctx context.Context, t *testing.T) *Client {
// By default we are authed with a token source, so don't have the context to
// read some of the fields from the keyfile
// Here we explictly send the key to the client
creds, err := findTestCredentials(ctx, "GCLOUD_TESTS_GOLANG_KEY", ScopeFullControl, "https://www.googleapis.com/auth/cloud-platform")
if err != nil {
t.Fatalf("unable to find test credentials: %v", err)
}

clientWithCredentials, err := newTestClient(ctx, option.WithCredentials(creds))
if err != nil {
t.Fatalf("NewClient: %v", err)
}
if clientWithCredentials == nil {
t.Skip("Integration tests skipped. See CONTRIBUTING.md for details")
}
return clientWithCredentials
}

func findTestCredentials(ctx context.Context, envVar string, scopes ...string) (*google.Credentials, error) {
key := os.Getenv(envVar)
var opts []option.ClientOption
if len(scopes) > 0 {
opts = append(opts, option.WithScopes(scopes...))
}
if key != "" {
opts = append(opts, option.WithCredentialsFile(key))
}
return transport.Creds(ctx, opts...)
}

type testHelper struct {
t *testing.T
}
Expand Down

0 comments on commit b824c89

Please sign in to comment.