From 83b36567c5540bdb6b0c352ccd9358f30f5fa91b Mon Sep 17 00:00:00 2001 From: Noah Dietz Date: Wed, 25 Aug 2021 10:23:51 -0700 Subject: [PATCH] chore(storage): add proto converters for Object metadata (#4583) --- storage/acl.go | 49 +++++++++++++++++++ storage/acl_test.go | 49 +++++++++++++++++++ storage/reader.go | 2 +- storage/storage.go | 103 ++++++++++++++++++++++++++++++++++++++-- storage/storage_test.go | 95 ++++++++++++++++++++++++++++++++++++ 5 files changed, 293 insertions(+), 5 deletions(-) diff --git a/storage/acl.go b/storage/acl.go index 7855d110ad4..a28b89b10e8 100644 --- a/storage/acl.go +++ b/storage/acl.go @@ -22,6 +22,7 @@ import ( "cloud.google.com/go/internal/trace" "google.golang.org/api/googleapi" raw "google.golang.org/api/storage/v1" + storagepb "google.golang.org/genproto/googleapis/storage/v2" ) // ACLRole is the level of access to grant. @@ -244,6 +245,14 @@ func toObjectACLRules(items []*raw.ObjectAccessControl) []ACLRule { return rs } +func fromProtoToObjectACLRules(items []*storagepb.ObjectAccessControl) []ACLRule { + var rs []ACLRule + for _, item := range items { + rs = append(rs, fromProtoToObjectACLRule(item)) + } + return rs +} + func toBucketACLRules(items []*raw.BucketAccessControl) []ACLRule { var rs []ACLRule for _, item := range items { @@ -263,6 +272,17 @@ func toObjectACLRule(a *raw.ObjectAccessControl) ACLRule { } } +func fromProtoToObjectACLRule(a *storagepb.ObjectAccessControl) ACLRule { + return ACLRule{ + Entity: ACLEntity(a.GetEntity()), + EntityID: a.GetEntityId(), + Role: ACLRole(a.GetRole()), + Domain: a.GetDomain(), + Email: a.GetEmail(), + ProjectTeam: fromProtoToObjectProjectTeam(a.GetProjectTeam()), + } +} + func toBucketACLRule(a *raw.BucketAccessControl) ACLRule { return ACLRule{ Entity: ACLEntity(a.Entity), @@ -285,6 +305,17 @@ func toRawObjectACL(rules []ACLRule) []*raw.ObjectAccessControl { return r } +func toProtoObjectACL(rules []ACLRule) []*storagepb.ObjectAccessControl { + if len(rules) == 0 { + return nil + } + r := make([]*storagepb.ObjectAccessControl, 0, len(rules)) + for _, rule := range rules { + r = append(r, rule.toProtoObjectAccessControl("")) // bucket name unnecessary + } + return r +} + func toRawBucketACL(rules []ACLRule) []*raw.BucketAccessControl { if len(rules) == 0 { return nil @@ -314,6 +345,14 @@ func (r ACLRule) toRawObjectAccessControl(bucket string) *raw.ObjectAccessContro } } +func (r ACLRule) toProtoObjectAccessControl(bucket string) *storagepb.ObjectAccessControl { + return &storagepb.ObjectAccessControl{ + Entity: string(r.Entity), + Role: string(r.Role), + // The other fields are not settable. + } +} + func toBucketProjectTeam(p *raw.BucketAccessControlProjectTeam) *ProjectTeam { if p == nil { return nil @@ -333,3 +372,13 @@ func toObjectProjectTeam(p *raw.ObjectAccessControlProjectTeam) *ProjectTeam { Team: p.Team, } } + +func fromProtoToObjectProjectTeam(p *storagepb.ProjectTeam) *ProjectTeam { + if p == nil { + return nil + } + return &ProjectTeam{ + ProjectNumber: p.GetProjectNumber(), + Team: p.GetTeam(), + } +} diff --git a/storage/acl_test.go b/storage/acl_test.go index 9b725127f6c..cd6eaaa7452 100644 --- a/storage/acl_test.go +++ b/storage/acl_test.go @@ -20,6 +20,7 @@ import ( "testing" "cloud.google.com/go/internal/testutil" + storagepb "google.golang.org/genproto/googleapis/storage/v2" ) func TestSetACL(t *testing.T) { @@ -63,3 +64,51 @@ func TestSetACL(t *testing.T) { } } } + +func TestToProtoObjectACL(t *testing.T) { + for i, tst := range []struct { + rules []ACLRule + want []*storagepb.ObjectAccessControl + }{ + {nil, nil}, + { + rules: []ACLRule{ + {Entity: "foo", Role: "bar", Domain: "do not copy me!", Email: "donotcopy@"}, + {Entity: "bar", Role: "foo", ProjectTeam: &ProjectTeam{ProjectNumber: "1234", Team: "donotcopy"}}, + }, + want: []*storagepb.ObjectAccessControl{ + {Entity: "foo", Role: "bar"}, + {Entity: "bar", Role: "foo"}, + }, + }, + } { + got := toProtoObjectACL(tst.rules) + if diff := testutil.Diff(got, tst.want); diff != "" { + t.Errorf("#%d: got(-),want(+):\n%s", i, diff) + } + } +} + +func TestFromProtoToObjectACLRules(t *testing.T) { + for i, tst := range []struct { + want []ACLRule + acls []*storagepb.ObjectAccessControl + }{ + {nil, nil}, + { + want: []ACLRule{ + {Entity: "foo", Role: "bar", ProjectTeam: &ProjectTeam{ProjectNumber: "1234", Team: "foo"}}, + {Entity: "bar", Role: "foo", EntityID: "baz", Domain: "domain"}, + }, + acls: []*storagepb.ObjectAccessControl{ + {Entity: "foo", Role: "bar", ProjectTeam: &storagepb.ProjectTeam{ProjectNumber: "1234", Team: "foo"}}, + {Entity: "bar", Role: "foo", EntityId: "baz", Domain: "domain"}, + }, + }, + } { + got := fromProtoToObjectACLRules(tst.acls) + if diff := testutil.Diff(got, tst.want); diff != "" { + t.Errorf("#%d: got(-),want(+):\n%s", i, diff) + } + } +} diff --git a/storage/reader.go b/storage/reader.go index e352ba8755a..0f06db0ab1b 100644 --- a/storage/reader.go +++ b/storage/reader.go @@ -446,7 +446,7 @@ func (o *ObjectHandle) newRangeReaderWithGRPC(ctx context.Context, offset, lengt // For now, there are only globally unique buckets, and "_" is the alias // project ID for such buckets. - b := bucket("_", o.bucket) + b := bucketResourceName("_", o.bucket) req := &storagepb.ReadObjectRequest{ Bucket: b, Object: o.object, diff --git a/storage/storage.go b/storage/storage.go index 1b2e25dadd1..fbf437f68d3 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -46,6 +46,9 @@ import ( "google.golang.org/api/option/internaloption" raw "google.golang.org/api/storage/v1" htransport "google.golang.org/api/transport/http" + storagepb "google.golang.org/genproto/googleapis/storage/v2" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" ) // Methods which can be used in signed URLs. @@ -1132,6 +1135,42 @@ func (o *ObjectAttrs) toRawObject(bucket string) *raw.Object { } } +// toProtoObject copies the editable attributes from o to the proto library's Object type. +func (o *ObjectAttrs) toProtoObject(b string) *storagepb.Object { + checksums := &storagepb.ObjectChecksums{Md5Hash: o.MD5} + if o.CRC32C > 0 { + checksums.Crc32C = proto.Uint32(o.CRC32C) + } + + // For now, there are only globally unique buckets, and "_" is the alias + // project ID for such buckets. + b = bucketResourceName("_", b) + + return &storagepb.Object{ + Bucket: b, + Name: o.Name, + EventBasedHold: proto.Bool(o.EventBasedHold), + TemporaryHold: o.TemporaryHold, + ContentType: o.ContentType, + ContentEncoding: o.ContentEncoding, + ContentLanguage: o.ContentLanguage, + CacheControl: o.CacheControl, + ContentDisposition: o.ContentDisposition, + StorageClass: o.StorageClass, + Acl: toProtoObjectACL(o.ACL), + Metadata: o.Metadata, + CreateTime: toProtoTimestamp(o.Created), + CustomTime: toProtoTimestamp(o.CustomTime), + DeleteTime: toProtoTimestamp(o.Deleted), + RetentionExpireTime: toProtoTimestamp(o.RetentionExpirationTime), + UpdateTime: toProtoTimestamp(o.Updated), + KmsKey: o.KMSKeyName, + Generation: o.Generation, + Size: o.Size, + Checksums: checksums, + } +} + // ObjectAttrs represents the metadata for a Google Cloud Storage (GCS) object. type ObjectAttrs struct { // Bucket is the name of the bucket containing this GCS object. @@ -1288,6 +1327,22 @@ func convertTime(t string) time.Time { return r } +func convertProtoTime(t *timestamppb.Timestamp) time.Time { + var r time.Time + if t != nil { + r = t.AsTime() + } + return r +} + +func toProtoTimestamp(t time.Time) *timestamppb.Timestamp { + if t.IsZero() { + return nil + } + + return timestamppb.New(t) +} + func newObject(o *raw.Object) *ObjectAttrs { if o == nil { return nil @@ -1333,6 +1388,40 @@ func newObject(o *raw.Object) *ObjectAttrs { } } +func newObjectFromProto(r *storagepb.WriteObjectResponse) *ObjectAttrs { + o := r.GetResource() + if r == nil || o == nil { + return nil + } + return &ObjectAttrs{ + Bucket: parseBucketName(o.Bucket), + Name: o.Name, + ContentType: o.ContentType, + ContentLanguage: o.ContentLanguage, + CacheControl: o.CacheControl, + EventBasedHold: o.GetEventBasedHold(), + TemporaryHold: o.TemporaryHold, + RetentionExpirationTime: convertProtoTime(o.GetRetentionExpireTime()), + ACL: fromProtoToObjectACLRules(o.GetAcl()), + Owner: o.GetOwner().GetEntity(), + ContentEncoding: o.ContentEncoding, + ContentDisposition: o.ContentDisposition, + Size: int64(o.Size), + MD5: o.GetChecksums().GetMd5Hash(), + CRC32C: o.GetChecksums().GetCrc32C(), + Metadata: o.Metadata, + Generation: o.Generation, + Metageneration: o.Metageneration, + StorageClass: o.StorageClass, + CustomerKeySHA256: o.GetCustomerEncryption().GetKeySha256(), + KMSKeyName: o.GetKmsKey(), + Created: convertProtoTime(o.GetCreateTime()), + Deleted: convertProtoTime(o.GetDeleteTime()), + Updated: convertProtoTime(o.GetUpdateTime()), + CustomTime: convertProtoTime(o.GetCustomTime()), + } +} + // Decode a uint32 encoded in Base64 in big-endian byte order. func decodeUint32(b64 string) (uint32, error) { d, err := base64.StdEncoding.DecodeString(b64) @@ -1687,9 +1776,15 @@ func (c *Client) ServiceAccount(ctx context.Context, projectID string) (string, return res.EmailAddress, nil } -// bucket formats the given project ID and bucket ID into a Bucket resource -// name. This is the format necessary for the gRPC API as it conforms to the -// Resource-oriented design practices in https://google.aip.dev/121. -func bucket(p, b string) string { +// bucketResourceName formats the given project ID and bucketResourceName ID +// into a Bucket resource name. This is the format necessary for the gRPC API as +// it conforms to the Resource-oriented design practices in https://google.aip.dev/121. +func bucketResourceName(p, b string) string { return fmt.Sprintf("projects/%s/buckets/%s", p, b) } + +// parseBucketName strips the leading resource path segment and returns the +// bucket ID, which is the simple Bucket name typical of the v1 API. +func parseBucketName(b string) string { + return strings.TrimPrefix(b, "projects/_/buckets/") +} diff --git a/storage/storage_test.go b/storage/storage_test.go index 4d48f93e512..b13a92812a8 100644 --- a/storage/storage_test.go +++ b/storage/storage_test.go @@ -39,6 +39,9 @@ import ( "google.golang.org/api/iterator" "google.golang.org/api/option" raw "google.golang.org/api/storage/v1" + storagepb "google.golang.org/genproto/googleapis/storage/v2" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" ) func TestV2HeaderSanitization(t *testing.T) { @@ -1216,6 +1219,98 @@ func TestObjectAttrsToRawObject(t *testing.T) { } } +func TestProtoObjectToObjectAttrs(t *testing.T) { + t.Parallel() + now := time.Now() + tests := []struct { + in *storagepb.Object + want *ObjectAttrs + }{ + {in: nil, want: nil}, + { + in: &storagepb.Object{ + Bucket: "Test", + ContentLanguage: "en-us", + ContentType: "video/mpeg", + CustomTime: timestamppb.New(now), + EventBasedHold: proto.Bool(false), + Generation: 7, + Checksums: &storagepb.ObjectChecksums{Md5Hash: []byte("14683cba444dbcc6db297645e683f5c1")}, + Name: "foo.mp4", + RetentionExpireTime: timestamppb.New(now), + Size: 1 << 20, + CreateTime: timestamppb.New(now), + DeleteTime: timestamppb.New(now), + TemporaryHold: true, + }, + want: &ObjectAttrs{ + Bucket: "Test", + Created: now, + ContentLanguage: "en-us", + ContentType: "video/mpeg", + CustomTime: now, + Deleted: now, + EventBasedHold: false, + Generation: 7, + MD5: []byte("14683cba444dbcc6db297645e683f5c1"), + Name: "foo.mp4", + RetentionExpirationTime: now, + Size: 1 << 20, + TemporaryHold: true, + }, + }, + } + + for i, tt := range tests { + r := &storagepb.WriteObjectResponse{WriteStatus: &storagepb.WriteObjectResponse_Resource{Resource: tt.in}} + got := newObjectFromProto(r) + if diff := testutil.Diff(got, tt.want); diff != "" { + t.Errorf("#%d: newObject mismatches:\ngot=-, want=+:\n%s", i, diff) + } + } +} + +func TestObjectAttrsToProtoObject(t *testing.T) { + t.Parallel() + now := time.Now() + + b := "bucket" + want := &storagepb.Object{ + Bucket: "projects/_/buckets/" + b, + ContentLanguage: "en-us", + ContentType: "video/mpeg", + CustomTime: timestamppb.New(now), + EventBasedHold: proto.Bool(false), + Generation: 7, + Checksums: &storagepb.ObjectChecksums{Md5Hash: []byte("14683cba444dbcc6db297645e683f5c1")}, + Name: "foo.mp4", + RetentionExpireTime: timestamppb.New(now), + Size: 1 << 20, + CreateTime: timestamppb.New(now), + DeleteTime: timestamppb.New(now), + TemporaryHold: true, + } + in := &ObjectAttrs{ + Created: now, + ContentLanguage: "en-us", + ContentType: "video/mpeg", + CustomTime: now, + Deleted: now, + EventBasedHold: false, + Generation: 7, + MD5: []byte("14683cba444dbcc6db297645e683f5c1"), + Name: "foo.mp4", + RetentionExpirationTime: now, + Size: 1 << 20, + TemporaryHold: true, + } + + got := in.toProtoObject(b) + if diff := testutil.Diff(got, want); diff != "" { + t.Errorf("toProtoObject mismatches:\ngot=-, want=+:\n%s", diff) + } +} + func TestAttrToFieldMapCoverage(t *testing.T) { t.Parallel()