feat(storage): support for soft delete policies and restore (#9520)

storage/bucket.go

@@ -479,6 +479,13 @@ type BucketAttrs struct {
 	// cannot be modified once the bucket is created.
 	// ObjectRetention cannot be configured or reported through the gRPC API.
 	ObjectRetentionMode string
+
+	// SoftDeletePolicy contains the bucket's soft delete policy, which defines
+	// the period of time that soft-deleted objects will be retained, and cannot
+	// be permanently deleted. By default, new buckets will be created with a
+	// 7 day retention duration. In order to fully disable soft delete, you need
+	// to set a policy with a RetentionDuration of 0.
+	SoftDeletePolicy *SoftDeletePolicy
 }
 
 // BucketPolicyOnly is an alias for UniformBucketLevelAccess.
@@ -766,6 +773,19 @@ type Autoclass struct {
 	TerminalStorageClassUpdateTime time.Time
 }
 
+// SoftDeletePolicy contains the bucket's soft delete policy, which defines the
+// period of time that soft-deleted objects will be retained, and cannot be
+// permanently deleted.
+type SoftDeletePolicy struct {
+	// EffectiveTime indicates the time from which the policy, or one with a
+	// greater retention, was effective. This field is read-only.
+	EffectiveTime time.Time
+
+	// RetentionDuration is the amount of time that soft-deleted objects in the
+	// bucket will be retained and cannot be permanently deleted.
+	RetentionDuration time.Duration
+}
+
 func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
 	if b == nil {
 		return nil, nil
@@ -803,6 +823,7 @@ func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
 		RPO:                      toRPO(b),
 		CustomPlacementConfig:    customPlacementFromRaw(b.CustomPlacementConfig),
 		Autoclass:                toAutoclassFromRaw(b.Autoclass),
+		SoftDeletePolicy:         toSoftDeletePolicyFromRaw(b.SoftDeletePolicy),
 	}, nil
 }
 
@@ -836,6 +857,7 @@ func newBucketFromProto(b *storagepb.Bucket) *BucketAttrs {
 		CustomPlacementConfig:    customPlacementFromProto(b.GetCustomPlacementConfig()),
 		ProjectNumber:            parseProjectNumber(b.GetProject()), // this can return 0 the project resource name is ID based
 		Autoclass:                toAutoclassFromProto(b.GetAutoclass()),
+		SoftDeletePolicy:         toSoftDeletePolicyFromProto(b.SoftDeletePolicy),
 	}
 }
 
@@ -891,6 +913,7 @@ func (b *BucketAttrs) toRawBucket() *raw.Bucket {
 		Rpo:                   b.RPO.String(),
 		CustomPlacementConfig: b.CustomPlacementConfig.toRawCustomPlacement(),
 		Autoclass:             b.Autoclass.toRawAutoclass(),
+		SoftDeletePolicy:      b.SoftDeletePolicy.toRawSoftDeletePolicy(),
 	}
 }
 
@@ -951,6 +974,7 @@ func (b *BucketAttrs) toProtoBucket() *storagepb.Bucket {
 		Rpo:                   b.RPO.String(),
 		CustomPlacementConfig: b.CustomPlacementConfig.toProtoCustomPlacement(),
 		Autoclass:             b.Autoclass.toProtoAutoclass(),
+		SoftDeletePolicy:      b.SoftDeletePolicy.toProtoSoftDeletePolicy(),
 	}
 }
 
@@ -1032,6 +1056,7 @@ func (ua *BucketAttrsToUpdate) toProtoBucket() *storagepb.Bucket {
 		IamConfig:             bktIAM,
 		Rpo:                   ua.RPO.String(),
 		Autoclass:             ua.Autoclass.toProtoAutoclass(),
+		SoftDeletePolicy:      ua.SoftDeletePolicy.toProtoSoftDeletePolicy(),
 		Labels:                ua.setLabels,
 	}
 }
@@ -1152,6 +1177,9 @@ type BucketAttrsToUpdate struct {
 	// See https://cloud.google.com/storage/docs/using-autoclass for more information.
 	Autoclass *Autoclass
 
+	// If set, updates the soft delete policy of the bucket.
+	SoftDeletePolicy *SoftDeletePolicy
+
 	// acl is the list of access control rules on the bucket.
 	// It is unexported and only used internally by the gRPC client.
 	// Library users should use ACLHandle methods directly.
@@ -1273,6 +1301,14 @@ func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket {
 		}
 		rb.ForceSendFields = append(rb.ForceSendFields, "Autoclass")
 	}
+	if ua.SoftDeletePolicy != nil {
+		if ua.SoftDeletePolicy.RetentionDuration == 0 {
+			rb.NullFields = append(rb.NullFields, "SoftDeletePolicy")
+			rb.SoftDeletePolicy = nil
+		} else {
+			rb.SoftDeletePolicy = ua.SoftDeletePolicy.toRawSoftDeletePolicy()
+		}
+	}
 	if ua.PredefinedACL != "" {
 		// Clear ACL or the call will fail.
 		rb.Acl = nil
@@ -2053,6 +2089,53 @@ func toAutoclassFromProto(a *storagepb.Bucket_Autoclass) *Autoclass {
 	}
 }
 
+func (p *SoftDeletePolicy) toRawSoftDeletePolicy() *raw.BucketSoftDeletePolicy {
+	if p == nil {
+		return nil
+	}
+	// Excluding read only field EffectiveTime.
+	return &raw.BucketSoftDeletePolicy{
+		RetentionDurationSeconds: int64(p.RetentionDuration.Seconds()),
+	}
+}
+
+func (p *SoftDeletePolicy) toProtoSoftDeletePolicy() *storagepb.Bucket_SoftDeletePolicy {
+	if p == nil {
+		return nil
+	}
+	// Excluding read only field EffectiveTime.
+	return &storagepb.Bucket_SoftDeletePolicy{
+		RetentionDuration: durationpb.New(p.RetentionDuration),
+	}
+}
+
+func toSoftDeletePolicyFromRaw(p *raw.BucketSoftDeletePolicy) *SoftDeletePolicy {
+	if p == nil {
+		return nil
+	}
+
+	policy := &SoftDeletePolicy{
+		RetentionDuration: time.Duration(p.RetentionDurationSeconds) * time.Second,
+	}
+
+	// Return EffectiveTime only if parsed to a valid value.
+	if t, err := time.Parse(time.RFC3339, p.EffectiveTime); err == nil {
+		policy.EffectiveTime = t
+	}
+
+	return policy
+}
+
+func toSoftDeletePolicyFromProto(p *storagepb.Bucket_SoftDeletePolicy) *SoftDeletePolicy {
+	if p == nil {
+		return nil
+	}
+	return &SoftDeletePolicy{
+		EffectiveTime:     p.GetEffectiveTime().AsTime(),
+		RetentionDuration: p.GetRetentionDuration().AsDuration(),
+	}
+}
+
 // Objects returns an iterator over the objects in the bucket that match the
 // Query q. If q is nil, no filtering is done. Objects will be iterated over
 // lexicographically by name.

storage/bucket_test.go

@@ -62,10 +62,11 @@ func TestBucketAttrsToRawBucket(t *testing.T) {
 				ResponseHeaders: []string{"FOO"},
 			},
 		},
-		Encryption: &BucketEncryption{DefaultKMSKeyName: "key"},
-		Logging:    &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
-		Website:    &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
-		Autoclass:  &Autoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
+		Encryption:       &BucketEncryption{DefaultKMSKeyName: "key"},
+		Logging:          &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+		Website:          &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+		Autoclass:        &Autoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
+		SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour},
 		Lifecycle: Lifecycle{
 			Rules: []LifecycleRule{{
 				Action: LifecycleAction{
@@ -166,10 +167,11 @@ func TestBucketAttrsToRawBucket(t *testing.T) {
 				ResponseHeader: []string{"FOO"},
 			},
 		},
-		Encryption: &raw.BucketEncryption{DefaultKmsKeyName: "key"},
-		Logging:    &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
-		Website:    &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
-		Autoclass:  &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
+		Encryption:       &raw.BucketEncryption{DefaultKmsKeyName: "key"},
+		Logging:          &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+		Website:          &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+		Autoclass:        &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
+		SoftDeletePolicy: &raw.BucketSoftDeletePolicy{RetentionDurationSeconds: 60 * 60},
 		Lifecycle: &raw.BucketLifecycle{
 			Rule: []*raw.BucketLifecycleRule{{
 				Action: &raw.BucketLifecycleRuleAction{
@@ -395,10 +397,11 @@ func TestBucketAttrsToUpdateToRawBucket(t *testing.T) {
 				},
 			},
 		},
-		Logging:      &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
-		Website:      &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
-		StorageClass: "NEARLINE",
-		Autoclass:    &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
+		Logging:          &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+		Website:          &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+		StorageClass:     "NEARLINE",
+		Autoclass:        &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
+		SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour},
 	}
 	au.SetLabel("a", "foo")
 	au.DeleteLabel("b")
@@ -439,11 +442,12 @@ func TestBucketAttrsToUpdateToRawBucket(t *testing.T) {
 				},
 			},
 		},
-		Logging:         &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
-		Website:         &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
-		StorageClass:    "NEARLINE",
-		Autoclass:       &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "ARCHIVE", ForceSendFields: []string{"Enabled"}},
-		ForceSendFields: []string{"DefaultEventBasedHold", "Lifecycle", "Autoclass"},
+		Logging:          &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+		Website:          &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+		StorageClass:     "NEARLINE",
+		Autoclass:        &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "ARCHIVE", ForceSendFields: []string{"Enabled"}},
+		SoftDeletePolicy: &raw.BucketSoftDeletePolicy{RetentionDurationSeconds: 3600},
+		ForceSendFields:  []string{"DefaultEventBasedHold", "Lifecycle", "Autoclass"},
 	}
 	if msg := testutil.Diff(got, want); msg != "" {
 		t.Error(msg)
@@ -463,14 +467,15 @@ func TestBucketAttrsToUpdateToRawBucket(t *testing.T) {
 
 	// Test nulls.
 	au3 := &BucketAttrsToUpdate{
-		RetentionPolicy: &RetentionPolicy{},
-		Encryption:      &BucketEncryption{},
-		Logging:         &BucketLogging{},
-		Website:         &BucketWebsite{},
+		RetentionPolicy:  &RetentionPolicy{},
+		Encryption:       &BucketEncryption{},
+		Logging:          &BucketLogging{},
+		Website:          &BucketWebsite{},
+		SoftDeletePolicy: &SoftDeletePolicy{},
 	}
 	got = au3.toRawBucket()
 	want = &raw.Bucket{
-		NullFields: []string{"RetentionPolicy", "Encryption", "Logging", "Website"},
+		NullFields: []string{"RetentionPolicy", "Encryption", "Logging", "Website", "SoftDeletePolicy"},
 	}
 	if msg := testutil.Diff(got, want); msg != "" {
 		t.Error(msg)
@@ -656,6 +661,10 @@ func TestNewBucket(t *testing.T) {
 			TerminalStorageClass:           "NEARLINE",
 			TerminalStorageClassUpdateTime: "2017-10-23T04:05:06Z",
 		},
+		SoftDeletePolicy: &raw.BucketSoftDeletePolicy{
+			EffectiveTime:            "2017-10-23T04:05:06Z",
+			RetentionDurationSeconds: 3600,
+		},
 	}
 	want := &BucketAttrs{
 		Name:                  "name",
@@ -713,6 +722,10 @@ func TestNewBucket(t *testing.T) {
 			TerminalStorageClass:           "NEARLINE",
 			TerminalStorageClassUpdateTime: time.Date(2017, 10, 23, 4, 5, 6, 0, time.UTC),
 		},
+		SoftDeletePolicy: &SoftDeletePolicy{
+			EffectiveTime:     time.Date(2017, 10, 23, 4, 5, 6, 0, time.UTC),
+			RetentionDuration: time.Hour,
+		},
 	}
 	got, err := newBucket(rb)
 	if err != nil {
@@ -768,6 +781,10 @@ func TestNewBucketFromProto(t *testing.T) {
 			TerminalStorageClass:           &autoclassTSC,
 			TerminalStorageClassUpdateTime: toProtoTimestamp(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)),
 		},
+		SoftDeletePolicy: &storagepb.Bucket_SoftDeletePolicy{
+			RetentionDuration: durationpb.New(3 * time.Hour),
+			EffectiveTime:     toProtoTimestamp(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)),
+		},
 		Lifecycle: &storagepb.Bucket_Lifecycle{
 			Rule: []*storagepb.Bucket_Lifecycle_Rule{
 				{
@@ -809,6 +826,10 @@ func TestNewBucketFromProto(t *testing.T) {
 		Logging:    &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
 		Website:    &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
 		Autoclass:  &Autoclass{Enabled: true, ToggleTime: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), TerminalStorageClass: "NEARLINE", TerminalStorageClassUpdateTime: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)},
+		SoftDeletePolicy: &SoftDeletePolicy{
+			EffectiveTime:     time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC),
+			RetentionDuration: time.Hour * 3,
+		},
 		Lifecycle: Lifecycle{
 			Rules: []LifecycleRule{{
 				Action: LifecycleAction{
@@ -853,10 +874,11 @@ func TestBucketAttrsToProtoBucket(t *testing.T) {
 				ResponseHeaders: []string{"FOO"},
 			},
 		},
-		Encryption: &BucketEncryption{DefaultKMSKeyName: "key"},
-		Logging:    &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
-		Website:    &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
-		Autoclass:  &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
+		Encryption:       &BucketEncryption{DefaultKMSKeyName: "key"},
+		Logging:          &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+		Website:          &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+		Autoclass:        &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
+		SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour * 2},
 		Lifecycle: Lifecycle{
 			Rules: []LifecycleRule{{
 				Action: LifecycleAction{
@@ -903,10 +925,11 @@ func TestBucketAttrsToProtoBucket(t *testing.T) {
 				ResponseHeader: []string{"FOO"},
 			},
 		},
-		Encryption: &storagepb.Bucket_Encryption{DefaultKmsKey: "key"},
-		Logging:    &storagepb.Bucket_Logging{LogBucket: "projects/_/buckets/lb", LogObjectPrefix: "p"},
-		Website:    &storagepb.Bucket_Website{MainPageSuffix: "mps", NotFoundPage: "404"},
-		Autoclass:  &storagepb.Bucket_Autoclass{Enabled: true, TerminalStorageClass: &autoclassTSC},
+		Encryption:       &storagepb.Bucket_Encryption{DefaultKmsKey: "key"},
+		Logging:          &storagepb.Bucket_Logging{LogBucket: "projects/_/buckets/lb", LogObjectPrefix: "p"},
+		Website:          &storagepb.Bucket_Website{MainPageSuffix: "mps", NotFoundPage: "404"},
+		Autoclass:        &storagepb.Bucket_Autoclass{Enabled: true, TerminalStorageClass: &autoclassTSC},
+		SoftDeletePolicy: &storagepb.Bucket_SoftDeletePolicy{RetentionDuration: durationpb.New(2 * time.Hour)},
 		Lifecycle: &storagepb.Bucket_Lifecycle{
 			Rule: []*storagepb.Bucket_Lifecycle_Rule{
 				{

storage/client.go

@@ -59,8 +59,9 @@ type storageClient interface {
 	// Object metadata methods.
 
 	DeleteObject(ctx context.Context, bucket, object string, gen int64, conds *Conditions, opts ...storageOption) error
-	GetObject(ctx context.Context, bucket, object string, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error)
+	GetObject(ctx context.Context, params *getObjectParams, opts ...storageOption) (*ObjectAttrs, error)
 	UpdateObject(ctx context.Context, params *updateObjectParams, opts ...storageOption) (*ObjectAttrs, error)
+	RestoreObject(ctx context.Context, params *restoreObjectParams, opts ...storageOption) (*ObjectAttrs, error)
 
 	// Default Object ACL methods.
 
@@ -294,6 +295,14 @@ type newRangeReaderParams struct {
 	readCompressed bool // Use accept-encoding: gzip. Only works for HTTP currently.
 }
 
+type getObjectParams struct {
+	bucket, object string
+	gen            int64
+	encryptionKey  []byte
+	conds          *Conditions
+	softDeleted    bool
+}
+
 type updateObjectParams struct {
 	bucket, object    string
 	uattrs            *ObjectAttrsToUpdate
@@ -303,6 +312,14 @@ type updateObjectParams struct {
 	overrideRetention *bool
 }
 
+type restoreObjectParams struct {
+	bucket, object string
+	gen            int64
+	encryptionKey  []byte
+	conds          *Conditions
+	copySourceACL  bool
+}
+
 type composeObjectRequest struct {
 	dstBucket     string
 	dstObject     destinationObject

storage/client_test.go

@@ -272,7 +272,7 @@ func TestGetObjectEmulated(t *testing.T) {
 		if err := w.Close(); err != nil {
 			t.Fatalf("closing object: %v", err)
 		}
-		got, err := client.GetObject(context.Background(), bucket, want.Name, defaultGen, nil, nil)
+		got, err := client.GetObject(context.Background(), &getObjectParams{bucket: bucket, object: want.Name, gen: defaultGen})
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -1332,7 +1332,7 @@ func TestObjectConditionsEmulated(t *testing.T) {
 					if err != nil {
 						return fmt.Errorf("creating object: %w", err)
 					}
-					_, err = client.GetObject(ctx, bucket, objName, gen, nil, &Conditions{GenerationMatch: gen, MetagenerationMatch: metaGen})
+					_, err = client.GetObject(ctx, &getObjectParams{bucket: bucket, object: objName, gen: gen, conds: &Conditions{GenerationMatch: gen, MetagenerationMatch: metaGen}})
 					return err
 				},
 			},