feat(storage): support for soft delete policies and restore (#9520)
diff --git a/storage/bucket.go b/storage/bucket.go
index 0344ef9..d2da86e 100644
--- a/storage/bucket.go
+++ b/storage/bucket.go
@@ -479,6 +479,13 @@
// cannot be modified once the bucket is created.
// ObjectRetention cannot be configured or reported through the gRPC API.
ObjectRetentionMode string
+
+ // SoftDeletePolicy contains the bucket's soft delete policy, which defines
+ // the period of time that soft-deleted objects will be retained, and cannot
+ // be permanently deleted. By default, new buckets will be created with a
+ // 7 day retention duration. In order to fully disable soft delete, you need
+ // to set a policy with a RetentionDuration of 0.
+ SoftDeletePolicy *SoftDeletePolicy
}
// BucketPolicyOnly is an alias for UniformBucketLevelAccess.
@@ -766,6 +773,19 @@
TerminalStorageClassUpdateTime time.Time
}
+// SoftDeletePolicy contains the bucket's soft delete policy, which defines the
+// period of time that soft-deleted objects will be retained, and cannot be
+// permanently deleted.
+type SoftDeletePolicy struct {
+ // EffectiveTime indicates the time from which the policy, or one with a
+ // greater retention, was effective. This field is read-only.
+ EffectiveTime time.Time
+
+ // RetentionDuration is the amount of time that soft-deleted objects in the
+ // bucket will be retained and cannot be permanently deleted.
+ RetentionDuration time.Duration
+}
+
func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
if b == nil {
return nil, nil
@@ -803,6 +823,7 @@
RPO: toRPO(b),
CustomPlacementConfig: customPlacementFromRaw(b.CustomPlacementConfig),
Autoclass: toAutoclassFromRaw(b.Autoclass),
+ SoftDeletePolicy: toSoftDeletePolicyFromRaw(b.SoftDeletePolicy),
}, nil
}
@@ -836,6 +857,7 @@
CustomPlacementConfig: customPlacementFromProto(b.GetCustomPlacementConfig()),
ProjectNumber: parseProjectNumber(b.GetProject()), // this can return 0 the project resource name is ID based
Autoclass: toAutoclassFromProto(b.GetAutoclass()),
+ SoftDeletePolicy: toSoftDeletePolicyFromProto(b.SoftDeletePolicy),
}
}
@@ -891,6 +913,7 @@
Rpo: b.RPO.String(),
CustomPlacementConfig: b.CustomPlacementConfig.toRawCustomPlacement(),
Autoclass: b.Autoclass.toRawAutoclass(),
+ SoftDeletePolicy: b.SoftDeletePolicy.toRawSoftDeletePolicy(),
}
}
@@ -951,6 +974,7 @@
Rpo: b.RPO.String(),
CustomPlacementConfig: b.CustomPlacementConfig.toProtoCustomPlacement(),
Autoclass: b.Autoclass.toProtoAutoclass(),
+ SoftDeletePolicy: b.SoftDeletePolicy.toProtoSoftDeletePolicy(),
}
}
@@ -1032,6 +1056,7 @@
IamConfig: bktIAM,
Rpo: ua.RPO.String(),
Autoclass: ua.Autoclass.toProtoAutoclass(),
+ SoftDeletePolicy: ua.SoftDeletePolicy.toProtoSoftDeletePolicy(),
Labels: ua.setLabels,
}
}
@@ -1152,6 +1177,9 @@
// See https://cloud.google.com/storage/docs/using-autoclass for more information.
Autoclass *Autoclass
+ // If set, updates the soft delete policy of the bucket.
+ SoftDeletePolicy *SoftDeletePolicy
+
// acl is the list of access control rules on the bucket.
// It is unexported and only used internally by the gRPC client.
// Library users should use ACLHandle methods directly.
@@ -1273,6 +1301,14 @@
}
rb.ForceSendFields = append(rb.ForceSendFields, "Autoclass")
}
+ if ua.SoftDeletePolicy != nil {
+ if ua.SoftDeletePolicy.RetentionDuration == 0 {
+ rb.NullFields = append(rb.NullFields, "SoftDeletePolicy")
+ rb.SoftDeletePolicy = nil
+ } else {
+ rb.SoftDeletePolicy = ua.SoftDeletePolicy.toRawSoftDeletePolicy()
+ }
+ }
if ua.PredefinedACL != "" {
// Clear ACL or the call will fail.
rb.Acl = nil
@@ -2053,6 +2089,53 @@
}
}
+func (p *SoftDeletePolicy) toRawSoftDeletePolicy() *raw.BucketSoftDeletePolicy {
+ if p == nil {
+ return nil
+ }
+ // Excluding read only field EffectiveTime.
+ return &raw.BucketSoftDeletePolicy{
+ RetentionDurationSeconds: int64(p.RetentionDuration.Seconds()),
+ }
+}
+
+func (p *SoftDeletePolicy) toProtoSoftDeletePolicy() *storagepb.Bucket_SoftDeletePolicy {
+ if p == nil {
+ return nil
+ }
+ // Excluding read only field EffectiveTime.
+ return &storagepb.Bucket_SoftDeletePolicy{
+ RetentionDuration: durationpb.New(p.RetentionDuration),
+ }
+}
+
+func toSoftDeletePolicyFromRaw(p *raw.BucketSoftDeletePolicy) *SoftDeletePolicy {
+ if p == nil {
+ return nil
+ }
+
+ policy := &SoftDeletePolicy{
+ RetentionDuration: time.Duration(p.RetentionDurationSeconds) * time.Second,
+ }
+
+ // Return EffectiveTime only if parsed to a valid value.
+ if t, err := time.Parse(time.RFC3339, p.EffectiveTime); err == nil {
+ policy.EffectiveTime = t
+ }
+
+ return policy
+}
+
+func toSoftDeletePolicyFromProto(p *storagepb.Bucket_SoftDeletePolicy) *SoftDeletePolicy {
+ if p == nil {
+ return nil
+ }
+ return &SoftDeletePolicy{
+ EffectiveTime: p.GetEffectiveTime().AsTime(),
+ RetentionDuration: p.GetRetentionDuration().AsDuration(),
+ }
+}
+
// Objects returns an iterator over the objects in the bucket that match the
// Query q. If q is nil, no filtering is done. Objects will be iterated over
// lexicographically by name.
diff --git a/storage/bucket_test.go b/storage/bucket_test.go
index 7e8b182..71d5c89 100644
--- a/storage/bucket_test.go
+++ b/storage/bucket_test.go
@@ -62,10 +62,11 @@
ResponseHeaders: []string{"FOO"},
},
},
- Encryption: &BucketEncryption{DefaultKMSKeyName: "key"},
- Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
- Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
- Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
+ Encryption: &BucketEncryption{DefaultKMSKeyName: "key"},
+ Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+ Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+ Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
+ SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour},
Lifecycle: Lifecycle{
Rules: []LifecycleRule{{
Action: LifecycleAction{
@@ -166,10 +167,11 @@
ResponseHeader: []string{"FOO"},
},
},
- Encryption: &raw.BucketEncryption{DefaultKmsKeyName: "key"},
- Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
- Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
- Autoclass: &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
+ Encryption: &raw.BucketEncryption{DefaultKmsKeyName: "key"},
+ Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+ Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+ Autoclass: &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
+ SoftDeletePolicy: &raw.BucketSoftDeletePolicy{RetentionDurationSeconds: 60 * 60},
Lifecycle: &raw.BucketLifecycle{
Rule: []*raw.BucketLifecycleRule{{
Action: &raw.BucketLifecycleRuleAction{
@@ -395,10 +397,11 @@
},
},
},
- Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
- Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
- StorageClass: "NEARLINE",
- Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
+ Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+ Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+ StorageClass: "NEARLINE",
+ Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
+ SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour},
}
au.SetLabel("a", "foo")
au.DeleteLabel("b")
@@ -439,11 +442,12 @@
},
},
},
- Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
- Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
- StorageClass: "NEARLINE",
- Autoclass: &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "ARCHIVE", ForceSendFields: []string{"Enabled"}},
- ForceSendFields: []string{"DefaultEventBasedHold", "Lifecycle", "Autoclass"},
+ Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+ Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+ StorageClass: "NEARLINE",
+ Autoclass: &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "ARCHIVE", ForceSendFields: []string{"Enabled"}},
+ SoftDeletePolicy: &raw.BucketSoftDeletePolicy{RetentionDurationSeconds: 3600},
+ ForceSendFields: []string{"DefaultEventBasedHold", "Lifecycle", "Autoclass"},
}
if msg := testutil.Diff(got, want); msg != "" {
t.Error(msg)
@@ -463,14 +467,15 @@
// Test nulls.
au3 := &BucketAttrsToUpdate{
- RetentionPolicy: &RetentionPolicy{},
- Encryption: &BucketEncryption{},
- Logging: &BucketLogging{},
- Website: &BucketWebsite{},
+ RetentionPolicy: &RetentionPolicy{},
+ Encryption: &BucketEncryption{},
+ Logging: &BucketLogging{},
+ Website: &BucketWebsite{},
+ SoftDeletePolicy: &SoftDeletePolicy{},
}
got = au3.toRawBucket()
want = &raw.Bucket{
- NullFields: []string{"RetentionPolicy", "Encryption", "Logging", "Website"},
+ NullFields: []string{"RetentionPolicy", "Encryption", "Logging", "Website", "SoftDeletePolicy"},
}
if msg := testutil.Diff(got, want); msg != "" {
t.Error(msg)
@@ -656,6 +661,10 @@
TerminalStorageClass: "NEARLINE",
TerminalStorageClassUpdateTime: "2017-10-23T04:05:06Z",
},
+ SoftDeletePolicy: &raw.BucketSoftDeletePolicy{
+ EffectiveTime: "2017-10-23T04:05:06Z",
+ RetentionDurationSeconds: 3600,
+ },
}
want := &BucketAttrs{
Name: "name",
@@ -713,6 +722,10 @@
TerminalStorageClass: "NEARLINE",
TerminalStorageClassUpdateTime: time.Date(2017, 10, 23, 4, 5, 6, 0, time.UTC),
},
+ SoftDeletePolicy: &SoftDeletePolicy{
+ EffectiveTime: time.Date(2017, 10, 23, 4, 5, 6, 0, time.UTC),
+ RetentionDuration: time.Hour,
+ },
}
got, err := newBucket(rb)
if err != nil {
@@ -768,6 +781,10 @@
TerminalStorageClass: &autoclassTSC,
TerminalStorageClassUpdateTime: toProtoTimestamp(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)),
},
+ SoftDeletePolicy: &storagepb.Bucket_SoftDeletePolicy{
+ RetentionDuration: durationpb.New(3 * time.Hour),
+ EffectiveTime: toProtoTimestamp(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)),
+ },
Lifecycle: &storagepb.Bucket_Lifecycle{
Rule: []*storagepb.Bucket_Lifecycle_Rule{
{
@@ -809,6 +826,10 @@
Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
Autoclass: &Autoclass{Enabled: true, ToggleTime: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), TerminalStorageClass: "NEARLINE", TerminalStorageClassUpdateTime: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)},
+ SoftDeletePolicy: &SoftDeletePolicy{
+ EffectiveTime: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC),
+ RetentionDuration: time.Hour * 3,
+ },
Lifecycle: Lifecycle{
Rules: []LifecycleRule{{
Action: LifecycleAction{
@@ -853,10 +874,11 @@
ResponseHeaders: []string{"FOO"},
},
},
- Encryption: &BucketEncryption{DefaultKMSKeyName: "key"},
- Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
- Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
- Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
+ Encryption: &BucketEncryption{DefaultKMSKeyName: "key"},
+ Logging: &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+ Website: &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+ Autoclass: &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
+ SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour * 2},
Lifecycle: Lifecycle{
Rules: []LifecycleRule{{
Action: LifecycleAction{
@@ -903,10 +925,11 @@
ResponseHeader: []string{"FOO"},
},
},
- Encryption: &storagepb.Bucket_Encryption{DefaultKmsKey: "key"},
- Logging: &storagepb.Bucket_Logging{LogBucket: "projects/_/buckets/lb", LogObjectPrefix: "p"},
- Website: &storagepb.Bucket_Website{MainPageSuffix: "mps", NotFoundPage: "404"},
- Autoclass: &storagepb.Bucket_Autoclass{Enabled: true, TerminalStorageClass: &autoclassTSC},
+ Encryption: &storagepb.Bucket_Encryption{DefaultKmsKey: "key"},
+ Logging: &storagepb.Bucket_Logging{LogBucket: "projects/_/buckets/lb", LogObjectPrefix: "p"},
+ Website: &storagepb.Bucket_Website{MainPageSuffix: "mps", NotFoundPage: "404"},
+ Autoclass: &storagepb.Bucket_Autoclass{Enabled: true, TerminalStorageClass: &autoclassTSC},
+ SoftDeletePolicy: &storagepb.Bucket_SoftDeletePolicy{RetentionDuration: durationpb.New(2 * time.Hour)},
Lifecycle: &storagepb.Bucket_Lifecycle{
Rule: []*storagepb.Bucket_Lifecycle_Rule{
{
diff --git a/storage/client.go b/storage/client.go
index 70b2a28..187c3ec 100644
--- a/storage/client.go
+++ b/storage/client.go
@@ -59,8 +59,9 @@
// Object metadata methods.
DeleteObject(ctx context.Context, bucket, object string, gen int64, conds *Conditions, opts ...storageOption) error
- GetObject(ctx context.Context, bucket, object string, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error)
+ GetObject(ctx context.Context, params *getObjectParams, opts ...storageOption) (*ObjectAttrs, error)
UpdateObject(ctx context.Context, params *updateObjectParams, opts ...storageOption) (*ObjectAttrs, error)
+ RestoreObject(ctx context.Context, params *restoreObjectParams, opts ...storageOption) (*ObjectAttrs, error)
// Default Object ACL methods.
@@ -294,6 +295,14 @@
readCompressed bool // Use accept-encoding: gzip. Only works for HTTP currently.
}
+type getObjectParams struct {
+ bucket, object string
+ gen int64
+ encryptionKey []byte
+ conds *Conditions
+ softDeleted bool
+}
+
type updateObjectParams struct {
bucket, object string
uattrs *ObjectAttrsToUpdate
@@ -303,6 +312,14 @@
overrideRetention *bool
}
+type restoreObjectParams struct {
+ bucket, object string
+ gen int64
+ encryptionKey []byte
+ conds *Conditions
+ copySourceACL bool
+}
+
type composeObjectRequest struct {
dstBucket string
dstObject destinationObject
diff --git a/storage/client_test.go b/storage/client_test.go
index 4124c22..af8fc4c 100644
--- a/storage/client_test.go
+++ b/storage/client_test.go
@@ -272,7 +272,7 @@
if err := w.Close(); err != nil {
t.Fatalf("closing object: %v", err)
}
- got, err := client.GetObject(context.Background(), bucket, want.Name, defaultGen, nil, nil)
+ got, err := client.GetObject(context.Background(), &getObjectParams{bucket: bucket, object: want.Name, gen: defaultGen})
if err != nil {
t.Fatal(err)
}
@@ -1332,7 +1332,7 @@
if err != nil {
return fmt.Errorf("creating object: %w", err)
}
- _, err = client.GetObject(ctx, bucket, objName, gen, nil, &Conditions{GenerationMatch: gen, MetagenerationMatch: metaGen})
+ _, err = client.GetObject(ctx, &getObjectParams{bucket: bucket, object: objName, gen: gen, conds: &Conditions{GenerationMatch: gen, MetagenerationMatch: metaGen}})
return err
},
},
diff --git a/storage/grpc_client.go b/storage/grpc_client.go
index 1f34dff..c5e2e39 100644
--- a/storage/grpc_client.go
+++ b/storage/grpc_client.go
@@ -367,6 +367,9 @@
if uattrs.Autoclass != nil {
fieldMask.Paths = append(fieldMask.Paths, "autoclass")
}
+ if uattrs.SoftDeletePolicy != nil {
+ fieldMask.Paths = append(fieldMask.Paths, "soft_delete_policy")
+ }
for label := range uattrs.setLabels {
fieldMask.Paths = append(fieldMask.Paths, fmt.Sprintf("labels.%s", label))
@@ -421,6 +424,7 @@
IncludeTrailingDelimiter: it.query.IncludeTrailingDelimiter,
MatchGlob: it.query.MatchGlob,
ReadMask: q.toFieldMask(), // a nil Query still results in a "*" FieldMask
+ SoftDeleted: it.query.SoftDeleted,
}
if s.userProject != "" {
ctx = setUserProjectMetadata(ctx, s.userProject)
@@ -490,22 +494,25 @@
return err
}
-func (c *grpcStorageClient) GetObject(ctx context.Context, bucket, object string, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) {
+func (c *grpcStorageClient) GetObject(ctx context.Context, params *getObjectParams, opts ...storageOption) (*ObjectAttrs, error) {
s := callSettings(c.settings, opts...)
req := &storagepb.GetObjectRequest{
- Bucket: bucketResourceName(globalProjectAlias, bucket),
- Object: object,
+ Bucket: bucketResourceName(globalProjectAlias, params.bucket),
+ Object: params.object,
// ProjectionFull by default.
ReadMask: &fieldmaskpb.FieldMask{Paths: []string{"*"}},
}
- if err := applyCondsProto("grpcStorageClient.GetObject", gen, conds, req); err != nil {
+ if err := applyCondsProto("grpcStorageClient.GetObject", params.gen, params.conds, req); err != nil {
return nil, err
}
if s.userProject != "" {
ctx = setUserProjectMetadata(ctx, s.userProject)
}
- if encryptionKey != nil {
- req.CommonObjectRequestParams = toProtoCommonObjectRequestParams(encryptionKey)
+ if params.encryptionKey != nil {
+ req.CommonObjectRequestParams = toProtoCommonObjectRequestParams(params.encryptionKey)
+ }
+ if params.softDeleted {
+ req.SoftDeleted = ¶ms.softDeleted
}
var attrs *ObjectAttrs
@@ -608,6 +615,32 @@
return attrs, err
}
+func (c *grpcStorageClient) RestoreObject(ctx context.Context, params *restoreObjectParams, opts ...storageOption) (*ObjectAttrs, error) {
+ s := callSettings(c.settings, opts...)
+ req := &storagepb.RestoreObjectRequest{
+ Bucket: bucketResourceName(globalProjectAlias, params.bucket),
+ Object: params.object,
+ CopySourceAcl: ¶ms.copySourceACL,
+ }
+ if err := applyCondsProto("grpcStorageClient.RestoreObject", params.gen, params.conds, req); err != nil {
+ return nil, err
+ }
+ if s.userProject != "" {
+ ctx = setUserProjectMetadata(ctx, s.userProject)
+ }
+
+ var attrs *ObjectAttrs
+ err := run(ctx, func(ctx context.Context) error {
+ res, err := c.raw.RestoreObject(ctx, req, s.gax...)
+ attrs = newObjectFromProto(res)
+ return err
+ }, s.retry, s.idempotent)
+ if s, ok := status.FromError(err); ok && s.Code() == codes.NotFound {
+ return nil, ErrObjectNotExist
+ }
+ return attrs, err
+}
+
// Default Object ACL methods.
func (c *grpcStorageClient) DeleteDefaultObjectACL(ctx context.Context, bucket string, entity ACLEntity, opts ...storageOption) error {
@@ -737,7 +770,7 @@
func (c *grpcStorageClient) DeleteObjectACL(ctx context.Context, bucket, object string, entity ACLEntity, opts ...storageOption) error {
// There is no separate API for PATCH in gRPC.
// Make a GET call first to retrieve ObjectAttrs.
- attrs, err := c.GetObject(ctx, bucket, object, defaultGen, nil, nil, opts...)
+ attrs, err := c.GetObject(ctx, &getObjectParams{bucket, object, defaultGen, nil, nil, false}, opts...)
if err != nil {
return err
}
@@ -770,7 +803,7 @@
// ListObjectACLs retrieves object ACL entries. By default, it operates on the latest generation of this object.
// Selecting a specific generation of this object is not currently supported by the client.
func (c *grpcStorageClient) ListObjectACLs(ctx context.Context, bucket, object string, opts ...storageOption) ([]ACLRule, error) {
- o, err := c.GetObject(ctx, bucket, object, defaultGen, nil, nil, opts...)
+ o, err := c.GetObject(ctx, &getObjectParams{bucket, object, defaultGen, nil, nil, false}, opts...)
if err != nil {
return nil, err
}
@@ -780,7 +813,7 @@
func (c *grpcStorageClient) UpdateObjectACL(ctx context.Context, bucket, object string, entity ACLEntity, role ACLRole, opts ...storageOption) error {
// There is no separate API for PATCH in gRPC.
// Make a GET call first to retrieve ObjectAttrs.
- attrs, err := c.GetObject(ctx, bucket, object, defaultGen, nil, nil, opts...)
+ attrs, err := c.GetObject(ctx, &getObjectParams{bucket, object, defaultGen, nil, nil, false}, opts...)
if err != nil {
return err
}
diff --git a/storage/http_client.go b/storage/http_client.go
index f75d938..4130a5e 100644
--- a/storage/http_client.go
+++ b/storage/http_client.go
@@ -337,6 +337,9 @@
}
fetch := func(pageSize int, pageToken string) (string, error) {
req := c.raw.Objects.List(bucket)
+ if it.query.SoftDeleted {
+ req.SoftDeleted(it.query.SoftDeleted)
+ }
setClientHeader(req.Header())
projection := it.query.Projection
if projection == ProjectionDefault {
@@ -409,18 +412,22 @@
return err
}
-func (c *httpStorageClient) GetObject(ctx context.Context, bucket, object string, gen int64, encryptionKey []byte, conds *Conditions, opts ...storageOption) (*ObjectAttrs, error) {
+func (c *httpStorageClient) GetObject(ctx context.Context, params *getObjectParams, opts ...storageOption) (*ObjectAttrs, error) {
s := callSettings(c.settings, opts...)
- req := c.raw.Objects.Get(bucket, object).Projection("full").Context(ctx)
- if err := applyConds("Attrs", gen, conds, req); err != nil {
+ req := c.raw.Objects.Get(params.bucket, params.object).Projection("full").Context(ctx)
+ if err := applyConds("Attrs", params.gen, params.conds, req); err != nil {
return nil, err
}
if s.userProject != "" {
req.UserProject(s.userProject)
}
- if err := setEncryptionHeaders(req.Header(), encryptionKey, false); err != nil {
+ if err := setEncryptionHeaders(req.Header(), params.encryptionKey, false); err != nil {
return nil, err
}
+ if params.softDeleted {
+ req.SoftDeleted(params.softDeleted)
+ }
+
var obj *raw.Object
var err error
err = run(ctx, func(ctx context.Context) error {
@@ -547,6 +554,33 @@
return newObject(obj), nil
}
+func (c *httpStorageClient) RestoreObject(ctx context.Context, params *restoreObjectParams, opts ...storageOption) (*ObjectAttrs, error) {
+ s := callSettings(c.settings, opts...)
+ req := c.raw.Objects.Restore(params.bucket, params.object, params.gen).Context(ctx)
+ // Do not set the generation here since it's not an optional condition; it gets set above.
+ if err := applyConds("RestoreObject", defaultGen, params.conds, req); err != nil {
+ return nil, err
+ }
+ if s.userProject != "" {
+ req.UserProject(s.userProject)
+ }
+ if params.copySourceACL {
+ req.CopySourceAcl(params.copySourceACL)
+ }
+ if err := setEncryptionHeaders(req.Header(), params.encryptionKey, false); err != nil {
+ return nil, err
+ }
+
+ var obj *raw.Object
+ var err error
+ err = run(ctx, func(ctx context.Context) error { obj, err = req.Context(ctx).Do(); return err }, s.retry, s.idempotent)
+ var e *googleapi.Error
+ if ok := errors.As(err, &e); ok && e.Code == http.StatusNotFound {
+ return nil, ErrObjectNotExist
+ }
+ return newObject(obj), err
+}
+
// Default Object ACL methods.
func (c *httpStorageClient) DeleteDefaultObjectACL(ctx context.Context, bucket string, entity ACLEntity, opts ...storageOption) error {
diff --git a/storage/integration_test.go b/storage/integration_test.go
index 6301515..954a482 100644
--- a/storage/integration_test.go
+++ b/storage/integration_test.go
@@ -4264,6 +4264,133 @@
})
}
+func TestIntegration_SoftDelete(t *testing.T) {
+ multiTransportTest(skipJSONReads(context.Background(), "does not test reads"), t, func(t *testing.T, ctx context.Context, _ string, prefix string, client *Client) {
+ h := testHelper{t}
+ testStart := time.Now()
+
+ policy := &SoftDeletePolicy{
+ RetentionDuration: time.Hour * 24 * 8,
+ }
+
+ b := client.Bucket(prefix + uidSpace.New())
+
+ // Create bucket with soft delete policy.
+ if err := b.Create(ctx, testutil.ProjID(), &BucketAttrs{SoftDeletePolicy: policy}); err != nil {
+ t.Fatalf("error creating bucket with soft delete policy set: %v", err)
+ }
+ t.Cleanup(func() { h.mustDeleteBucket(b) })
+
+ // Get bucket's soft delete policy and confirm accuracy.
+ attrs, err := b.Attrs(ctx)
+ if err != nil {
+ t.Fatalf("b.Attrs(%q): %v", b.name, err)
+ }
+
+ got := attrs.SoftDeletePolicy
+ if got == nil {
+ t.Fatal("got nil soft delete policy")
+ }
+ if got.RetentionDuration != policy.RetentionDuration {
+ t.Fatalf("mismatching retention duration; got soft delete policy: %+v, expected: %+v", got, policy)
+ }
+ if got.EffectiveTime.Before(testStart) {
+ t.Fatalf("effective time of soft delete policy should not be in the past, got: %v, test start: %v", got.EffectiveTime, testStart.UTC())
+ }
+
+ // Update the soft delete policy.
+ policy.RetentionDuration = time.Hour * 24 * 9
+
+ attrs, err = b.Update(ctx, BucketAttrsToUpdate{SoftDeletePolicy: policy})
+ if err != nil {
+ t.Fatalf("b.Update: %v", err)
+ }
+
+ if got, expect := attrs.SoftDeletePolicy.RetentionDuration, policy.RetentionDuration; got != expect {
+ t.Fatalf("mismatching retention duration; got: %+v, expected: %+v", got, expect)
+ }
+
+ // Create 2 objects and delete one of them.
+ deletedObject := b.Object("soft-delete" + uidSpaceObjects.New())
+ liveObject := b.Object("not-soft-delete" + uidSpaceObjects.New())
+
+ h.mustWrite(deletedObject.NewWriter(ctx), []byte("soft-deleted"))
+ h.mustWrite(liveObject.NewWriter(ctx), []byte("soft-delete"))
+ t.Cleanup(func() {
+ h.mustDeleteObject(liveObject)
+ h.mustDeleteObject(deletedObject)
+ })
+
+ h.mustDeleteObject(deletedObject)
+
+ var gen int64
+ // List soft deleted objects.
+ it := b.Objects(ctx, &Query{SoftDeleted: true})
+ var gotNames []string
+ for {
+ attrs, err := it.Next()
+ if err == iterator.Done {
+ break
+ }
+ if err != nil {
+ t.Fatalf("iterator.Next: %v", err)
+ }
+ gotNames = append(gotNames, attrs.Name)
+
+ // Get the generation here as the test will fail if there is more than one object
+ gen = attrs.Generation
+ }
+ if len(gotNames) != 1 || gotNames[0] != deletedObject.ObjectName() {
+ t.Fatalf("list soft deleted objects; got: %v, expected only one object named: %s", gotNames, deletedObject.ObjectName())
+ }
+
+ // List live objects.
+ gotNames = []string{}
+ it = b.Objects(ctx, nil)
+ for {
+ attrs, err := it.Next()
+ if err == iterator.Done {
+ break
+ }
+ if err != nil {
+ t.Fatalf("iterator.Next: %v", err)
+ }
+ gotNames = append(gotNames, attrs.Name)
+ }
+ if len(gotNames) != 1 || gotNames[0] != liveObject.ObjectName() {
+ t.Fatalf("list objects that are not soft deleted; got: %v, expected only one object named: %s", gotNames, liveObject.ObjectName())
+ }
+
+ // Get a soft deleted object and check soft and hard delete times.
+ oAttrs, err := deletedObject.Generation(gen).SoftDeleted().Attrs(ctx)
+ if err != nil {
+ t.Fatalf("deletedObject.SoftDeleted().Attrs: %v", err)
+ }
+ if oAttrs.SoftDeleteTime.Before(testStart) {
+ t.Fatalf("SoftDeleteTime of soft deleted object should not be in the past, got: %v, test start: %v", oAttrs.SoftDeleteTime, testStart.UTC())
+ }
+ if got, expected := oAttrs.HardDeleteTime, oAttrs.SoftDeleteTime.Add(policy.RetentionDuration); !expected.Equal(got) {
+ t.Fatalf("HardDeleteTime of soft deleted object should be equal to SoftDeleteTime+RetentionDuration, got: %v, expected: %v", got, expected)
+ }
+
+ // Restore a soft deleted object.
+ _, err = deletedObject.Generation(gen).Restore(ctx, &RestoreOptions{CopySourceACL: true})
+ if err != nil {
+ t.Fatalf("Object(deletedObject).Restore: %v", err)
+ }
+
+ // Update the soft delete policy to remove it.
+ attrs, err = b.Update(ctx, BucketAttrsToUpdate{SoftDeletePolicy: &SoftDeletePolicy{}})
+ if err != nil {
+ t.Fatalf("b.Update: %v", err)
+ }
+
+ if got, expect := attrs.SoftDeletePolicy.RetentionDuration, time.Duration(0); got != expect {
+ t.Fatalf("mismatching retention duration; got: %+v, expected: %+v", got, expect)
+ }
+ })
+}
+
func TestIntegration_KMS(t *testing.T) {
multiTransportTest(context.Background(), t, func(t *testing.T, ctx context.Context, bucket, prefix string, client *Client) {
h := testHelper{t}
diff --git a/storage/storage.go b/storage/storage.go
index c01085f..f80dde9 100644
--- a/storage/storage.go
+++ b/storage/storage.go
@@ -898,6 +898,7 @@
readCompressed bool // Accept-Encoding: gzip
retry *retryConfig
overrideRetention *bool
+ softDeleted bool
}
// ACL provides access to the object's access control list.
@@ -952,7 +953,7 @@
return nil, err
}
opts := makeStorageOpts(true, o.retry, o.userProject)
- return o.c.tc.GetObject(ctx, o.bucket, o.object, o.gen, o.encryptionKey, o.conds, opts...)
+ return o.c.tc.GetObject(ctx, &getObjectParams{o.bucket, o.object, o.gen, o.encryptionKey, o.conds, o.softDeleted}, opts...)
}
// Update updates an object with the provided attributes. See
@@ -1057,6 +1058,50 @@
return &o2
}
+// SoftDeleted returns an object handle that can be used to get an object that
+// has been soft deleted. To get a soft deleted object, the generation must be
+// set on the object using ObjectHandle.Generation.
+// Note that an error will be returned if a live object is queried using this.
+func (o *ObjectHandle) SoftDeleted() *ObjectHandle {
+ o2 := *o
+ o2.softDeleted = true
+ return &o2
+}
+
+// RestoreOptions allows you to set options when restoring an object.
+type RestoreOptions struct {
+ /// CopySourceACL indicates whether the restored object should copy the
+ // access controls of the source object. Only valid for buckets with
+ // fine-grained access. If uniform bucket-level access is enabled, setting
+ // CopySourceACL will cause an error.
+ CopySourceACL bool
+}
+
+// Restore will restore a soft-deleted object to a live object.
+// Note that you must specify a generation to use this method.
+func (o *ObjectHandle) Restore(ctx context.Context, opts *RestoreOptions) (*ObjectAttrs, error) {
+ if err := o.validate(); err != nil {
+ return nil, err
+ }
+
+ // Since the generation is required by restore calls, we set the default to
+ // 0 instead of a negative value, which returns a more descriptive error.
+ gen := o.gen
+ if o.gen == defaultGen {
+ gen = 0
+ }
+
+ // Restore is always idempotent because Generation is a required param.
+ sOpts := makeStorageOpts(true, o.retry, o.userProject)
+ return o.c.tc.RestoreObject(ctx, &restoreObjectParams{
+ bucket: o.bucket,
+ object: o.object,
+ gen: gen,
+ conds: o.conds,
+ copySourceACL: opts.CopySourceACL,
+ }, sOpts...)
+}
+
// NewWriter returns a storage Writer that writes to the GCS object
// associated with this ObjectHandle.
//
@@ -1390,6 +1435,21 @@
// Retention contains the retention configuration for this object.
// ObjectRetention cannot be configured or reported through the gRPC API.
Retention *ObjectRetention
+
+ // SoftDeleteTime is the time when the object became soft-deleted.
+ // Soft-deleted objects are only accessible on an object handle returned by
+ // ObjectHandle.SoftDeleted; if ObjectHandle.SoftDeleted has not been set,
+ // ObjectHandle.Attrs will return ErrObjectNotExist if the object is soft-deleted.
+ // This field is read-only.
+ SoftDeleteTime time.Time
+
+ // HardDeleteTime is the time when the object will be permanently deleted.
+ // Only set when an object becomes soft-deleted with a soft delete policy.
+ // Soft-deleted objects are only accessible on an object handle returned by
+ // ObjectHandle.SoftDeleted; if ObjectHandle.SoftDeleted has not been set,
+ // ObjectHandle.Attrs will return ErrObjectNotExist if the object is soft-deleted.
+ // This field is read-only.
+ HardDeleteTime time.Time
}
// ObjectRetention contains the retention configuration for this object.
@@ -1494,6 +1554,8 @@
CustomTime: convertTime(o.CustomTime),
ComponentCount: o.ComponentCount,
Retention: toObjectRetention(o.Retention),
+ SoftDeleteTime: convertTime(o.SoftDeleteTime),
+ HardDeleteTime: convertTime(o.HardDeleteTime),
}
}
@@ -1529,6 +1591,8 @@
Updated: convertProtoTime(o.GetUpdateTime()),
CustomTime: convertProtoTime(o.GetCustomTime()),
ComponentCount: int64(o.ComponentCount),
+ SoftDeleteTime: convertProtoTime(o.GetSoftDeleteTime()),
+ HardDeleteTime: convertProtoTime(o.GetHardDeleteTime()),
}
}
@@ -1637,6 +1701,11 @@
// prefixes returned by the query. Only applicable if Delimiter is set to /.
// IncludeFoldersAsPrefixes is not yet implemented in the gRPC API.
IncludeFoldersAsPrefixes bool
+
+ // SoftDeleted indicates whether to list soft-deleted objects.
+ // If true, only objects that have been soft-deleted will be listed.
+ // By default, soft-deleted objects are not listed.
+ SoftDeleted bool
}
// attrToFieldMap maps the field names of ObjectAttrs to the underlying field
@@ -1672,6 +1741,8 @@
"CustomTime": "customTime",
"ComponentCount": "componentCount",
"Retention": "retention",
+ "HardDeleteTime": "hardDeleteTime",
+ "SoftDeleteTime": "softDeleteTime",
}
// attrToProtoFieldMap maps the field names of ObjectAttrs to the underlying field
@@ -1704,6 +1775,8 @@
"CustomerKeySHA256": "customer_encryption",
"CustomTime": "custom_time",
"ComponentCount": "component_count",
+ "HardDeleteTime": "hard_delete_time",
+ "SoftDeleteTime": "soft_delete_time",
// MediaLink was explicitly excluded from the proto as it is an HTTP-ism.
// "MediaLink": "mediaLink",
// TODO: add object retention - b/308194853
