test(storage): add methods to retry conformance tests (#4997)
diff --git a/storage/internal/test/conformance/retry_tests.json b/storage/internal/test/conformance/retry_tests.json
index 0fe1147..09f384c 100644
--- a/storage/internal/test/conformance/retry_tests.json
+++ b/storage/internal/test/conformance/retry_tests.json
@@ -40,6 +40,156 @@
],
"preconditionProvided": false,
"expectSuccess": true
+ },
+ {
+ "id": 2,
+ "description": "conditionally_idempotent_retries_when_precondition_is_present",
+ "cases": [
+ {
+ "instructions": ["return-503", "return-503"]
+ },
+ {
+ "instructions": ["return-reset-connection", "return-reset-connection"]
+ },
+ {
+ "instructions": ["return-reset-connection", "return-503"]
+ }
+ ],
+ "methods": [
+ {"name": "storage.buckets.patch", "resources": ["BUCKET"]},
+ {"name": "storage.buckets.setIamPolicy", "resources": ["BUCKET"]},
+ {"name": "storage.buckets.update", "resources": ["BUCKET"]},
+ {"name": "storage.hmacKey.update", "resources": ["HMAC_KEY"]},
+ {"name": "storage.objects.compose", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.copy", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.delete", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.insert", "resources": ["BUCKET"]},
+ {"name": "storage.objects.patch", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.rewrite", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.update", "resources": ["BUCKET", "OBJECT"]}
+ ],
+ "preconditionProvided": true,
+ "expectSuccess": true
+ },
+ {
+ "id": 3,
+ "description": "conditionally_idempotent_no_retries_when_precondition_is_absent",
+ "cases": [
+ {
+ "instructions": ["return-503"]
+ },
+ {
+ "instructions": ["return-reset-connection"]
+ }
+ ],
+ "methods": [
+ {"name": "storage.buckets.patch", "resources": ["BUCKET"]},
+ {"name": "storage.buckets.setIamPolicy", "resources": ["BUCKET"]},
+ {"name": "storage.buckets.update", "resources": ["BUCKET"]},
+ {"name": "storage.hmacKey.update", "resources": ["HMAC_KEY"]},
+ {"name": "storage.objects.compose", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.copy", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.delete", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.insert", "resources": ["BUCKET"]},
+ {"name": "storage.objects.patch", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.rewrite", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.update", "resources": ["BUCKET", "OBJECT"]}
+ ],
+ "preconditionProvided": false,
+ "expectSuccess": false
+ },
+ {
+ "id": 4,
+ "description": "non_idempotent",
+ "cases": [
+ {
+ "instructions": ["return-503"]
+ },
+ {
+ "instructions": ["return-reset-connection"]
+ }
+ ],
+ "methods": [
+ {"name": "storage.bucket_acl.delete", "resources": ["BUCKET"]},
+ {"name": "storage.bucket_acl.insert", "resources": ["BUCKET"]},
+ {"name": "storage.bucket_acl.patch", "resources": ["BUCKET"]},
+ {"name": "storage.bucket_acl.update", "resources": ["BUCKET"]},
+ {"name": "storage.default_object_acl.delete", "resources": ["BUCKET"]},
+ {"name": "storage.default_object_acl.insert", "resources": ["BUCKET"]},
+ {"name": "storage.default_object_acl.patch", "resources": ["BUCKET"]},
+ {"name": "storage.default_object_acl.update", "resources": ["BUCKET"]},
+ {"name": "storage.hmacKey.create", "resources": []},
+ {"name": "storage.notifications.insert", "resources": ["BUCKET"]},
+ {"name": "storage.object_acl.delete", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.object_acl.insert", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.object_acl.patch", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.object_acl.update", "resources": ["BUCKET", "OBJECT"]}
+ ],
+ "preconditionProvided": false,
+ "expectSuccess": false
+ },
+ {
+ "id": 5,
+ "description": "non-retryable errors",
+ "cases": [
+ {
+ "instructions": ["return-400"]
+ },
+ {
+ "instructions": ["return-401"]
+ }
+ ],
+ "methods": [
+ {"name": "storage.bucket_acl.delete", "resources": ["BUCKET"]},
+ {"name": "storage.bucket_acl.get", "resources": ["BUCKET"]},
+ {"name": "storage.bucket_acl.insert", "resources": ["BUCKET"]},
+ {"name": "storage.bucket_acl.list", "resources": ["BUCKET"]},
+ {"name": "storage.bucket_acl.patch", "resources": ["BUCKET"]},
+ {"name": "storage.bucket_acl.update", "resources": ["BUCKET"]},
+ {"name": "storage.buckets.delete", "resources": ["BUCKET"]},
+ {"name": "storage.buckets.get", "resources": ["BUCKET"]},
+ {"name": "storage.buckets.getIamPolicy", "resources": ["BUCKET"]},
+ {"name": "storage.buckets.insert", "resources": ["BUCKET"]},
+ {"name": "storage.buckets.list", "resources": ["BUCKET"]},
+ {"name": "storage.buckets.lockRetentionPolicy", "resources": ["BUCKET"]},
+ {"name": "storage.buckets.patch", "resources": ["BUCKET"]},
+ {"name": "storage.buckets.setIamPolicy", "resources": ["BUCKET"]},
+ {"name": "storage.buckets.testIamPermissions", "resources": ["BUCKET"]},
+ {"name": "storage.buckets.update", "resources": ["BUCKET"]},
+ {"name": "storage.default_object_acl.delete", "resources": ["BUCKET"]},
+ {"name": "storage.default_object_acl.get", "resources": ["BUCKET"]},
+ {"name": "storage.default_object_acl.insert", "resources": ["BUCKET"]},
+ {"name": "storage.default_object_acl.list", "resources": ["BUCKET"]},
+ {"name": "storage.default_object_acl.patch", "resources": ["BUCKET"]},
+ {"name": "storage.default_object_acl.update", "resources": ["BUCKET"]},
+ {"name": "storage.hmacKey.create", "resources": []},
+ {"name": "storage.hmacKey.delete", "resources": ["HMAC_KEY"]},
+ {"name": "storage.hmacKey.get", "resources": ["HMAC_KEY"]},
+ {"name": "storage.hmacKey.list", "resources": ["HMAC_KEY"]},
+ {"name": "storage.hmacKey.update", "resources": ["HMAC_KEY"]},
+ {"name": "storage.notifications.delete", "resources": ["BUCKET", "NOTIFICATION"]},
+ {"name": "storage.notifications.get", "resources": ["BUCKET", "NOTIFICATION"]},
+ {"name": "storage.notifications.insert", "resources": ["BUCKET", "NOTIFICATION"]},
+ {"name": "storage.notifications.list", "resources": ["BUCKET", "NOTIFICATION"]},
+ {"name": "storage.object_acl.delete", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.object_acl.get", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.object_acl.insert", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.object_acl.list", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.object_acl.patch", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.object_acl.update", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.compose", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.copy", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.delete", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.get", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.insert", "resources": ["BUCKET"]},
+ {"name": "storage.objects.list", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.patch", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.rewrite", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.objects.update", "resources": ["BUCKET", "OBJECT"]},
+ {"name": "storage.serviceaccount.get", "resources": []}
+ ],
+ "preconditionProvided": false,
+ "expectSuccess": false
}
]
}
diff --git a/storage/retry_conformance_test.go b/storage/retry_conformance_test.go
index 894fb5f..13ceea8 100644
--- a/storage/retry_conformance_test.go
+++ b/storage/retry_conformance_test.go
@@ -19,6 +19,8 @@
"context"
"encoding/json"
"fmt"
+ "io"
+ "io/ioutil"
"net/http"
"net/http/httputil"
"net/url"
@@ -28,6 +30,7 @@
"cloud.google.com/go/internal/uid"
storage_v1_tests "cloud.google.com/go/storage/internal/test/conformance"
+ "google.golang.org/api/iterator"
"google.golang.org/api/option"
raw "google.golang.org/api/storage/v1"
htransport "google.golang.org/api/transport/http"
@@ -50,13 +53,329 @@
// wrap library methods that implement these calls. There may be multiple values
// because multiple library methods may use the same call (e.g. get could be a
// read or just a metadata get).
+//
+// There may be missing methods with respect to the json API as not all methods
+// are used in the client library. The following are not used:
+// storage.bucket_acl.get
+// storage.bucket_acl.insert
+// storage.bucket_acl.patch
+// storage.buckets.update
+// storage.default_object_acl.get
+// storage.default_object_acl.insert
+// storage.default_object_acl.patch
+// storage.notifications.get
+// storage.object_acl.get
+// storage.object_acl.insert
+// storage.object_acl.patch
+// storage.objects.copy
+// storage.objects.update
var methods = map[string][]retryFunc{
+ // Idempotent operations
+ "storage.bucket_acl.list": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ _, err := c.Bucket(fs.bucket.Name).ACL().List(ctx)
+ return err
+ },
+ },
+ "storage.buckets.delete": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ // Delete files from bucket before deleting bucket
+ it := c.Bucket(fs.bucket.Name).Objects(ctx, nil)
+ for {
+ attrs, err := it.Next()
+ if err == iterator.Done {
+ break
+ }
+ if err != nil {
+ return err
+ }
+ if err := c.Bucket(fs.bucket.Name).Object(attrs.Name).Delete(ctx); err != nil {
+ return err
+ }
+ }
+ return c.Bucket(fs.bucket.Name).Delete(ctx)
+ },
+ },
"storage.buckets.get": {
func(ctx context.Context, c *Client, fs *resources, _ bool) error {
_, err := c.Bucket(fs.bucket.Name).Attrs(ctx)
return err
},
},
+ "storage.buckets.getIamPolicy": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ _, err := c.Bucket(fs.bucket.Name).IAM().Policy(ctx)
+ return err
+ },
+ },
+ "storage.buckets.insert": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ return c.Bucket("bucket").Create(ctx, projectID, nil)
+ },
+ },
+ "storage.buckets.list": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ it := c.Buckets(ctx, projectID)
+ for {
+ _, err := it.Next()
+ if err == iterator.Done {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ }
+ },
+ },
+ "storage.buckets.lockRetentionPolicy": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ // buckets.lockRetentionPolicy is always idempotent, but is a special case because IfMetagenerationMatch is always required
+ return c.Bucket(fs.bucket.Name).If(BucketConditions{MetagenerationMatch: fs.bucket.MetaGeneration}).LockRetentionPolicy(ctx)
+ },
+ },
+ "storage.buckets.testIamPermissions": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ _, err := c.Bucket(fs.bucket.Name).IAM().TestPermissions(ctx, nil)
+ return err
+ },
+ },
+ "storage.default_object_acl.list": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ _, err := c.Bucket(fs.bucket.Name).DefaultObjectACL().List(ctx)
+ return err
+ },
+ },
+ "storage.hmacKey.delete": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ // key must be inactive to delete:
+ c.HMACKeyHandle(projectID, fs.hmacKey.AccessID).Update(ctx, HMACKeyAttrsToUpdate{State: "INACTIVE"})
+ return c.HMACKeyHandle(projectID, fs.hmacKey.AccessID).Delete(ctx)
+ },
+ },
+ "storage.hmacKey.get": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ _, err := c.HMACKeyHandle(projectID, fs.hmacKey.AccessID).Get(ctx)
+ return err
+ },
+ },
+ "storage.hmacKey.list": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ it := c.ListHMACKeys(ctx, projectID)
+ for {
+ _, err := it.Next()
+ if err == iterator.Done {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ }
+ },
+ },
+ "storage.notifications.delete": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ return c.Bucket(fs.bucket.Name).DeleteNotification(ctx, fs.notification.ID)
+ },
+ },
+ "storage.notifications.list": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ _, err := c.Bucket(fs.bucket.Name).Notifications(ctx)
+ return err
+ },
+ },
+ "storage.object_acl.list": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ _, err := c.Bucket(fs.bucket.Name).Object(fs.object.Name).ACL().List(ctx)
+ return err
+ },
+ },
+ "storage.objects.get": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ _, err := c.Bucket(fs.bucket.Name).Object(fs.object.Name).Attrs(ctx)
+ return err
+ },
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ r, err := c.Bucket(fs.bucket.Name).Object(fs.object.Name).NewReader(ctx)
+ if err != nil {
+ return err
+ }
+ _, err = io.Copy(ioutil.Discard, r)
+ return err
+ },
+ },
+ "storage.objects.list": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ it := c.Bucket(fs.bucket.Name).Objects(ctx, nil)
+ for {
+ _, err := it.Next()
+ if err == iterator.Done {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ }
+ },
+ },
+ "storage.serviceaccount.get": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ _, err := c.ServiceAccount(ctx, projectID)
+ return err
+ },
+ },
+ // Conditionally idempotent operations
+ // (all conditionally idempotent operations currently fail)
+ "storage.buckets.patch": {
+ func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+ uattrs := BucketAttrsToUpdate{StorageClass: "ARCHIVE"}
+ bkt := c.Bucket(fs.bucket.Name)
+ if preconditions {
+ bkt = c.Bucket(fs.bucket.Name).If(BucketConditions{MetagenerationMatch: fs.bucket.MetaGeneration})
+ }
+ _, err := bkt.Update(ctx, uattrs)
+ return err
+ },
+ },
+ "storage.buckets.setIamPolicy": {
+ func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+ bkt := c.Bucket(fs.bucket.Name)
+ policy, err := bkt.IAM().Policy(ctx)
+ if err != nil {
+ return err
+ }
+
+ if err := bkt.IAM().SetPolicy(ctx, policy); err != nil {
+ return err
+ }
+ return fmt.Errorf("Etag preconditions not supported")
+ },
+ },
+ "storage.hmacKey.update": {
+ func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+ key := c.HMACKeyHandle(projectID, fs.hmacKey.AccessID)
+
+ _, err := key.Update(ctx, HMACKeyAttrsToUpdate{State: "INACTIVE"})
+ if err != nil {
+ return err
+ }
+ return fmt.Errorf("Etag preconditions not supported")
+ },
+ },
+ "storage.objects.compose": {
+ func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+ dstName := "new-object"
+ src := c.Bucket(fs.bucket.Name).Object(fs.object.Name)
+ dst := c.Bucket(fs.bucket.Name).Object(dstName)
+
+ if preconditions {
+ dst = c.Bucket(fs.bucket.Name).Object(dstName).If(Conditions{DoesNotExist: true})
+ }
+
+ _, err := dst.ComposerFrom(src).Run(ctx)
+ return err
+ },
+ },
+ "storage.objects.delete": {
+ func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+ obj := c.Bucket(fs.bucket.Name).Object(fs.object.Name)
+
+ if preconditions {
+ obj = c.Bucket(fs.bucket.Name).Object(fs.object.Name).If(Conditions{GenerationMatch: fs.object.Generation})
+ }
+ return obj.Delete(ctx)
+ },
+ },
+ "storage.objects.insert": {
+ func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+ obj := c.Bucket(fs.bucket.Name).Object("new-object.txt")
+
+ if preconditions {
+ obj = obj.If(Conditions{DoesNotExist: true})
+ }
+
+ objW := obj.NewWriter(ctx)
+ if _, err := io.Copy(objW, strings.NewReader("object body")); err != nil {
+ return fmt.Errorf("io.Copy: %v", err)
+ }
+ if err := objW.Close(); err != nil {
+ return fmt.Errorf("Writer.Close: %v", err)
+ }
+ return nil
+ },
+ },
+ "storage.objects.patch": {
+ func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+ uattrs := ObjectAttrsToUpdate{Metadata: map[string]string{"foo": "bar"}}
+ obj := c.Bucket(fs.bucket.Name).Object(fs.object.Name)
+ if preconditions {
+ obj = obj.If(Conditions{MetagenerationMatch: fs.object.Metageneration})
+ }
+ _, err := obj.Update(ctx, uattrs)
+ return err
+ },
+ },
+ "storage.objects.rewrite": {
+ func(ctx context.Context, c *Client, fs *resources, preconditions bool) error {
+ dstName := "new-object"
+ src := c.Bucket(fs.bucket.Name).Object(fs.object.Name)
+ dst := c.Bucket(fs.bucket.Name).Object(dstName)
+
+ if preconditions {
+ dst = c.Bucket(fs.bucket.Name).Object(dstName).If(Conditions{DoesNotExist: true})
+ }
+
+ _, err := dst.CopierFrom(src).Run(ctx)
+ return err
+ },
+ },
+ // Non-idempotent operations
+ "storage.bucket_acl.delete": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ return c.Bucket(fs.bucket.Name).ACL().Delete(ctx, AllUsers)
+ },
+ },
+ "storage.bucket_acl.update": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ return c.Bucket(fs.bucket.Name).ACL().Set(ctx, AllAuthenticatedUsers, RoleOwner)
+ },
+ },
+ "storage.default_object_acl.delete": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ return c.Bucket(fs.bucket.Name).DefaultObjectACL().Delete(ctx, AllAuthenticatedUsers)
+ },
+ },
+ "storage.default_object_acl.update": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ return c.Bucket(fs.bucket.Name).DefaultObjectACL().Set(ctx, AllAuthenticatedUsers, RoleOwner)
+ },
+ },
+ "storage.hmacKey.create": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ _, err := c.CreateHMACKey(ctx, projectID, serviceAccountEmail)
+ return err
+ },
+ },
+ "storage.notifications.insert": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ notification := Notification{
+ TopicID: "my-topic",
+ TopicProjectID: projectID,
+ PayloadFormat: "json",
+ }
+ _, err := c.Bucket(fs.bucket.Name).AddNotification(ctx, ¬ification)
+ return err
+ },
+ },
+ "storage.object_acl.delete": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ return c.Bucket(fs.bucket.Name).Object(fs.object.Name).ACL().Delete(ctx, AllAuthenticatedUsers)
+ },
+ },
+ "storage.object_acl.update": {
+ func(ctx context.Context, c *Client, fs *resources, _ bool) error {
+ return c.Bucket(fs.bucket.Name).Object(fs.object.Name).ACL().Set(ctx, AllAuthenticatedUsers, RoleOwner)
+ },
+ },
}
func TestRetryConformance(t *testing.T) {
