diff options
Diffstat (limited to 'vendor/cloud.google.com/go')
| -rw-r--r-- | vendor/cloud.google.com/go/internal/optional/optional.go | 94 | ||||
| -rw-r--r-- | vendor/cloud.google.com/go/storage/acl.go | 48 | ||||
| -rw-r--r-- | vendor/cloud.google.com/go/storage/bucket.go | 89 | ||||
| -rw-r--r-- | vendor/cloud.google.com/go/storage/copy.go | 51 | ||||
| -rw-r--r-- | vendor/cloud.google.com/go/storage/doc.go | 7 | ||||
| -rw-r--r-- | vendor/cloud.google.com/go/storage/invoke.go | 46 | ||||
| -rw-r--r-- | vendor/cloud.google.com/go/storage/storage.go | 507 | ||||
| -rw-r--r-- | vendor/cloud.google.com/go/storage/writer.go | 23 |
8 files changed, 567 insertions, 298 deletions
diff --git a/vendor/cloud.google.com/go/internal/optional/optional.go b/vendor/cloud.google.com/go/internal/optional/optional.go new file mode 100644 index 0000000..f9102f3 --- /dev/null +++ b/vendor/cloud.google.com/go/internal/optional/optional.go @@ -0,0 +1,94 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package optional provides versions of primitive types that can +// be nil. These are useful in methods that update some of an API object's +// fields. +package optional + +import ( + "fmt" + "strings" +) + +type ( + // Bool is either a bool or nil. + Bool interface{} + + // String is either a string or nil. + String interface{} + + // Int is either an int or nil. + Int interface{} + + // Uint is either a uint or nil. + Uint interface{} + + // Float64 is either a float64 or nil. + Float64 interface{} +) + +// ToBool returns its argument as a bool. +// It panics if its argument is nil or not a bool. +func ToBool(v Bool) bool { + x, ok := v.(bool) + if !ok { + doPanic("Bool", v) + } + return x +} + +// ToString returns its argument as a string. +// It panics if its argument is nil or not a string. +func ToString(v String) string { + x, ok := v.(string) + if !ok { + doPanic("String", v) + } + return x +} + +// ToInt returns its argument as an int. +// It panics if its argument is nil or not an int. +func ToInt(v Int) int { + x, ok := v.(int) + if !ok { + doPanic("Int", v) + } + return x +} + +// ToUint returns its argument as a uint. +// It panics if its argument is nil or not a uint. +func ToUint(v Uint) uint { + x, ok := v.(uint) + if !ok { + doPanic("Uint", v) + } + return x +} + +// ToFloat64 returns its argument as a float64. +// It panics if its argument is nil or not a float64. +func ToFloat64(v Float64) float64 { + x, ok := v.(float64) + if !ok { + doPanic("Float64", v) + } + return x +} + +func doPanic(capType string, v interface{}) { + panic(fmt.Sprintf("optional.%s value should be %s, got %T", capType, strings.ToLower(capType), v)) +} diff --git a/vendor/cloud.google.com/go/storage/acl.go b/vendor/cloud.google.com/go/storage/acl.go index e0cb948..714d280 100644 --- a/vendor/cloud.google.com/go/storage/acl.go +++ b/vendor/cloud.google.com/go/storage/acl.go @@ -92,7 +92,12 @@ func (a *ACLHandle) List(ctx context.Context) ([]ACLRule, error) { } func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) { - acls, err := a.c.raw.DefaultObjectAccessControls.List(a.bucket).Context(ctx).Do() + var acls *raw.ObjectAccessControls + var err error + err = runWithRetry(ctx, func() error { + acls, err = a.c.raw.DefaultObjectAccessControls.List(a.bucket).Context(ctx).Do() + return err + }) if err != nil { return nil, fmt.Errorf("storage: error listing default object ACL for bucket %q: %v", a.bucket, err) } @@ -105,7 +110,10 @@ func (a *ACLHandle) bucketDefaultSet(ctx context.Context, entity ACLEntity, role Entity: string(entity), Role: string(role), } - _, err := a.c.raw.DefaultObjectAccessControls.Update(a.bucket, string(entity), acl).Context(ctx).Do() + err := runWithRetry(ctx, func() error { + _, err := a.c.raw.DefaultObjectAccessControls.Update(a.bucket, string(entity), acl).Context(ctx).Do() + return err + }) if err != nil { return fmt.Errorf("storage: error updating default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err) } @@ -113,7 +121,9 @@ func (a *ACLHandle) bucketDefaultSet(ctx context.Context, entity ACLEntity, role } func (a *ACLHandle) bucketDefaultDelete(ctx context.Context, entity ACLEntity) error { - err := a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity)).Context(ctx).Do() + err := runWithRetry(ctx, func() error { + return a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity)).Context(ctx).Do() + }) if err != nil { return fmt.Errorf("storage: error deleting default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err) } @@ -121,7 +131,12 @@ func (a *ACLHandle) bucketDefaultDelete(ctx context.Context, entity ACLEntity) e } func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) { - acls, err := a.c.raw.BucketAccessControls.List(a.bucket).Context(ctx).Do() + var acls *raw.BucketAccessControls + var err error + err = runWithRetry(ctx, func() error { + acls, err = a.c.raw.BucketAccessControls.List(a.bucket).Context(ctx).Do() + return err + }) if err != nil { return nil, fmt.Errorf("storage: error listing bucket ACL for bucket %q: %v", a.bucket, err) } @@ -139,7 +154,10 @@ func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRol Entity: string(entity), Role: string(role), } - _, err := a.c.raw.BucketAccessControls.Update(a.bucket, string(entity), acl).Context(ctx).Do() + err := runWithRetry(ctx, func() error { + _, err := a.c.raw.BucketAccessControls.Update(a.bucket, string(entity), acl).Context(ctx).Do() + return err + }) if err != nil { return fmt.Errorf("storage: error updating bucket ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err) } @@ -147,7 +165,9 @@ func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRol } func (a *ACLHandle) bucketDelete(ctx context.Context, entity ACLEntity) error { - err := a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity)).Context(ctx).Do() + err := runWithRetry(ctx, func() error { + return a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity)).Context(ctx).Do() + }) if err != nil { return fmt.Errorf("storage: error deleting bucket ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err) } @@ -155,7 +175,12 @@ func (a *ACLHandle) bucketDelete(ctx context.Context, entity ACLEntity) error { } func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) { - acls, err := a.c.raw.ObjectAccessControls.List(a.bucket, a.object).Context(ctx).Do() + var acls *raw.ObjectAccessControls + var err error + err = runWithRetry(ctx, func() error { + acls, err = a.c.raw.ObjectAccessControls.List(a.bucket, a.object).Context(ctx).Do() + return err + }) if err != nil { return nil, fmt.Errorf("storage: error listing object ACL for bucket %q, file %q: %v", a.bucket, a.object, err) } @@ -168,7 +193,10 @@ func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRol Entity: string(entity), Role: string(role), } - _, err := a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl).Context(ctx).Do() + err := runWithRetry(ctx, func() error { + _, err := a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl).Context(ctx).Do() + return err + }) if err != nil { return fmt.Errorf("storage: error updating object ACL entry for bucket %q, file %q, entity %q: %v", a.bucket, a.object, entity, err) } @@ -176,7 +204,9 @@ func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRol } func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error { - err := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity)).Context(ctx).Do() + err := runWithRetry(ctx, func() error { + return a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity)).Context(ctx).Do() + }) if err != nil { return fmt.Errorf("storage: error deleting object ACL entry for bucket %q, file %q, entity %q: %v", a.bucket, a.object, entity, err) } diff --git a/vendor/cloud.google.com/go/storage/bucket.go b/vendor/cloud.google.com/go/storage/bucket.go index 0875f7d..a2be0f4 100644 --- a/vendor/cloud.google.com/go/storage/bucket.go +++ b/vendor/cloud.google.com/go/storage/bucket.go @@ -35,14 +35,13 @@ func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *Buck } bkt.Name = b.name req := b.c.raw.Buckets.Insert(projectID, bkt) - _, err := req.Context(ctx).Do() - return err + return runWithRetry(ctx, func() error { _, err := req.Context(ctx).Do(); return err }) } // Delete deletes the Bucket. func (b *BucketHandle) Delete(ctx context.Context) error { req := b.c.raw.Buckets.Delete(b.name) - return req.Context(ctx).Do() + return runWithRetry(ctx, func() error { return req.Context(ctx).Do() }) } // ACL returns an ACLHandle, which provides access to the bucket's access control list. @@ -75,12 +74,18 @@ func (b *BucketHandle) Object(name string) *ObjectHandle { bucket: b.name, object: name, }, + gen: -1, } } // Attrs returns the metadata for the bucket. func (b *BucketHandle) Attrs(ctx context.Context) (*BucketAttrs, error) { - resp, err := b.c.raw.Buckets.Get(b.name).Projection("full").Context(ctx).Do() + var resp *raw.Bucket + var err error + err = runWithRetry(ctx, func() error { + resp, err = b.c.raw.Buckets.Get(b.name).Projection("full").Context(ctx).Do() + return err + }) if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { return nil, ErrBucketNotExist } @@ -110,8 +115,11 @@ type BucketAttrs struct { // StorageClass is the storage class of the bucket. This defines // how objects in the bucket are stored and determines the SLA - // and the cost of storage. Typical values are "STANDARD" and - // "DURABLE_REDUCED_AVAILABILITY". Defaults to "STANDARD". + // and the cost of storage. Typical values are "MULTI_REGIONAL", + // "REGIONAL", "NEARLINE", "COLDLINE", "STANDARD" and + // "DURABLE_REDUCED_AVAILABILITY". Defaults to "STANDARD", which + // is equivalent to "MULTI_REGIONAL" or "REGIONAL" depending on + // the bucket's location settings. StorageClass string // Created is the creation time of the bucket. @@ -175,47 +183,6 @@ func (b *BucketAttrs) toRawBucket() *raw.Bucket { } } -// ObjectList represents a list of objects returned from a bucket List call. -type ObjectList struct { - // Results represent a list of object results. - Results []*ObjectAttrs - - // Next is the continuation query to retrieve more - // results with the same filtering criteria. If there - // are no more results to retrieve, it is nil. - Next *Query - - // Prefixes represents prefixes of objects - // matching-but-not-listed up to and including - // the requested delimiter. - Prefixes []string -} - -// List lists objects from the bucket. You can specify a query -// to filter the results. If q is nil, no filtering is applied. -// -// Deprecated. Use BucketHandle.Objects instead. -func (b *BucketHandle) List(ctx context.Context, q *Query) (*ObjectList, error) { - it := b.Objects(ctx, q) - nextToken, err := it.fetch(it.pageInfo.MaxSize, it.pageInfo.Token) - if err != nil { - return nil, err - } - list := &ObjectList{} - for _, item := range it.items { - if item.Prefix != "" { - list.Prefixes = append(list.Prefixes, item.Prefix) - } else { - list.Results = append(list.Results, item) - } - } - if nextToken != "" { - it.query.Cursor = nextToken - list.Next = &it.query - } - return list, nil -} - // Objects returns an iterator over the objects in the bucket that match the Query q. // If q is nil, no filtering is done. func (b *BucketHandle) Objects(ctx context.Context, q *Query) *ObjectIterator { @@ -229,8 +196,6 @@ func (b *BucketHandle) Objects(ctx context.Context, q *Query) *ObjectIterator { func() interface{} { b := it.items; it.items = nil; return b }) if q != nil { it.query = *q - it.pageInfo.MaxSize = q.MaxResults - it.pageInfo.Token = q.Cursor } return it } @@ -248,9 +213,9 @@ type ObjectIterator struct { // PageInfo supports pagination. See the google.golang.org/api/iterator package for details. func (it *ObjectIterator) PageInfo() *iterator.PageInfo { return it.pageInfo } -// Next returns the next result. Its second return value is Done if there are -// no more results. Once Next returns Done, all subsequent calls will return -// Done. +// Next returns the next result. Its second return value is iterator.Done if +// there are no more results. Once Next returns iterator.Done, all subsequent +// calls will return iterator.Done. // // If Query.Delimiter is non-empty, some of the ObjectAttrs returned by Next will // have a non-empty Prefix field, and a zero value for all other fields. These @@ -274,7 +239,12 @@ func (it *ObjectIterator) fetch(pageSize int, pageToken string) (string, error) if pageSize > 0 { req.MaxResults(int64(pageSize)) } - resp, err := req.Context(it.ctx).Do() + var resp *raw.Objects + var err error + err = runWithRetry(it.ctx, func() error { + resp, err = req.Context(it.ctx).Do() + return err + }) if err != nil { return "", err } @@ -319,9 +289,9 @@ type BucketIterator struct { nextFunc func() error } -// Next returns the next result. Its second return value is Done if there are -// no more results. Once Next returns Done, all subsequent calls will return -// Done. +// Next returns the next result. Its second return value is iterator.Done if +// there are no more results. Once Next returns iterator.Done, all subsequent +// calls will return iterator.Done. func (it *BucketIterator) Next() (*BucketAttrs, error) { if err := it.nextFunc(); err != nil { return nil, err @@ -342,7 +312,12 @@ func (it *BucketIterator) fetch(pageSize int, pageToken string) (string, error) if pageSize > 0 { req.MaxResults(int64(pageSize)) } - resp, err := req.Context(it.ctx).Do() + var resp *raw.Buckets + var err error + err = runWithRetry(it.ctx, func() error { + resp, err = req.Context(it.ctx).Do() + return err + }) if err != nil { return "", err } diff --git a/vendor/cloud.google.com/go/storage/copy.go b/vendor/cloud.google.com/go/storage/copy.go index c0e4041..6adb566 100644 --- a/vendor/cloud.google.com/go/storage/copy.go +++ b/vendor/cloud.google.com/go/storage/copy.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "reflect" - "unicode/utf8" "golang.org/x/net/context" raw "google.golang.org/api/storage/v1" @@ -66,18 +65,11 @@ type Copier struct { // Run performs the copy. func (c *Copier) Run(ctx context.Context) (*ObjectAttrs, error) { - // TODO(jba): add ObjectHandle.validate to do these checks. - if c.src.bucket == "" || c.dst.bucket == "" { - return nil, errors.New("storage: the source and destination bucket names must both be non-empty") - } - if c.src.object == "" || c.dst.object == "" { - return nil, errors.New("storage: the source and destination object names must both be non-empty") - } - if !utf8.ValidString(c.src.object) { - return nil, fmt.Errorf("storage: object name %q is not valid UTF-8", c.src.object) + if err := c.src.validate(); err != nil { + return nil, err } - if !utf8.ValidString(c.dst.object) { - return nil, fmt.Errorf("storage: dst name %q is not valid UTF-8", c.dst.object) + if err := c.dst.validate(); err != nil { + return nil, err } var rawObject *raw.Object // If any attribute was set, then we make sure the name matches the destination @@ -112,13 +104,15 @@ func (c *Copier) callRewrite(ctx context.Context, src *ObjectHandle, rawObj *raw if c.RewriteToken != "" { call.RewriteToken(c.RewriteToken) } - if err := applyConds("Copy destination", c.dst.conds, call); err != nil { + if err := applyConds("Copy destination", c.dst.gen, c.dst.conds, call); err != nil { return nil, err } - if err := applyConds("Copy source", toSourceConds(c.src.conds), call); err != nil { + if err := applySourceConds(c.src.gen, c.src.conds, call); err != nil { return nil, err } - res, err := call.Do() + var res *raw.RewriteResponse + var err error + err = runWithRetry(ctx, func() error { res, err = call.Do(); return err }) if err != nil { return nil, err } @@ -146,41 +140,40 @@ type Composer struct { // Run performs the compose operation. func (c *Composer) Run(ctx context.Context) (*ObjectAttrs, error) { - if c.dst.bucket == "" || c.dst.object == "" { - return nil, errors.New("storage: the destination bucket and object names must be non-empty") + if err := c.dst.validate(); err != nil { + return nil, err } if len(c.srcs) == 0 { return nil, errors.New("storage: at least one source object must be specified") } req := &raw.ComposeRequest{} - if !reflect.DeepEqual(c.ObjectAttrs, ObjectAttrs{}) { - req.Destination = c.ObjectAttrs.toRawObject(c.dst.bucket) - req.Destination.Name = c.dst.object - } - + // Compose requires a non-empty Destination, so we always set it, + // even if the caller-provided ObjectAttrs is the zero value. + req.Destination = c.ObjectAttrs.toRawObject(c.dst.bucket) for _, src := range c.srcs { + if err := src.validate(); err != nil { + return nil, err + } if src.bucket != c.dst.bucket { return nil, fmt.Errorf("storage: all source objects must be in bucket %q, found %q", c.dst.bucket, src.bucket) } - if src.object == "" { - return nil, errors.New("storage: all source object names must be non-empty") - } srcObj := &raw.ComposeRequestSourceObjects{ Name: src.object, } - if err := applyConds("ComposeFrom source", src.conds, composeSourceObj{srcObj}); err != nil { + if err := applyConds("ComposeFrom source", src.gen, src.conds, composeSourceObj{srcObj}); err != nil { return nil, err } req.SourceObjects = append(req.SourceObjects, srcObj) } call := c.dst.c.raw.Objects.Compose(c.dst.bucket, c.dst.object, req).Context(ctx) - if err := applyConds("ComposeFrom destination", c.dst.conds, call); err != nil { + if err := applyConds("ComposeFrom destination", c.dst.gen, c.dst.conds, call); err != nil { return nil, err } - - obj, err := call.Do() + var obj *raw.Object + var err error + err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) if err != nil { return nil, err } diff --git a/vendor/cloud.google.com/go/storage/doc.go b/vendor/cloud.google.com/go/storage/doc.go index 39d54df..c23f2c8 100644 --- a/vendor/cloud.google.com/go/storage/doc.go +++ b/vendor/cloud.google.com/go/storage/doc.go @@ -19,6 +19,10 @@ Google Cloud Storage stores data in named objects, which are grouped into bucket More information about Google Cloud Storage is available at https://cloud.google.com/storage/docs. +All of the methods of this package use exponential backoff to retry calls +that fail with certain errors, as described in +https://cloud.google.com/storage/docs/exponential-backoff. + Note: This package is experimental and may make backwards-incompatible changes. @@ -134,8 +138,7 @@ For example, say you've read an object's metadata into objAttrs. Now you want to write to that object, but only if its contents haven't changed since you read it. Here is how to express that: - cond := storage.IfGenerationMatch(objAttrs.Generation) - w = obj.WithConditions(cond).NewWriter(ctx) + w = obj.If(storage.Conditions{GenerationMatch: objAttrs.Generation}).NewWriter(ctx) // Proceed with writing as above. Signed URLs diff --git a/vendor/cloud.google.com/go/storage/invoke.go b/vendor/cloud.google.com/go/storage/invoke.go new file mode 100644 index 0000000..03b98f4 --- /dev/null +++ b/vendor/cloud.google.com/go/storage/invoke.go @@ -0,0 +1,46 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + gax "github.com/googleapis/gax-go" + "golang.org/x/net/context" + "google.golang.org/api/googleapi" +) + +// runWithRetry calls the function until it returns nil or a non-retryable error, or +// the context is done. +func runWithRetry(ctx context.Context, call func() error) error { + var backoff gax.Backoff // use defaults for gax exponential backoff + for { + err := call() + if err == nil { + return nil + } + e, ok := err.(*googleapi.Error) + if !ok { + return err + } + // Retry on 429 and 5xx, according to + // https://cloud.google.com/storage/docs/exponential-backoff. + if e.Code == 429 || (e.Code >= 500 && e.Code < 600) { + if err := gax.Sleep(ctx, backoff.Pause()); err != nil { + return err + } + continue + } + return err + } +} diff --git a/vendor/cloud.google.com/go/storage/storage.go b/vendor/cloud.google.com/go/storage/storage.go index 3707ea9..9dccf5a 100644 --- a/vendor/cloud.google.com/go/storage/storage.go +++ b/vendor/cloud.google.com/go/storage/storage.go @@ -35,10 +35,10 @@ import ( "time" "unicode/utf8" - "google.golang.org/api/iterator" "google.golang.org/api/option" "google.golang.org/api/transport" + "cloud.google.com/go/internal/optional" "golang.org/x/net/context" "google.golang.org/api/googleapi" raw "google.golang.org/api/storage/v1" @@ -47,9 +47,6 @@ import ( var ( ErrBucketNotExist = errors.New("storage: bucket doesn't exist") ErrObjectNotExist = errors.New("storage: object doesn't exist") - - // Done is returned by iterators in this package when they have no more items. - Done = iterator.Done ) const userAgent = "gcloud-golang-storage/20151204" @@ -68,49 +65,6 @@ const ( ScopeReadWrite = raw.DevstorageReadWriteScope ) -// AdminClient is a client type for performing admin operations on a project's -// buckets. -// -// Deprecated: Client has all of AdminClient's methods. -type AdminClient struct { - c *Client - projectID string -} - -// NewAdminClient creates a new AdminClient for a given project. -// -// Deprecated: use NewClient instead. -func NewAdminClient(ctx context.Context, projectID string, opts ...option.ClientOption) (*AdminClient, error) { - c, err := NewClient(ctx, opts...) - if err != nil { - return nil, err - } - return &AdminClient{ - c: c, - projectID: projectID, - }, nil -} - -// Close closes the AdminClient. -func (c *AdminClient) Close() error { - return c.c.Close() -} - -// Create creates a Bucket in the project. -// If attrs is nil the API defaults will be used. -// -// Deprecated: use BucketHandle.Create instead. -func (c *AdminClient) CreateBucket(ctx context.Context, bucketName string, attrs *BucketAttrs) error { - return c.c.Bucket(bucketName).Create(ctx, c.projectID, attrs) -} - -// Delete deletes a Bucket in the project. -// -// Deprecated: use BucketHandle.Delete instead. -func (c *AdminClient) DeleteBucket(ctx context.Context, bucketName string) error { - return c.c.Bucket(bucketName).Delete(ctx) -} - // Client is a client for interacting with Google Cloud Storage. // // Clients should be reused instead of created as needed. @@ -321,9 +275,9 @@ type ObjectHandle struct { c *Client bucket string object string - - acl ACLHandle - conds []Condition + acl ACLHandle + gen int64 // a negative value indicates latest + conds *Conditions } // ACL provides access to the object's access control list. @@ -333,24 +287,41 @@ func (o *ObjectHandle) ACL() *ACLHandle { return &o.acl } -// WithConditions returns a copy of o using the provided conditions. -func (o *ObjectHandle) WithConditions(conds ...Condition) *ObjectHandle { +// Generation returns a new ObjectHandle that operates on a specific generation +// of the object. +// By default, the handle operates on the latest generation. Not +// all operations work when given a specific generation; check the API +// endpoints at https://cloud.google.com/storage/docs/json_api/ for details. +func (o *ObjectHandle) Generation(gen int64) *ObjectHandle { + o2 := *o + o2.gen = gen + return &o2 +} + +// If returns a new ObjectHandle that applies a set of preconditions. +// Preconditions already set on the ObjectHandle are ignored. +// Operations on the new handle will only occur if the preconditions are +// satisfied. See https://cloud.google.com/storage/docs/generations-preconditions +// for more details. +func (o *ObjectHandle) If(conds Conditions) *ObjectHandle { o2 := *o - o2.conds = conds + o2.conds = &conds return &o2 } // Attrs returns meta information about the object. // ErrObjectNotExist will be returned if the object is not found. func (o *ObjectHandle) Attrs(ctx context.Context) (*ObjectAttrs, error) { - if !utf8.ValidString(o.object) { - return nil, fmt.Errorf("storage: object name %q is not valid UTF-8", o.object) + if err := o.validate(); err != nil { + return nil, err } call := o.c.raw.Objects.Get(o.bucket, o.object).Projection("full").Context(ctx) - if err := applyConds("Attrs", o.conds, call); err != nil { + if err := applyConds("Attrs", o.gen, o.conds, call); err != nil { return nil, err } - obj, err := call.Do() + var obj *raw.Object + var err error + err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { return nil, ErrObjectNotExist } @@ -363,15 +334,64 @@ func (o *ObjectHandle) Attrs(ctx context.Context) (*ObjectAttrs, error) { // Update updates an object with the provided attributes. // All zero-value attributes are ignored. // ErrObjectNotExist will be returned if the object is not found. -func (o *ObjectHandle) Update(ctx context.Context, attrs ObjectAttrs) (*ObjectAttrs, error) { - if !utf8.ValidString(o.object) { - return nil, fmt.Errorf("storage: object name %q is not valid UTF-8", o.object) +func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (*ObjectAttrs, error) { + if err := o.validate(); err != nil { + return nil, err } - call := o.c.raw.Objects.Patch(o.bucket, o.object, attrs.toRawObject(o.bucket)).Projection("full").Context(ctx) - if err := applyConds("Update", o.conds, call); err != nil { + var attrs ObjectAttrs + // Lists of fields to send, and set to null, in the JSON. + var forceSendFields, nullFields []string + if uattrs.ContentType != nil { + attrs.ContentType = optional.ToString(uattrs.ContentType) + forceSendFields = append(forceSendFields, "ContentType") + } + if uattrs.ContentLanguage != nil { + attrs.ContentLanguage = optional.ToString(uattrs.ContentLanguage) + // For ContentLanguage It's an error to send the empty string. + // Instead we send a null. + if attrs.ContentLanguage == "" { + nullFields = append(nullFields, "ContentLanguage") + } else { + forceSendFields = append(forceSendFields, "ContentLanguage") + } + } + if uattrs.ContentEncoding != nil { + attrs.ContentEncoding = optional.ToString(uattrs.ContentEncoding) + forceSendFields = append(forceSendFields, "ContentType") + } + if uattrs.ContentDisposition != nil { + attrs.ContentDisposition = optional.ToString(uattrs.ContentDisposition) + forceSendFields = append(forceSendFields, "ContentDisposition") + } + if uattrs.CacheControl != nil { + attrs.CacheControl = optional.ToString(uattrs.CacheControl) + forceSendFields = append(forceSendFields, "CacheControl") + } + if uattrs.Metadata != nil { + attrs.Metadata = uattrs.Metadata + if len(attrs.Metadata) == 0 { + // Sending the empty map is a no-op. We send null instead. + nullFields = append(nullFields, "Metadata") + } else { + forceSendFields = append(forceSendFields, "Metadata") + } + } + if uattrs.ACL != nil { + attrs.ACL = uattrs.ACL + // It's an error to attempt to delete the ACL, so + // we don't append to nullFields here. + forceSendFields = append(forceSendFields, "Acl") + } + rawObj := attrs.toRawObject(o.bucket) + rawObj.ForceSendFields = forceSendFields + rawObj.NullFields = nullFields + call := o.c.raw.Objects.Patch(o.bucket, o.object, rawObj).Projection("full").Context(ctx) + if err := applyConds("Update", o.gen, o.conds, call); err != nil { return nil, err } - obj, err := call.Do() + var obj *raw.Object + var err error + err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { return nil, ErrObjectNotExist } @@ -381,16 +401,37 @@ func (o *ObjectHandle) Update(ctx context.Context, attrs ObjectAttrs) (*ObjectAt return newObject(obj), nil } +// ObjectAttrsToUpdate is used to update the attributes of an object. +// Only fields set to non-nil values will be updated. +// Set a field to its zero value to delete it. +// +// For example, to change ContentType and delete ContentEncoding and +// Metadata, use +// ObjectAttrsToUpdate{ +// ContentType: "text/html", +// ContentEncoding: "", +// Metadata: map[string]string{}, +// } +type ObjectAttrsToUpdate struct { + ContentType optional.String + ContentLanguage optional.String + ContentEncoding optional.String + ContentDisposition optional.String + CacheControl optional.String + Metadata map[string]string // set to map[string]string{} to delete + ACL []ACLRule +} + // Delete deletes the single specified object. func (o *ObjectHandle) Delete(ctx context.Context) error { - if !utf8.ValidString(o.object) { - return fmt.Errorf("storage: object name %q is not valid UTF-8", o.object) + if err := o.validate(); err != nil { + return err } call := o.c.raw.Objects.Delete(o.bucket, o.object).Context(ctx) - if err := applyConds("Delete", o.conds, call); err != nil { + if err := applyConds("Delete", o.gen, o.conds, call); err != nil { return err } - err := call.Do() + err := runWithRetry(ctx, func() error { return call.Do() }) switch e := err.(type) { case nil: return nil @@ -402,32 +443,6 @@ func (o *ObjectHandle) Delete(ctx context.Context) error { return err } -// CopyTo copies the object to the given dst. -// The copied object's attributes are overwritten by attrs if non-nil. -// -// Deprecated: use ObjectHandle.CopierFrom instead. -func (o *ObjectHandle) CopyTo(ctx context.Context, dst *ObjectHandle, attrs *ObjectAttrs) (*ObjectAttrs, error) { - c := dst.CopierFrom(o) - if attrs != nil { - c.ObjectAttrs = *attrs - } - return c.Run(ctx) -} - -// ComposeFrom concatenates the provided slice of source objects into a new -// object whose destination is the receiver. The provided attrs, if not nil, -// are used to set the attributes on the newly-created object. All source -// objects must reside within the same bucket as the destination. -// -// Deprecated: use ObjectHandle.ComposerFrom instead. -func (o *ObjectHandle) ComposeFrom(ctx context.Context, srcs []*ObjectHandle, attrs *ObjectAttrs) (*ObjectAttrs, error) { - c := o.ComposerFrom(srcs...) - if attrs != nil { - c.ObjectAttrs = *attrs - } - return c.Run(ctx) -} - // NewReader creates a new Reader to read the contents of the // object. // ErrObjectNotExist will be returned if the object is not found. @@ -438,19 +453,25 @@ func (o *ObjectHandle) NewReader(ctx context.Context) (*Reader, error) { } // NewRangeReader reads part of an object, reading at most length bytes -// starting at the given offset. If length is negative, the object is read +// starting at the given offset. If length is negative, the object is read // until the end. func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (*Reader, error) { - if !utf8.ValidString(o.object) { - return nil, fmt.Errorf("storage: object name %q is not valid UTF-8", o.object) + if err := o.validate(); err != nil { + return nil, err } if offset < 0 { return nil, fmt.Errorf("storage: invalid offset %d < 0", offset) } + if o.conds != nil { + if err := o.conds.validate("NewRangeReader"); err != nil { + return nil, err + } + } u := &url.URL{ - Scheme: "https", - Host: "storage.googleapis.com", - Path: fmt.Sprintf("/%s/%s", o.bucket, o.object), + Scheme: "https", + Host: "storage.googleapis.com", + Path: fmt.Sprintf("/%s/%s", o.bucket, o.object), + RawQuery: conditionsQuery(o.gen, o.conds), } verb := "GET" if length == 0 { @@ -460,15 +481,13 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) if err != nil { return nil, err } - if err := applyConds("NewReader", o.conds, objectsGetCall{req}); err != nil { - return nil, err - } if length < 0 && offset > 0 { req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) } else if length > 0 { req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) } - res, err := o.c.hc.Do(req) + var res *http.Response + err = runWithRetry(ctx, func() error { res, err = o.c.hc.Do(req); return err }) if err != nil { return nil, err } @@ -542,7 +561,21 @@ func (o *ObjectHandle) NewWriter(ctx context.Context) *Writer { o: o, donec: make(chan struct{}), ObjectAttrs: ObjectAttrs{Name: o.object}, + ChunkSize: googleapi.DefaultUploadChunkSize, + } +} + +func (o *ObjectHandle) validate() error { + if o.bucket == "" { + return errors.New("storage: bucket name is empty") + } + if o.object == "" { + return errors.New("storage: object name is empty") } + if !utf8.ValidString(o.object) { + return fmt.Errorf("storage: object name %q is not valid UTF-8", o.object) + } + return nil } // parseKey converts the binary contents of a private key file @@ -664,8 +697,11 @@ type ObjectAttrs struct { // StorageClass is the storage class of the bucket. // This value defines how objects in the bucket are stored and // determines the SLA and the cost of storage. Typical values are - // "STANDARD" and "DURABLE_REDUCED_AVAILABILITY". - // It defaults to "STANDARD". This field is read-only. + // "MULTI_REGIONAL", "REGIONAL", "NEARLINE", "COLDLINE", "STANDARD" + // and "DURABLE_REDUCED_AVAILABILITY". + // It defaults to "STANDARD", which is equivalent to "MULTI_REGIONAL" + // or "REGIONAL" depending on the bucket's location settings. This + // field is read-only. StorageClass string // Created is the time the object was created. This field is read-only. @@ -760,21 +796,6 @@ type Query struct { // Versions indicates whether multiple versions of the same // object will be included in the results. Versions bool - - // Cursor is a previously-returned page token - // representing part of the larger set of results to view. - // Optional. - // - // Deprecated: Use ObjectIterator.PageInfo().Token instead. - Cursor string - - // MaxResults is the maximum number of items plus prefixes - // to return. As duplicate prefixes are omitted, - // fewer total results may be returned than requested. - // The default page limit is used if it is negative or zero. - // - // Deprecated: Use ObjectIterator.PageInfo().MaxSize instead. - MaxResults int } // contentTyper implements ContentTyper to enable an @@ -788,105 +809,195 @@ func (c *contentTyper) ContentType() string { return c.t } -// A Condition constrains methods to act on specific generations of +// Conditions constrain methods to act on specific generations of // resources. // -// Not all conditions or combinations of conditions are applicable to -// all methods. -type Condition interface { - // method is the high-level ObjectHandle method name, for - // error messages. call is the call object to modify. - modifyCall(method string, call interface{}) error -} - -// applyConds modifies the provided call using the conditions in conds. -// call is something that quacks like a *raw.WhateverCall. -func applyConds(method string, conds []Condition, call interface{}) error { - for _, cond := range conds { - if err := cond.modifyCall(method, call); err != nil { - return err - } +// The zero value is an empty set of constraints. Not all conditions or +// combinations of conditions are applicable to all methods. +// See https://cloud.google.com/storage/docs/generations-preconditions +// for details on how these operate. +type Conditions struct { + // Generation constraints. + // At most one of the following can be set to a non-zero value. + + // GenerationMatch specifies that the object must have the given generation + // for the operation to occur. + // If GenerationMatch is zero, it has no effect. + // Use DoesNotExist to specify that the object does not exist in the bucket. + GenerationMatch int64 + + // GenerationNotMatch specifies that the object must not have the given + // generation for the operation to occur. + // If GenerationNotMatch is zero, it has no effect. + GenerationNotMatch int64 + + // DoesNotExist specifies that the object must not exist in the bucket for + // the operation to occur. + // If DoesNotExist is false, it has no effect. + DoesNotExist bool + + // Metadata generation constraints. + // At most one of the following can be set to a non-zero value. + + // MetagenerationMatch specifies that the object must have the given + // metageneration for the operation to occur. + // If MetagenerationMatch is zero, it has no effect. + MetagenerationMatch int64 + + // MetagenerationNotMatch specifies that the object must not have the given + // metageneration for the operation to occur. + // If MetagenerationNotMatch is zero, it has no effect. + MetagenerationNotMatch int64 +} + +func (c *Conditions) validate(method string) error { + if *c == (Conditions{}) { + return fmt.Errorf("storage: %s: empty conditions", method) + } + if !c.isGenerationValid() { + return fmt.Errorf("storage: %s: multiple conditions specified for generation", method) + } + if !c.isMetagenerationValid() { + return fmt.Errorf("storage: %s: multiple conditions specified for metageneration", method) } return nil } -// toSourceConds returns a slice of Conditions derived from Conds that instead -// function on the equivalent Source methods of a call. -func toSourceConds(conds []Condition) []Condition { - out := make([]Condition, 0, len(conds)) - for _, c := range conds { - switch c := c.(type) { - case genCond: - var m string - if strings.HasPrefix(c.method, "If") { - m = "IfSource" + c.method[2:] - } else { - m = "Source" + c.method - } - out = append(out, genCond{method: m, val: c.val}) - default: - // NOTE(djd): If the message from unsupportedCond becomes - // confusing, we'll need to find a way for Conditions to - // identify themselves. - out = append(out, unsupportedCond{}) - } +func (c *Conditions) isGenerationValid() bool { + n := 0 + if c.GenerationMatch != 0 { + n++ } - return out + if c.GenerationNotMatch != 0 { + n++ + } + if c.DoesNotExist { + n++ + } + return n <= 1 } -func Generation(gen int64) Condition { return genCond{"Generation", gen} } -func IfGenerationMatch(gen int64) Condition { return genCond{"IfGenerationMatch", gen} } -func IfGenerationNotMatch(gen int64) Condition { return genCond{"IfGenerationNotMatch", gen} } -func IfMetaGenerationMatch(gen int64) Condition { return genCond{"IfMetagenerationMatch", gen} } -func IfMetaGenerationNotMatch(gen int64) Condition { return genCond{"IfMetagenerationNotMatch", gen} } - -type genCond struct { - method string - val int64 +func (c *Conditions) isMetagenerationValid() bool { + return c.MetagenerationMatch == 0 || c.MetagenerationNotMatch == 0 } -func (g genCond) modifyCall(srcMethod string, call interface{}) error { - rv := reflect.ValueOf(call) - meth := rv.MethodByName(g.method) - if !meth.IsValid() { - return fmt.Errorf("%s: condition %s not supported", srcMethod, g.method) +// applyConds modifies the provided call using the conditions in conds. +// call is something that quacks like a *raw.WhateverCall. +func applyConds(method string, gen int64, conds *Conditions, call interface{}) error { + cval := reflect.ValueOf(call) + if gen >= 0 { + if !setConditionField(cval, "Generation", gen) { + return fmt.Errorf("storage: %s: generation not supported", method) + } + } + if conds == nil { + return nil + } + if err := conds.validate(method); err != nil { + return err + } + switch { + case conds.GenerationMatch != 0: + if !setConditionField(cval, "IfGenerationMatch", conds.GenerationMatch) { + return fmt.Errorf("storage: %s: ifGenerationMatch not supported", method) + } + case conds.GenerationNotMatch != 0: + if !setConditionField(cval, "IfGenerationNotMatch", conds.GenerationNotMatch) { + return fmt.Errorf("storage: %s: ifGenerationNotMatch not supported", method) + } + case conds.DoesNotExist: + if !setConditionField(cval, "IfGenerationMatch", int64(0)) { + return fmt.Errorf("storage: %s: DoesNotExist not supported", method) + } + } + switch { + case conds.MetagenerationMatch != 0: + if !setConditionField(cval, "IfMetagenerationMatch", conds.MetagenerationMatch) { + return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method) + } + case conds.MetagenerationNotMatch != 0: + if !setConditionField(cval, "IfMetagenerationNotMatch", conds.MetagenerationNotMatch) { + return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method) + } } - meth.Call([]reflect.Value{reflect.ValueOf(g.val)}) return nil } -type unsupportedCond struct{} - -func (unsupportedCond) modifyCall(srcMethod string, call interface{}) error { - return fmt.Errorf("%s: condition not supported", srcMethod) +func applySourceConds(gen int64, conds *Conditions, call *raw.ObjectsRewriteCall) error { + if gen >= 0 { + call.SourceGeneration(gen) + } + if conds == nil { + return nil + } + if err := conds.validate("CopyTo source"); err != nil { + return err + } + switch { + case conds.GenerationMatch != 0: + call.IfSourceGenerationMatch(conds.GenerationMatch) + case conds.GenerationNotMatch != 0: + call.IfSourceGenerationNotMatch(conds.GenerationNotMatch) + case conds.DoesNotExist: + call.IfSourceGenerationMatch(0) + } + switch { + case conds.MetagenerationMatch != 0: + call.IfSourceMetagenerationMatch(conds.MetagenerationMatch) + case conds.MetagenerationNotMatch != 0: + call.IfSourceMetagenerationNotMatch(conds.MetagenerationNotMatch) + } + return nil } -func appendParam(req *http.Request, k, v string) { - sep := "" - if req.URL.RawQuery != "" { - sep = "&" +// setConditionField sets a field on a *raw.WhateverCall. +// We can't use anonymous interfaces because the return type is +// different, since the field setters are builders. +func setConditionField(call reflect.Value, name string, value interface{}) bool { + m := call.MethodByName(name) + if !m.IsValid() { + return false } - req.URL.RawQuery += sep + url.QueryEscape(k) + "=" + url.QueryEscape(v) + m.Call([]reflect.Value{reflect.ValueOf(value)}) + return true } -// objectsGetCall wraps an *http.Request for an object fetch call, but adds the methods -// that modifyCall searches for by name. (the same names as the raw, auto-generated API) -type objectsGetCall struct{ req *http.Request } +// conditionsQuery returns the generation and conditions as a URL query +// string suitable for URL.RawQuery. It assumes that the conditions +// have been validated. +func conditionsQuery(gen int64, conds *Conditions) string { + // URL escapes are elided because integer strings are URL-safe. + var buf []byte -func (c objectsGetCall) Generation(gen int64) { - appendParam(c.req, "generation", fmt.Sprint(gen)) -} -func (c objectsGetCall) IfGenerationMatch(gen int64) { - appendParam(c.req, "ifGenerationMatch", fmt.Sprint(gen)) -} -func (c objectsGetCall) IfGenerationNotMatch(gen int64) { - appendParam(c.req, "ifGenerationNotMatch", fmt.Sprint(gen)) -} -func (c objectsGetCall) IfMetagenerationMatch(gen int64) { - appendParam(c.req, "ifMetagenerationMatch", fmt.Sprint(gen)) -} -func (c objectsGetCall) IfMetagenerationNotMatch(gen int64) { - appendParam(c.req, "ifMetagenerationNotMatch", fmt.Sprint(gen)) + appendParam := func(s string, n int64) { + if len(buf) > 0 { + buf = append(buf, '&') + } + buf = append(buf, s...) + buf = strconv.AppendInt(buf, n, 10) + } + + if gen >= 0 { + appendParam("generation=", gen) + } + if conds == nil { + return string(buf) + } + switch { + case conds.GenerationMatch != 0: + appendParam("ifGenerationMatch=", conds.GenerationMatch) + case conds.GenerationNotMatch != 0: + appendParam("ifGenerationNotMatch=", conds.GenerationNotMatch) + case conds.DoesNotExist: + appendParam("ifGenerationMatch=", 0) + } + switch { + case conds.MetagenerationMatch != 0: + appendParam("ifMetagenerationMatch=", conds.MetagenerationMatch) + case conds.MetagenerationNotMatch != 0: + appendParam("ifMetagenerationNotMatch=", conds.MetagenerationNotMatch) + } + return string(buf) } // composeSourceObj wraps a *raw.ComposeRequestSourceObjects, but adds the methods diff --git a/vendor/cloud.google.com/go/storage/writer.go b/vendor/cloud.google.com/go/storage/writer.go index 79ab791..8f98156 100644 --- a/vendor/cloud.google.com/go/storage/writer.go +++ b/vendor/cloud.google.com/go/storage/writer.go @@ -15,6 +15,7 @@ package storage import ( + "errors" "fmt" "io" "unicode/utf8" @@ -31,6 +32,17 @@ type Writer struct { // attributes are ignored. ObjectAttrs + // ChunkSize controls the maximum number of bytes of the object that the + // Writer will attempt to send to the server in a single request. Objects + // smaller than the size will be sent in a single request, while larger + // objects will be split over multiple requests. The size will be rounded up + // to the nearest multiple of 256K. If zero, chunking will be disabled and + // the object will be uploaded in a single request. + // + // ChunkSize will default to a reasonable value. Any custom configuration + // must be done before the first Write call. + ChunkSize int + ctx context.Context o *ObjectHandle @@ -56,7 +68,12 @@ func (w *Writer) open() error { w.pw = pw w.opened = true - var mediaOpts []googleapi.MediaOption + if w.ChunkSize < 0 { + return errors.New("storage: Writer.ChunkSize must non-negative") + } + mediaOpts := []googleapi.MediaOption{ + googleapi.ChunkSize(w.ChunkSize), + } if c := attrs.ContentType; c != "" { mediaOpts = append(mediaOpts, googleapi.ContentType(c)) } @@ -70,9 +87,9 @@ func (w *Writer) open() error { Context(w.ctx) var resp *raw.Object - err := applyConds("NewWriter", w.o.conds, call) + err := applyConds("NewWriter", w.o.gen, w.o.conds, call) if err == nil { - resp, err = call.Do() + err = runWithRetry(w.ctx, func() error { resp, err = call.Do(); return err }) } if err != nil { w.err = err |