diff options
Diffstat (limited to 'vendor/cloud.google.com/go/storage')
-rw-r--r-- | vendor/cloud.google.com/go/storage/acl.go | 51 | ||||
-rw-r--r-- | vendor/cloud.google.com/go/storage/bucket.go | 271 | ||||
-rw-r--r-- | vendor/cloud.google.com/go/storage/copy.go | 29 | ||||
-rw-r--r-- | vendor/cloud.google.com/go/storage/doc.go | 30 | ||||
-rw-r--r-- | vendor/cloud.google.com/go/storage/go110.go | 2 | ||||
-rw-r--r-- | vendor/cloud.google.com/go/storage/go17.go | 2 | ||||
-rw-r--r-- | vendor/cloud.google.com/go/storage/iam.go | 55 | ||||
-rw-r--r-- | vendor/cloud.google.com/go/storage/invoke.go | 2 | ||||
-rw-r--r-- | vendor/cloud.google.com/go/storage/not_go110.go | 2 | ||||
-rw-r--r-- | vendor/cloud.google.com/go/storage/not_go17.go | 2 | ||||
-rw-r--r-- | vendor/cloud.google.com/go/storage/notifications.go | 188 | ||||
-rw-r--r-- | vendor/cloud.google.com/go/storage/reader.go | 234 | ||||
-rw-r--r-- | vendor/cloud.google.com/go/storage/storage.go | 236 | ||||
-rw-r--r-- | vendor/cloud.google.com/go/storage/writer.go | 39 |
14 files changed, 888 insertions, 255 deletions
diff --git a/vendor/cloud.google.com/go/storage/acl.go b/vendor/cloud.google.com/go/storage/acl.go index a1b2b6d..d56795c 100644 --- a/vendor/cloud.google.com/go/storage/acl.go +++ b/vendor/cloud.google.com/go/storage/acl.go @@ -1,4 +1,4 @@ -// Copyright 2014 Google Inc. All Rights Reserved. +// Copyright 2014 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,10 +15,10 @@ package storage import ( - "fmt" "net/http" "reflect" + "cloud.google.com/go/internal/trace" "golang.org/x/net/context" "google.golang.org/api/googleapi" raw "google.golang.org/api/storage/v1" @@ -64,7 +64,10 @@ type ACLHandle struct { } // Delete permanently deletes the ACL entry for the given entity. -func (a *ACLHandle) Delete(ctx context.Context, entity ACLEntity) error { +func (a *ACLHandle) Delete(ctx context.Context, entity ACLEntity) (err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.ACL.Delete") + defer func() { trace.EndSpan(ctx, err) }() + if a.object != "" { return a.objectDelete(ctx, entity) } @@ -75,7 +78,10 @@ func (a *ACLHandle) Delete(ctx context.Context, entity ACLEntity) error { } // Set sets the permission level for the given entity. -func (a *ACLHandle) Set(ctx context.Context, entity ACLEntity, role ACLRole) error { +func (a *ACLHandle) Set(ctx context.Context, entity ACLEntity, role ACLRole) (err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.ACL.Set") + defer func() { trace.EndSpan(ctx, err) }() + if a.object != "" { return a.objectSet(ctx, entity, role, false) } @@ -86,7 +92,10 @@ func (a *ACLHandle) Set(ctx context.Context, entity ACLEntity, role ACLRole) err } // List retrieves ACL entries. -func (a *ACLHandle) List(ctx context.Context) ([]ACLRule, error) { +func (a *ACLHandle) List(ctx context.Context) (rules []ACLRule, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.ACL.List") + defer func() { trace.EndSpan(ctx, err) }() + if a.object != "" { return a.objectList(ctx) } @@ -106,21 +115,17 @@ func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) { return err }) if err != nil { - return nil, fmt.Errorf("storage: error listing default object ACL for bucket %q: %v", a.bucket, err) + return nil, err } return toACLRules(acls.Items), nil } func (a *ACLHandle) bucketDefaultDelete(ctx context.Context, entity ACLEntity) error { - err := runWithRetry(ctx, func() error { + return runWithRetry(ctx, func() error { req := a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity)) a.configureCall(req, ctx) return req.Do() }) - if err != nil { - return fmt.Errorf("storage: error deleting default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err) - } - return nil } func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) { @@ -133,7 +138,7 @@ func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) { return err }) if err != nil { - return nil, fmt.Errorf("storage: error listing bucket ACL for bucket %q: %v", a.bucket, err) + return nil, err } r := make([]ACLRule, len(acls.Items)) for i, v := range acls.Items { @@ -156,7 +161,7 @@ func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRol return err }) if err != nil { - return fmt.Errorf("storage: error updating bucket ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err) + return err } return nil } @@ -168,7 +173,7 @@ func (a *ACLHandle) bucketDelete(ctx context.Context, entity ACLEntity) error { return req.Do() }) if err != nil { - return fmt.Errorf("storage: error deleting bucket ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err) + return err } return nil } @@ -183,7 +188,7 @@ func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) { return err }) if err != nil { - return nil, fmt.Errorf("storage: error listing object ACL for bucket %q, file %q: %v", a.bucket, a.object, err) + return nil, err } return toACLRules(acls.Items), nil } @@ -206,30 +211,18 @@ func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRol req = a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl) } a.configureCall(req, ctx) - err := runWithRetry(ctx, func() error { + return runWithRetry(ctx, func() error { _, err := req.Do() return err }) - if err != nil { - if isBucketDefault { - return fmt.Errorf("storage: error updating default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err) - } else { - return fmt.Errorf("storage: error updating object ACL entry for bucket %q, object %q, entity %q: %v", a.bucket, a.object, entity, err) - } - } - return nil } func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error { - err := runWithRetry(ctx, func() error { + return runWithRetry(ctx, func() error { req := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity)) a.configureCall(req, ctx) return req.Do() }) - if err != nil { - return fmt.Errorf("storage: error deleting object ACL entry for bucket %q, file %q, entity %q: %v", a.bucket, a.object, entity, err) - } - return nil } func (a *ACLHandle) configureCall(call interface { diff --git a/vendor/cloud.google.com/go/storage/bucket.go b/vendor/cloud.google.com/go/storage/bucket.go index 07852a5..93473ce 100644 --- a/vendor/cloud.google.com/go/storage/bucket.go +++ b/vendor/cloud.google.com/go/storage/bucket.go @@ -1,4 +1,4 @@ -// Copyright 2014 Google Inc. LiveAndArchived Rights Reserved. +// Copyright 2014 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import ( "time" "cloud.google.com/go/internal/optional" + "cloud.google.com/go/internal/trace" "golang.org/x/net/context" "google.golang.org/api/googleapi" "google.golang.org/api/iterator" @@ -35,7 +36,7 @@ type BucketHandle struct { acl ACLHandle defaultObjectACL ACLHandle conds *BucketConditions - userProject string // project for requester-pays buckets + userProject string // project for Requester Pays buckets } // Bucket returns a BucketHandle, which provides operations on the named bucket. @@ -63,7 +64,10 @@ func (c *Client) Bucket(name string) *BucketHandle { // Create creates the Bucket in the project. // If attrs is nil the API defaults will be used. -func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *BucketAttrs) error { +func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *BucketAttrs) (err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Create") + defer func() { trace.EndSpan(ctx, err) }() + var bkt *raw.Bucket if attrs != nil { bkt = attrs.toRawBucket() @@ -82,7 +86,10 @@ func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *Buck } // Delete deletes the Bucket. -func (b *BucketHandle) Delete(ctx context.Context) error { +func (b *BucketHandle) Delete(ctx context.Context) (err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Delete") + defer func() { trace.EndSpan(ctx, err) }() + req, err := b.newDeleteCall() if err != nil { return err @@ -139,7 +146,10 @@ func (b *BucketHandle) Object(name string) *ObjectHandle { } // Attrs returns the metadata for the bucket. -func (b *BucketHandle) Attrs(ctx context.Context) (*BucketAttrs, error) { +func (b *BucketHandle) Attrs(ctx context.Context) (attrs *BucketAttrs, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Attrs") + defer func() { trace.EndSpan(ctx, err) }() + req, err := b.newGetCall() if err != nil { return nil, err @@ -155,7 +165,7 @@ func (b *BucketHandle) Attrs(ctx context.Context) (*BucketAttrs, error) { if err != nil { return nil, err } - return newBucket(resp), nil + return newBucket(resp) } func (b *BucketHandle) newGetCall() (*raw.BucketsGetCall, error) { @@ -170,7 +180,10 @@ func (b *BucketHandle) newGetCall() (*raw.BucketsGetCall, error) { return req, nil } -func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (*BucketAttrs, error) { +func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (attrs *BucketAttrs, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Create") + defer func() { trace.EndSpan(ctx, err) }() + req, err := b.newPatchCall(&uattrs) if err != nil { return nil, err @@ -180,7 +193,7 @@ func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) ( if err != nil { return nil, err } - return newBucket(rb), nil + return newBucket(rb) } func (b *BucketHandle) newPatchCall(uattrs *BucketAttrsToUpdate) (*raw.BucketsPatchCall, error) { @@ -237,9 +250,28 @@ type BucketAttrs struct { Labels map[string]string // RequesterPays reports whether the bucket is a Requester Pays bucket. + // Clients performing operations on Requester Pays buckets must provide + // a user project (see BucketHandle.UserProject), which will be billed + // for the operations. RequesterPays bool + // Lifecycle is the lifecycle configuration for objects in the bucket. Lifecycle Lifecycle + + // Retention policy enforces a minimum retention time for all objects + // contained in the bucket. A RetentionPolicy of nil implies the bucket + // has no minimum data retention. + // + // This feature is in private alpha release. It is not currently available to + // most customers. It might be changed in backwards-incompatible ways and is not + // subject to any SLA or deprecation policy. + RetentionPolicy *RetentionPolicy + + // The bucket's Cross-Origin Resource Sharing (CORS) configuration. + CORS []CORS + + // The encryption configuration used by default for newly inserted objects. + Encryption *BucketEncryption } // Lifecycle is the lifecycle configuration for objects in the bucket. @@ -247,12 +279,37 @@ type Lifecycle struct { Rules []LifecycleRule } +// Retention policy enforces a minimum retention time for all objects +// contained in the bucket. +// +// Any attempt to overwrite or delete objects younger than the retention +// period will result in an error. An unlocked retention policy can be +// modified or removed from the bucket via the Update method. A +// locked retention policy cannot be removed or shortened in duration +// for the lifetime of the bucket. +// +// This feature is in private alpha release. It is not currently available to +// most customers. It might be changed in backwards-incompatible ways and is not +// subject to any SLA or deprecation policy. +type RetentionPolicy struct { + // RetentionPeriod specifies the duration that objects need to be + // retained. Retention duration must be greater than zero and less than + // 100 years. Note that enforcement of retention periods less than a day + // is not guaranteed. Such periods should only be used for testing + // purposes. + RetentionPeriod time.Duration + + // EffectiveTime is the time from which the policy was enforced and + // effective. This field is read-only. + EffectiveTime time.Time +} + const ( // RFC3339 date with only the date segment, used for CreatedBefore in LifecycleRule. rfc3339Date = "2006-01-02" // DeleteAction is a lifecycle action that deletes a live and/or archived - // objects. Takes precendence over SetStorageClass actions. + // objects. Takes precedence over SetStorageClass actions. DeleteAction = "Delete" // SetStorageClassAction changes the storage class of live and/or archived @@ -332,9 +389,13 @@ type LifecycleCondition struct { NumNewerVersions int64 } -func newBucket(b *raw.Bucket) *BucketAttrs { +func newBucket(b *raw.Bucket) (*BucketAttrs, error) { if b == nil { - return nil + return nil, nil + } + rp, err := toRetentionPolicy(b.RetentionPolicy) + if err != nil { + return nil, err } bucket := &BucketAttrs{ Name: b.Name, @@ -346,6 +407,9 @@ func newBucket(b *raw.Bucket) *BucketAttrs { Labels: b.Labels, RequesterPays: b.Billing != nil && b.Billing.RequesterPays, Lifecycle: toLifecycle(b.Lifecycle), + RetentionPolicy: rp, + CORS: toCORS(b.Cors), + Encryption: toBucketEncryption(b.Encryption), } acl := make([]ACLRule, len(b.Acl)) for i, rule := range b.Acl { @@ -363,7 +427,7 @@ func newBucket(b *raw.Bucket) *BucketAttrs { } } bucket.DefaultObjectACL = objACL - return bucket + return bucket, nil } // toRawBucket copies the editable attribute from b to the raw library's Bucket type. @@ -408,16 +472,70 @@ func (b *BucketAttrs) toRawBucket() *raw.Bucket { Labels: labels, Billing: bb, Lifecycle: toRawLifecycle(b.Lifecycle), + RetentionPolicy: b.RetentionPolicy.toRawRetentionPolicy(), + Cors: toRawCORS(b.CORS), + Encryption: b.Encryption.toRawBucketEncryption(), } } +// CORS is the bucket's Cross-Origin Resource Sharing (CORS) configuration. +type CORS struct { + // MaxAge is the value to return in the Access-Control-Max-Age + // header used in preflight responses. + MaxAge time.Duration + + // Methods is the list of HTTP methods on which to include CORS response + // headers, (GET, OPTIONS, POST, etc) Note: "*" is permitted in the list + // of methods, and means "any method". + Methods []string + + // Origins is the list of Origins eligible to receive CORS response + // headers. Note: "*" is permitted in the list of origins, and means + // "any Origin". + Origins []string + + // ResponseHeaders is the list of HTTP headers other than the simple + // response headers to give permission for the user-agent to share + // across domains. + ResponseHeaders []string +} + +// BucketEncryption is a bucket's encryption configuration. +type BucketEncryption struct { + // A Cloud KMS key name, in the form + // projects/P/locations/L/keyRings/R/cryptoKeys/K, that will be used to encrypt + // objects inserted into this bucket, if no encryption method is specified. + // The key's location must be the same as the bucket's. + DefaultKMSKeyName string +} + type BucketAttrsToUpdate struct { - // VersioningEnabled, if set, updates whether the bucket uses versioning. + // If set, updates whether the bucket uses versioning. VersioningEnabled optional.Bool - // RequesterPays, if set, updates whether the bucket is a Requester Pays bucket. + // If set, updates whether the bucket is a Requester Pays bucket. RequesterPays optional.Bool + // If set, updates the retention policy of the bucket. Using + // RetentionPolicy.RetentionPeriod = 0 will delete the existing policy. + // + // This feature is in private alpha release. It is not currently available to + // most customers. It might be changed in backwards-incompatible ways and is not + // subject to any SLA or deprecation policy. + RetentionPolicy *RetentionPolicy + + // If set, replaces the CORS configuration with a new configuration. + // An empty (rather than nil) slice causes all CORS policies to be removed. + CORS []CORS + + // If set, replaces the encryption configuration of the bucket. Using + // BucketEncryption.DefaultKMSKeyName = "" will delete the existing + // configuration. + Encryption *BucketEncryption + + // If set, replaces the lifecycle configuration of the bucket. + Lifecycle *Lifecycle + setLabels map[string]string deleteLabels map[string]bool } @@ -442,6 +560,18 @@ func (ua *BucketAttrsToUpdate) DeleteLabel(name string) { func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket { rb := &raw.Bucket{} + if ua.CORS != nil { + rb.Cors = toRawCORS(ua.CORS) + rb.ForceSendFields = append(rb.ForceSendFields, "Cors") + } + if ua.RetentionPolicy != nil { + if ua.RetentionPolicy.RetentionPeriod == 0 { + rb.NullFields = append(rb.NullFields, "RetentionPolicy") + rb.RetentionPolicy = nil + } else { + rb.RetentionPolicy = ua.RetentionPolicy.toRawRetentionPolicy() + } + } if ua.VersioningEnabled != nil { rb.Versioning = &raw.BucketVersioning{ Enabled: optional.ToBool(ua.VersioningEnabled), @@ -454,6 +584,17 @@ func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket { ForceSendFields: []string{"RequesterPays"}, } } + if ua.Encryption != nil { + if ua.Encryption.DefaultKMSKeyName == "" { + rb.NullFields = append(rb.NullFields, "Encryption") + rb.Encryption = nil + } else { + rb.Encryption = ua.Encryption.toRawBucketEncryption() + } + } + if ua.Lifecycle != nil { + rb.Lifecycle = toRawLifecycle(*ua.Lifecycle) + } if ua.setLabels != nil || ua.deleteLabels != nil { rb.Labels = map[string]string{} for k, v := range ua.setLabels { @@ -506,8 +647,10 @@ func (c *BucketConditions) validate(method string) error { } // UserProject returns a new BucketHandle that passes the project ID as the user -// project for all subsequent calls. A user project is required for all operations -// on requester-pays buckets. +// project for all subsequent calls. Calls with a user project will be billed to that +// project rather than to the bucket's owning project. +// +// A user project is required for all operations on Requester Pays buckets. func (b *BucketHandle) UserProject(projectID string) *BucketHandle { b2 := *b b2.userProject = projectID @@ -516,6 +659,25 @@ func (b *BucketHandle) UserProject(projectID string) *BucketHandle { return &b2 } +// LockRetentionPolicy locks a bucket's retention policy until a previously-configured +// RetentionPeriod past the EffectiveTime. Note that if RetentionPeriod is set to less +// than a day, the retention policy is treated as a development configuration and locking +// will have no effect. The BucketHandle must have a metageneration condition that +// matches the bucket's metageneration. See BucketHandle.If. +// +// This feature is in private alpha release. It is not currently available to +// most customers. It might be changed in backwards-incompatible ways and is not +// subject to any SLA or deprecation policy. +func (b *BucketHandle) LockRetentionPolicy(ctx context.Context) error { + var metageneration int64 + if b.conds != nil { + metageneration = b.conds.MetagenerationMatch + } + req := b.c.raw.Buckets.LockRetentionPolicy(b.name, metageneration) + _, err := req.Context(ctx).Do() + return err +} + // applyBucketConds modifies the provided call using the conditions in conds. // call is something that quacks like a *raw.WhateverCall. func applyBucketConds(method string, conds *BucketConditions, call interface{}) error { @@ -539,6 +701,55 @@ func applyBucketConds(method string, conds *BucketConditions, call interface{}) return nil } +func (rp *RetentionPolicy) toRawRetentionPolicy() *raw.BucketRetentionPolicy { + if rp == nil { + return nil + } + return &raw.BucketRetentionPolicy{ + RetentionPeriod: int64(rp.RetentionPeriod / time.Second), + } +} + +func toRetentionPolicy(rp *raw.BucketRetentionPolicy) (*RetentionPolicy, error) { + if rp == nil { + return nil, nil + } + t, err := time.Parse(time.RFC3339, rp.EffectiveTime) + if err != nil { + return nil, err + } + return &RetentionPolicy{ + RetentionPeriod: time.Duration(rp.RetentionPeriod) * time.Second, + EffectiveTime: t, + }, nil +} + +func toRawCORS(c []CORS) []*raw.BucketCors { + var out []*raw.BucketCors + for _, v := range c { + out = append(out, &raw.BucketCors{ + MaxAgeSeconds: int64(v.MaxAge / time.Second), + Method: v.Methods, + Origin: v.Origins, + ResponseHeader: v.ResponseHeaders, + }) + } + return out +} + +func toCORS(rc []*raw.BucketCors) []CORS { + var out []CORS + for _, v := range rc { + out = append(out, CORS{ + MaxAge: time.Duration(v.MaxAgeSeconds) * time.Second, + Methods: v.Method, + Origins: v.Origin, + ResponseHeaders: v.ResponseHeader, + }) + } + return out +} + func toRawLifecycle(l Lifecycle) *raw.BucketLifecycle { var rl raw.BucketLifecycle if len(l.Rules) == 0 { @@ -604,10 +815,27 @@ func toLifecycle(rl *raw.BucketLifecycle) Lifecycle { if rr.Condition.CreatedBefore != "" { r.Condition.CreatedBefore, _ = time.Parse(rfc3339Date, rr.Condition.CreatedBefore) } + l.Rules = append(l.Rules, r) } return l } +func (e *BucketEncryption) toRawBucketEncryption() *raw.BucketEncryption { + if e == nil { + return nil + } + return &raw.BucketEncryption{ + DefaultKmsKeyName: e.DefaultKMSKeyName, + } +} + +func toBucketEncryption(e *raw.BucketEncryption) *BucketEncryption { + if e == nil { + return nil + } + return &BucketEncryption{DefaultKMSKeyName: e.DefaultKmsKeyName} +} + // Objects returns an iterator over the objects in the bucket that match the Query q. // If q is nil, no filtering is done. func (b *BucketHandle) Objects(ctx context.Context, q *Query) *ObjectIterator { @@ -689,8 +917,6 @@ func (it *ObjectIterator) fetch(pageSize int, pageToken string) (string, error) return resp.NextPageToken, nil } -// TODO(jbd): Add storage.buckets.update. - // Buckets returns an iterator over the buckets in the project. You may // optionally set the iterator's Prefix field to restrict the list to buckets // whose names begin with the prefix. By default, all buckets in the project @@ -736,7 +962,7 @@ func (it *BucketIterator) Next() (*BucketAttrs, error) { // PageInfo supports pagination. See the google.golang.org/api/iterator package for details. func (it *BucketIterator) PageInfo() *iterator.PageInfo { return it.pageInfo } -func (it *BucketIterator) fetch(pageSize int, pageToken string) (string, error) { +func (it *BucketIterator) fetch(pageSize int, pageToken string) (token string, err error) { req := it.client.raw.Buckets.List(it.projectID) setClientHeader(req.Header()) req.Projection("full") @@ -746,7 +972,6 @@ func (it *BucketIterator) fetch(pageSize int, pageToken string) (string, error) req.MaxResults(int64(pageSize)) } var resp *raw.Buckets - var err error err = runWithRetry(it.ctx, func() error { resp, err = req.Context(it.ctx).Do() return err @@ -755,7 +980,11 @@ func (it *BucketIterator) fetch(pageSize int, pageToken string) (string, error) return "", err } for _, item := range resp.Items { - it.buckets = append(it.buckets, newBucket(item)) + b, err := newBucket(item) + if err != nil { + return "", err + } + it.buckets = append(it.buckets, b) } return resp.NextPageToken, nil } diff --git a/vendor/cloud.google.com/go/storage/copy.go b/vendor/cloud.google.com/go/storage/copy.go index d0a999c..98ca5bd 100644 --- a/vendor/cloud.google.com/go/storage/copy.go +++ b/vendor/cloud.google.com/go/storage/copy.go @@ -1,4 +1,4 @@ -// Copyright 2016 Google Inc. All Rights Reserved. +// Copyright 2016 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ import ( "errors" "fmt" + "cloud.google.com/go/internal/trace" "golang.org/x/net/context" raw "google.golang.org/api/storage/v1" ) @@ -59,17 +60,32 @@ type Copier struct { // ProgressFunc should return quickly without blocking. ProgressFunc func(copiedBytes, totalBytes uint64) + // The Cloud KMS key, in the form projects/P/locations/L/keyRings/R/cryptoKeys/K, + // that will be used to encrypt the object. Overrides the object's KMSKeyName, if + // any. + // + // Providing both a DestinationKMSKeyName and a customer-supplied encryption key + // (via ObjectHandle.Key) on the destination object will result in an error when + // Run is called. + DestinationKMSKeyName string + dst, src *ObjectHandle } // Run performs the copy. -func (c *Copier) Run(ctx context.Context) (*ObjectAttrs, error) { +func (c *Copier) Run(ctx context.Context) (attrs *ObjectAttrs, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Copier.Run") + defer func() { trace.EndSpan(ctx, err) }() + if err := c.src.validate(); err != nil { return nil, err } if err := c.dst.validate(); err != nil { return nil, err } + if c.DestinationKMSKeyName != "" && c.dst.encryptionKey != nil { + return nil, errors.New("storage: cannot use DestinationKMSKeyName with a customer-supplied encryption key") + } // Convert destination attributes to raw form, omitting the bucket. // If the bucket is included but name or content-type aren't, the service // returns a 400 with "Required" as the only message. Omitting the bucket @@ -96,6 +112,9 @@ func (c *Copier) callRewrite(ctx context.Context, rawObj *raw.Object) (*raw.Rewr if c.RewriteToken != "" { call.RewriteToken(c.RewriteToken) } + if c.DestinationKMSKeyName != "" { + call.DestinationKmsKeyName(c.DestinationKMSKeyName) + } if err := applyConds("Copy destination", c.dst.gen, c.dst.conds, call); err != nil { return nil, err } @@ -149,7 +168,10 @@ type Composer struct { } // Run performs the compose operation. -func (c *Composer) Run(ctx context.Context) (*ObjectAttrs, error) { +func (c *Composer) Run(ctx context.Context) (attrs *ObjectAttrs, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Composer.Run") + defer func() { trace.EndSpan(ctx, err) }() + if err := c.dst.validate(); err != nil { return nil, err } @@ -191,7 +213,6 @@ func (c *Composer) Run(ctx context.Context) (*ObjectAttrs, error) { return nil, err } var obj *raw.Object - var err error setClientHeader(call.Header()) err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) if err != nil { diff --git a/vendor/cloud.google.com/go/storage/doc.go b/vendor/cloud.google.com/go/storage/doc.go index 951391f..5bd7708 100644 --- a/vendor/cloud.google.com/go/storage/doc.go +++ b/vendor/cloud.google.com/go/storage/doc.go @@ -1,4 +1,4 @@ -// Copyright 2016 Google Inc. All Rights Reserved. +// Copyright 2016 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,12 +19,13 @@ Google Cloud Storage stores data in named objects, which are grouped into bucket More information about Google Cloud Storage is available at https://cloud.google.com/storage/docs. +See https://godoc.org/cloud.google.com/go for authentication, timeouts, +connection pooling and similar aspects of this package. + All of the methods of this package use exponential backoff to retry calls that fail with certain errors, as described in https://cloud.google.com/storage/docs/exponential-backoff. -Note: This package is in beta. Some backwards-incompatible changes may occur. - Creating a Client @@ -36,6 +37,13 @@ To start working with this package, create a client: // TODO: Handle error. } +The client will use your default application credentials. + +If you only wish to access public data, you can create +an unauthenticated client with + + client, err := storage.NewClient(ctx, option.WithoutAuthentication()) + Buckets A Google Cloud Storage bucket is a collection of objects. To work with a @@ -56,7 +64,7 @@ global across all projects. Each bucket has associated metadata, represented in this package by BucketAttrs. The third argument to BucketHandle.Create allows you to set -the intial BucketAttrs of a bucket. To retrieve a bucket's attributes, use +the initial BucketAttrs of a bucket. To retrieve a bucket's attributes, use Attrs: attrs, err := bkt.Attrs(ctx) @@ -69,15 +77,16 @@ Attrs: Objects An object holds arbitrary data as a sequence of bytes, like a file. You -refer to objects using a handle, just as with buckets. You can use the -standard Go io.Reader and io.Writer interfaces to read and write -object data: +refer to objects using a handle, just as with buckets, but unlike buckets +you don't explicitly create an object. Instead, the first time you write +to an object it will be created. You can use the standard Go io.Reader +and io.Writer interfaces to read and write object data: obj := bkt.Object("data") // Write something to obj. // w implements io.Writer. w := obj.NewWriter(ctx) - // Write some text to obj. This will overwrite whatever is there. + // Write some text to obj. This will either create the object or overwrite whatever is there already. if _, err := fmt.Fprintf(w, "This object contains text.\n"); err != nil { // TODO: Handle error. } @@ -152,10 +161,5 @@ SignedURL for details. // TODO: Handle error. } fmt.Println(url) - -Authentication - -See examples of authorization and authentication at -https://godoc.org/cloud.google.com/go#pkg-examples. */ package storage // import "cloud.google.com/go/storage" diff --git a/vendor/cloud.google.com/go/storage/go110.go b/vendor/cloud.google.com/go/storage/go110.go index b85e8c3..75be3cb 100644 --- a/vendor/cloud.google.com/go/storage/go110.go +++ b/vendor/cloud.google.com/go/storage/go110.go @@ -1,4 +1,4 @@ -// Copyright 2017 Google Inc. All Rights Reserved. +// Copyright 2017 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/vendor/cloud.google.com/go/storage/go17.go b/vendor/cloud.google.com/go/storage/go17.go index 982db4e..39b8177 100644 --- a/vendor/cloud.google.com/go/storage/go17.go +++ b/vendor/cloud.google.com/go/storage/go17.go @@ -1,4 +1,4 @@ -// Copyright 2017 Google Inc. All Rights Reserved. +// Copyright 2017 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/vendor/cloud.google.com/go/storage/iam.go b/vendor/cloud.google.com/go/storage/iam.go index 6607d8c..d2cef42 100644 --- a/vendor/cloud.google.com/go/storage/iam.go +++ b/vendor/cloud.google.com/go/storage/iam.go @@ -1,4 +1,4 @@ -// Copyright 2017 Google Inc. All Rights Reserved. +// Copyright 2017 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ package storage import ( "cloud.google.com/go/iam" + "cloud.google.com/go/internal/trace" "golang.org/x/net/context" raw "google.golang.org/api/storage/v1" iampb "google.golang.org/genproto/googleapis/iam/v1" @@ -23,21 +24,30 @@ import ( // IAM provides access to IAM access control for the bucket. func (b *BucketHandle) IAM() *iam.Handle { - return iam.InternalNewHandleClient(&iamClient{raw: b.c.raw}, b.name) + return iam.InternalNewHandleClient(&iamClient{ + raw: b.c.raw, + userProject: b.userProject, + }, b.name) } // iamClient implements the iam.client interface. type iamClient struct { - raw *raw.Service + raw *raw.Service + userProject string } -func (c *iamClient) Get(ctx context.Context, resource string) (*iampb.Policy, error) { - req := c.raw.Buckets.GetIamPolicy(resource) - setClientHeader(req.Header()) +func (c *iamClient) Get(ctx context.Context, resource string) (p *iampb.Policy, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.IAM.Get") + defer func() { trace.EndSpan(ctx, err) }() + + call := c.raw.Buckets.GetIamPolicy(resource) + setClientHeader(call.Header()) + if c.userProject != "" { + call.UserProject(c.userProject) + } var rp *raw.Policy - var err error err = runWithRetry(ctx, func() error { - rp, err = req.Context(ctx).Do() + rp, err = call.Context(ctx).Do() return err }) if err != nil { @@ -46,23 +56,34 @@ func (c *iamClient) Get(ctx context.Context, resource string) (*iampb.Policy, er return iamFromStoragePolicy(rp), nil } -func (c *iamClient) Set(ctx context.Context, resource string, p *iampb.Policy) error { +func (c *iamClient) Set(ctx context.Context, resource string, p *iampb.Policy) (err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.IAM.Set") + defer func() { trace.EndSpan(ctx, err) }() + rp := iamToStoragePolicy(p) - req := c.raw.Buckets.SetIamPolicy(resource, rp) - setClientHeader(req.Header()) + call := c.raw.Buckets.SetIamPolicy(resource, rp) + setClientHeader(call.Header()) + if c.userProject != "" { + call.UserProject(c.userProject) + } return runWithRetry(ctx, func() error { - _, err := req.Context(ctx).Do() + _, err := call.Context(ctx).Do() return err }) } -func (c *iamClient) Test(ctx context.Context, resource string, perms []string) ([]string, error) { - req := c.raw.Buckets.TestIamPermissions(resource, perms) - setClientHeader(req.Header()) +func (c *iamClient) Test(ctx context.Context, resource string, perms []string) (permissions []string, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.IAM.Test") + defer func() { trace.EndSpan(ctx, err) }() + + call := c.raw.Buckets.TestIamPermissions(resource, perms) + setClientHeader(call.Header()) + if c.userProject != "" { + call.UserProject(c.userProject) + } var res *raw.TestIamPermissionsResponse - var err error err = runWithRetry(ctx, func() error { - res, err = req.Context(ctx).Do() + res, err = call.Context(ctx).Do() return err }) if err != nil { diff --git a/vendor/cloud.google.com/go/storage/invoke.go b/vendor/cloud.google.com/go/storage/invoke.go index 46423a8..955ef72 100644 --- a/vendor/cloud.google.com/go/storage/invoke.go +++ b/vendor/cloud.google.com/go/storage/invoke.go @@ -1,4 +1,4 @@ -// Copyright 2014 Google Inc. All Rights Reserved. +// Copyright 2014 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/vendor/cloud.google.com/go/storage/not_go110.go b/vendor/cloud.google.com/go/storage/not_go110.go index c354e74..700fde1 100644 --- a/vendor/cloud.google.com/go/storage/not_go110.go +++ b/vendor/cloud.google.com/go/storage/not_go110.go @@ -1,4 +1,4 @@ -// Copyright 2017 Google Inc. All Rights Reserved. +// Copyright 2017 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/vendor/cloud.google.com/go/storage/not_go17.go b/vendor/cloud.google.com/go/storage/not_go17.go index 1f6f7ae..0efb3e9 100644 --- a/vendor/cloud.google.com/go/storage/not_go17.go +++ b/vendor/cloud.google.com/go/storage/not_go17.go @@ -1,4 +1,4 @@ -// Copyright 2017 Google Inc. All Rights Reserved. +// Copyright 2017 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/vendor/cloud.google.com/go/storage/notifications.go b/vendor/cloud.google.com/go/storage/notifications.go new file mode 100644 index 0000000..d5e1395 --- /dev/null +++ b/vendor/cloud.google.com/go/storage/notifications.go @@ -0,0 +1,188 @@ +// Copyright 2017 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "errors" + "fmt" + "regexp" + + "cloud.google.com/go/internal/trace" + "golang.org/x/net/context" + raw "google.golang.org/api/storage/v1" +) + +// A Notification describes how to send Cloud PubSub messages when certain +// events occur in a bucket. +type Notification struct { + //The ID of the notification. + ID string + + // The ID of the topic to which this subscription publishes. + TopicID string + + // The ID of the project to which the topic belongs. + TopicProjectID string + + // Only send notifications about listed event types. If empty, send notifications + // for all event types. + // See https://cloud.google.com/storage/docs/pubsub-notifications#events. + EventTypes []string + + // If present, only apply this notification configuration to object names that + // begin with this prefix. + ObjectNamePrefix string + + // An optional list of additional attributes to attach to each Cloud PubSub + // message published for this notification subscription. + CustomAttributes map[string]string + + // The contents of the message payload. + // See https://cloud.google.com/storage/docs/pubsub-notifications#payload. + PayloadFormat string +} + +// Values for Notification.PayloadFormat. +const ( + // Send no payload with notification messages. + NoPayload = "NONE" + + // Send object metadata as JSON with notification messages. + JSONPayload = "JSON_API_V1" +) + +// Values for Notification.EventTypes. +const ( + // Event that occurs when an object is successfully created. + ObjectFinalizeEvent = "OBJECT_FINALIZE" + + // Event that occurs when the metadata of an existing object changes. + ObjectMetadataUpdateEvent = "OBJECT_METADATA_UPDATE" + + // Event that occurs when an object is permanently deleted. + ObjectDeleteEvent = "OBJECT_DELETE" + + // Event that occurs when the live version of an object becomes an + // archived version. + ObjectArchiveEvent = "OBJECT_ARCHIVE" +) + +func toNotification(rn *raw.Notification) *Notification { + n := &Notification{ + ID: rn.Id, + EventTypes: rn.EventTypes, + ObjectNamePrefix: rn.ObjectNamePrefix, + CustomAttributes: rn.CustomAttributes, + PayloadFormat: rn.PayloadFormat, + } + n.TopicProjectID, n.TopicID = parseNotificationTopic(rn.Topic) + return n +} + +var topicRE = regexp.MustCompile("^//pubsub.googleapis.com/projects/([^/]+)/topics/([^/]+)") + +// parseNotificationTopic extracts the project and topic IDs from from the full +// resource name returned by the service. If the name is malformed, it returns +// "?" for both IDs. +func parseNotificationTopic(nt string) (projectID, topicID string) { + matches := topicRE.FindStringSubmatch(nt) + if matches == nil { + return "?", "?" + } + return matches[1], matches[2] +} + +func toRawNotification(n *Notification) *raw.Notification { + return &raw.Notification{ + Id: n.ID, + Topic: fmt.Sprintf("//pubsub.googleapis.com/projects/%s/topics/%s", + n.TopicProjectID, n.TopicID), + EventTypes: n.EventTypes, + ObjectNamePrefix: n.ObjectNamePrefix, + CustomAttributes: n.CustomAttributes, + PayloadFormat: string(n.PayloadFormat), + } +} + +// AddNotification adds a notification to b. You must set n's TopicProjectID, TopicID +// and PayloadFormat, and must not set its ID. The other fields are all optional. The +// returned Notification's ID can be used to refer to it. +func (b *BucketHandle) AddNotification(ctx context.Context, n *Notification) (ret *Notification, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.AddNotification") + defer func() { trace.EndSpan(ctx, err) }() + + if n.ID != "" { + return nil, errors.New("storage: AddNotification: ID must not be set") + } + if n.TopicProjectID == "" { + return nil, errors.New("storage: AddNotification: missing TopicProjectID") + } + if n.TopicID == "" { + return nil, errors.New("storage: AddNotification: missing TopicID") + } + call := b.c.raw.Notifications.Insert(b.name, toRawNotification(n)) + setClientHeader(call.Header()) + if b.userProject != "" { + call.UserProject(b.userProject) + } + rn, err := call.Context(ctx).Do() + if err != nil { + return nil, err + } + return toNotification(rn), nil +} + +// Notifications returns all the Notifications configured for this bucket, as a map +// indexed by notification ID. +func (b *BucketHandle) Notifications(ctx context.Context) (n map[string]*Notification, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Notifications") + defer func() { trace.EndSpan(ctx, err) }() + + call := b.c.raw.Notifications.List(b.name) + setClientHeader(call.Header()) + if b.userProject != "" { + call.UserProject(b.userProject) + } + var res *raw.Notifications + err = runWithRetry(ctx, func() error { + res, err = call.Context(ctx).Do() + return err + }) + if err != nil { + return nil, err + } + return notificationsToMap(res.Items), nil +} + +func notificationsToMap(rns []*raw.Notification) map[string]*Notification { + m := map[string]*Notification{} + for _, rn := range rns { + m[rn.Id] = toNotification(rn) + } + return m +} + +// DeleteNotification deletes the notification with the given ID. +func (b *BucketHandle) DeleteNotification(ctx context.Context, id string) (err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.DeleteNotification") + defer func() { trace.EndSpan(ctx, err) }() + + call := b.c.raw.Notifications.Delete(b.name, id) + setClientHeader(call.Header()) + if b.userProject != "" { + call.UserProject(b.userProject) + } + return call.Context(ctx).Do() +} diff --git a/vendor/cloud.google.com/go/storage/reader.go b/vendor/cloud.google.com/go/storage/reader.go index c96ca8a..c0f26bf 100644 --- a/vendor/cloud.google.com/go/storage/reader.go +++ b/vendor/cloud.google.com/go/storage/reader.go @@ -1,4 +1,4 @@ -// Copyright 2016 Google Inc. All Rights Reserved. +// Copyright 2016 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,23 +15,196 @@ package storage import ( + "errors" "fmt" "hash/crc32" "io" + "io/ioutil" + "net/http" + "net/url" + "reflect" + "strconv" + "strings" + + "cloud.google.com/go/internal/trace" + "golang.org/x/net/context" + "google.golang.org/api/googleapi" ) var crc32cTable = crc32.MakeTable(crc32.Castagnoli) +// NewReader creates a new Reader to read the contents of the +// object. +// ErrObjectNotExist will be returned if the object is not found. +// +// The caller must call Close on the returned Reader when done reading. +func (o *ObjectHandle) NewReader(ctx context.Context) (*Reader, error) { + return o.NewRangeReader(ctx, 0, -1) +} + +// NewRangeReader reads part of an object, reading at most length bytes +// starting at the given offset. If length is negative, the object is read +// until the end. +func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (r *Reader, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.NewRangeReader") + defer func() { trace.EndSpan(ctx, err) }() + + if err := o.validate(); err != nil { + return nil, err + } + if offset < 0 { + return nil, fmt.Errorf("storage: invalid offset %d < 0", offset) + } + if o.conds != nil { + if err := o.conds.validate("NewRangeReader"); err != nil { + return nil, err + } + } + u := &url.URL{ + Scheme: "https", + Host: "storage.googleapis.com", + Path: fmt.Sprintf("/%s/%s", o.bucket, o.object), + RawQuery: conditionsQuery(o.gen, o.conds), + } + verb := "GET" + if length == 0 { + verb = "HEAD" + } + req, err := http.NewRequest(verb, u.String(), nil) + if err != nil { + return nil, err + } + req = withContext(req, ctx) + if o.userProject != "" { + req.Header.Set("X-Goog-User-Project", o.userProject) + } + if o.readCompressed { + req.Header.Set("Accept-Encoding", "gzip") + } + if err := setEncryptionHeaders(req.Header, o.encryptionKey, false); err != nil { + return nil, err + } + + // Define a function that initiates a Read with offset and length, assuming we + // have already read seen bytes. + reopen := func(seen int64) (*http.Response, error) { + start := offset + seen + if length < 0 && start > 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", start)) + } else if length > 0 { + // The end character isn't affected by how many bytes we've seen. + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, offset+length-1)) + } + var res *http.Response + err = runWithRetry(ctx, func() error { + res, err = o.c.hc.Do(req) + if err != nil { + return err + } + if res.StatusCode == http.StatusNotFound { + res.Body.Close() + return ErrObjectNotExist + } + if res.StatusCode < 200 || res.StatusCode > 299 { + body, _ := ioutil.ReadAll(res.Body) + res.Body.Close() + return &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + Body: string(body), + } + } + if start > 0 && length != 0 && res.StatusCode != http.StatusPartialContent { + res.Body.Close() + return errors.New("storage: partial request not satisfied") + } + return nil + }) + if err != nil { + return nil, err + } + return res, nil + } + + res, err := reopen(0) + if err != nil { + return nil, err + } + var size int64 // total size of object, even if a range was requested. + if res.StatusCode == http.StatusPartialContent { + cr := strings.TrimSpace(res.Header.Get("Content-Range")) + if !strings.HasPrefix(cr, "bytes ") || !strings.Contains(cr, "/") { + + return nil, fmt.Errorf("storage: invalid Content-Range %q", cr) + } + size, err = strconv.ParseInt(cr[strings.LastIndex(cr, "/")+1:], 10, 64) + if err != nil { + return nil, fmt.Errorf("storage: invalid Content-Range %q", cr) + } + } else { + size = res.ContentLength + } + + remain := res.ContentLength + body := res.Body + if length == 0 { + remain = 0 + body.Close() + body = emptyBody + } + var ( + checkCRC bool + crc uint32 + ) + // Even if there is a CRC header, we can't compute the hash on partial data. + if remain == size { + crc, checkCRC = parseCRC32c(res) + } + return &Reader{ + body: body, + size: size, + remain: remain, + contentType: res.Header.Get("Content-Type"), + contentEncoding: res.Header.Get("Content-Encoding"), + cacheControl: res.Header.Get("Cache-Control"), + wantCRC: crc, + checkCRC: checkCRC, + reopen: reopen, + }, nil +} + +func parseCRC32c(res *http.Response) (uint32, bool) { + const prefix = "crc32c=" + for _, spec := range res.Header["X-Goog-Hash"] { + if strings.HasPrefix(spec, prefix) { + c, err := decodeUint32(spec[len(prefix):]) + if err == nil { + return c, true + } + } + } + return 0, false +} + +var emptyBody = ioutil.NopCloser(strings.NewReader("")) + // Reader reads a Cloud Storage object. // It implements io.Reader. +// +// Typically, a Reader computes the CRC of the downloaded content and compares it to +// the stored CRC, returning an error from Read if there is a mismatch. This integrity check +// is skipped if transcoding occurs. See https://cloud.google.com/storage/docs/transcoding. type Reader struct { - body io.ReadCloser - remain, size int64 - contentType string - cacheControl string - checkCRC bool // should we check the CRC? - wantCRC uint32 // the CRC32c value the server sent in the header - gotCRC uint32 // running crc + body io.ReadCloser + seen, remain, size int64 + contentType string + contentEncoding string + cacheControl string + checkCRC bool // should we check the CRC? + wantCRC uint32 // the CRC32c value the server sent in the header + gotCRC uint32 // running crc + checkedCRC bool // did we check the CRC? (For tests.) + reopen func(seen int64) (*http.Response, error) } // Close closes the Reader. It must be called when done reading. @@ -40,7 +213,7 @@ func (r *Reader) Close() error { } func (r *Reader) Read(p []byte) (int, error) { - n, err := r.body.Read(p) + n, err := r.readWithRetry(p) if r.remain != -1 { r.remain -= int64(n) } @@ -49,14 +222,46 @@ func (r *Reader) Read(p []byte) (int, error) { // Check CRC here. It would be natural to check it in Close, but // everybody defers Close on the assumption that it doesn't return // anything worth looking at. - if r.remain == 0 && r.gotCRC != r.wantCRC { - return n, fmt.Errorf("storage: bad CRC on read: got %d, want %d", - r.gotCRC, r.wantCRC) + if r.remain == 0 { // Only check if we have Content-Length. + r.checkedCRC = true + if r.gotCRC != r.wantCRC { + return n, fmt.Errorf("storage: bad CRC on read: got %d, want %d", + r.gotCRC, r.wantCRC) + } } } return n, err } +func (r *Reader) readWithRetry(p []byte) (int, error) { + n := 0 + for len(p[n:]) > 0 { + m, err := r.body.Read(p[n:]) + n += m + r.seen += int64(m) + if !shouldRetryRead(err) { + return n, err + } + // Read failed, but we will try again. Send a ranged read request that takes + // into account the number of bytes we've already seen. + res, err := r.reopen(r.seen) + if err != nil { + // reopen already retries + return n, err + } + r.body.Close() + r.body = res.Body + } + return n, nil +} + +func shouldRetryRead(err error) bool { + if err == nil { + return false + } + return strings.HasSuffix(err.Error(), "INTERNAL_ERROR") && strings.Contains(reflect.TypeOf(err).String(), "http2") +} + // Size returns the size of the object in bytes. // The returned value is always the same and is not affected by // calls to Read or Close. @@ -74,6 +279,11 @@ func (r *Reader) ContentType() string { return r.contentType } +// ContentEncoding returns the content encoding of the object. +func (r *Reader) ContentEncoding() string { + return r.contentEncoding +} + // CacheControl returns the cache control of the object. func (r *Reader) CacheControl() string { return r.cacheControl diff --git a/vendor/cloud.google.com/go/storage/storage.go b/vendor/cloud.google.com/go/storage/storage.go index 8ffa845..2681922 100644 --- a/vendor/cloud.google.com/go/storage/storage.go +++ b/vendor/cloud.google.com/go/storage/storage.go @@ -1,4 +1,4 @@ -// Copyright 2014 Google Inc. All Rights Reserved. +// Copyright 2014 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -26,15 +26,17 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "net/url" "reflect" + "regexp" + "sort" "strconv" "strings" "time" "unicode/utf8" + "cloud.google.com/go/internal/trace" "google.golang.org/api/option" htransport "google.golang.org/api/transport/http" @@ -182,6 +184,60 @@ type SignedURLOptions struct { MD5 string } +var ( + canonicalHeaderRegexp = regexp.MustCompile(`(?i)^(x-goog-[^:]+):(.*)?$`) + excludedCanonicalHeaders = map[string]bool{ + "x-goog-encryption-key": true, + "x-goog-encryption-key-sha256": true, + } +) + +// sanitizeHeaders applies the specifications for canonical extension headers at +// https://cloud.google.com/storage/docs/access-control/signed-urls#about-canonical-extension-headers. +func sanitizeHeaders(hdrs []string) []string { + headerMap := map[string][]string{} + for _, hdr := range hdrs { + // No leading or trailing whitespaces. + sanitizedHeader := strings.TrimSpace(hdr) + + // Only keep canonical headers, discard any others. + headerMatches := canonicalHeaderRegexp.FindStringSubmatch(sanitizedHeader) + if len(headerMatches) == 0 { + continue + } + + header := strings.ToLower(strings.TrimSpace(headerMatches[1])) + if excludedCanonicalHeaders[headerMatches[1]] { + // Do not keep any deliberately excluded canonical headers when signing. + continue + } + value := strings.TrimSpace(headerMatches[2]) + if len(value) > 0 { + // Remove duplicate headers by appending the values of duplicates + // in their order of appearance. + headerMap[header] = append(headerMap[header], value) + } + } + + var sanitizedHeaders []string + for header, values := range headerMap { + // There should be no spaces around the colon separating the + // header name from the header value or around the values + // themselves. The values should be separated by commas. + // NOTE: The semantics for headers without a value are not clear. + // However from specifications these should be edge-cases + // anyway and we should assume that there will be no + // canonical headers using empty values. Any such headers + // are discarded at the regexp stage above. + sanitizedHeaders = append( + sanitizedHeaders, + fmt.Sprintf("%s:%s", header, strings.Join(values, ",")), + ) + } + sort.Strings(sanitizedHeaders) + return sanitizedHeaders +} + // SignedURL returns a URL for the specified object. Signed URLs allow // the users access to a restricted resource for a limited time without having a // Google account or signing in. For more information about the signed @@ -208,6 +264,7 @@ func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) { return "", errors.New("storage: invalid MD5 checksum") } } + opts.Headers = sanitizeHeaders(opts.Headers) signBytes := opts.SignBytes if opts.PrivateKey != nil { @@ -258,14 +315,15 @@ func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) { // ObjectHandle provides operations on an object in a Google Cloud Storage bucket. // Use BucketHandle.Object to get a handle. type ObjectHandle struct { - c *Client - bucket string - object string - acl ACLHandle - gen int64 // a negative value indicates latest - conds *Conditions - encryptionKey []byte // AES-256 key - userProject string // for requester-pays buckets + c *Client + bucket string + object string + acl ACLHandle + gen int64 // a negative value indicates latest + conds *Conditions + encryptionKey []byte // AES-256 key + userProject string // for requester-pays buckets + readCompressed bool // Accept-Encoding: gzip } // ACL provides access to the object's access control list. @@ -310,7 +368,10 @@ func (o *ObjectHandle) Key(encryptionKey []byte) *ObjectHandle { // Attrs returns meta information about the object. // ErrObjectNotExist will be returned if the object is not found. -func (o *ObjectHandle) Attrs(ctx context.Context) (*ObjectAttrs, error) { +func (o *ObjectHandle) Attrs(ctx context.Context) (attrs *ObjectAttrs, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.Attrs") + defer func() { trace.EndSpan(ctx, err) }() + if err := o.validate(); err != nil { return nil, err } @@ -325,7 +386,6 @@ func (o *ObjectHandle) Attrs(ctx context.Context) (*ObjectAttrs, error) { return nil, err } var obj *raw.Object - var err error setClientHeader(call.Header()) err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { @@ -340,7 +400,10 @@ func (o *ObjectHandle) Attrs(ctx context.Context) (*ObjectAttrs, error) { // Update updates an object with the provided attributes. // All zero-value attributes are ignored. // ErrObjectNotExist will be returned if the object is not found. -func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (*ObjectAttrs, error) { +func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (oa *ObjectAttrs, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.Update") + defer func() { trace.EndSpan(ctx, err) }() + if err := o.validate(); err != nil { return nil, err } @@ -408,7 +471,6 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) ( return nil, err } var obj *raw.Object - var err error setClientHeader(call.Header()) err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { @@ -467,140 +529,13 @@ func (o *ObjectHandle) Delete(ctx context.Context) error { return err } -// NewReader creates a new Reader to read the contents of the -// object. -// ErrObjectNotExist will be returned if the object is not found. -// -// The caller must call Close on the returned Reader when done reading. -func (o *ObjectHandle) NewReader(ctx context.Context) (*Reader, error) { - return o.NewRangeReader(ctx, 0, -1) -} - -// NewRangeReader reads part of an object, reading at most length bytes -// starting at the given offset. If length is negative, the object is read -// until the end. -func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (*Reader, error) { - if err := o.validate(); err != nil { - return nil, err - } - if offset < 0 { - return nil, fmt.Errorf("storage: invalid offset %d < 0", offset) - } - if o.conds != nil { - if err := o.conds.validate("NewRangeReader"); err != nil { - return nil, err - } - } - u := &url.URL{ - Scheme: "https", - Host: "storage.googleapis.com", - Path: fmt.Sprintf("/%s/%s", o.bucket, o.object), - RawQuery: conditionsQuery(o.gen, o.conds), - } - verb := "GET" - if length == 0 { - verb = "HEAD" - } - req, err := http.NewRequest(verb, u.String(), nil) - if err != nil { - return nil, err - } - req = withContext(req, ctx) - if length < 0 && offset > 0 { - req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) - } else if length > 0 { - req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) - } - if o.userProject != "" { - req.Header.Set("X-Goog-User-Project", o.userProject) - } - if err := setEncryptionHeaders(req.Header, o.encryptionKey, false); err != nil { - return nil, err - } - var res *http.Response - err = runWithRetry(ctx, func() error { - res, err = o.c.hc.Do(req) - if err != nil { - return err - } - if res.StatusCode == http.StatusNotFound { - res.Body.Close() - return ErrObjectNotExist - } - if res.StatusCode < 200 || res.StatusCode > 299 { - body, _ := ioutil.ReadAll(res.Body) - res.Body.Close() - return &googleapi.Error{ - Code: res.StatusCode, - Header: res.Header, - Body: string(body), - } - } - if offset > 0 && length != 0 && res.StatusCode != http.StatusPartialContent { - res.Body.Close() - return errors.New("storage: partial request not satisfied") - } - return nil - }) - if err != nil { - return nil, err - } - - var size int64 // total size of object, even if a range was requested. - if res.StatusCode == http.StatusPartialContent { - cr := strings.TrimSpace(res.Header.Get("Content-Range")) - if !strings.HasPrefix(cr, "bytes ") || !strings.Contains(cr, "/") { - return nil, fmt.Errorf("storage: invalid Content-Range %q", cr) - } - size, err = strconv.ParseInt(cr[strings.LastIndex(cr, "/")+1:], 10, 64) - if err != nil { - return nil, fmt.Errorf("storage: invalid Content-Range %q", cr) - } - } else { - size = res.ContentLength - } - - remain := res.ContentLength - body := res.Body - if length == 0 { - remain = 0 - body.Close() - body = emptyBody - } - var ( - checkCRC bool - crc uint32 - ) - // Even if there is a CRC header, we can't compute the hash on partial data. - if remain == size { - crc, checkCRC = parseCRC32c(res) - } - return &Reader{ - body: body, - size: size, - remain: remain, - contentType: res.Header.Get("Content-Type"), - cacheControl: res.Header.Get("Cache-Control"), - wantCRC: crc, - checkCRC: checkCRC, - }, nil -} - -func parseCRC32c(res *http.Response) (uint32, bool) { - const prefix = "crc32c=" - for _, spec := range res.Header["X-Goog-Hash"] { - if strings.HasPrefix(spec, prefix) { - c, err := decodeUint32(spec[len(prefix):]) - if err == nil { - return c, true - } - } - } - return 0, false +// ReadCompressed when true causes the read to happen without decompressing. +func (o *ObjectHandle) ReadCompressed(compressed bool) *ObjectHandle { + o2 := *o + o2.readCompressed = compressed + return &o2 } -var emptyBody = ioutil.NopCloser(strings.NewReader("")) - // NewWriter returns a storage Writer that writes to the GCS object // associated with this ObjectHandle. // @@ -787,6 +722,14 @@ type ObjectAttrs struct { // encryption in Google Cloud Storage. CustomerKeySHA256 string + // Cloud KMS key name, in the form + // projects/P/locations/L/keyRings/R/cryptoKeys/K, used to encrypt this object, + // if the object is encrypted by such a key. + // + // Providing both a KMSKeyName and a customer-supplied encryption key (via + // ObjectHandle.Key) will result in an error when writing an object. + KMSKeyName string + // Prefix is set only for ObjectAttrs which represent synthetic "directory // entries" when iterating over buckets using Query.Delimiter. See // ObjectIterator.Next. When set, no other fields in ObjectAttrs will be @@ -844,6 +787,7 @@ func newObject(o *raw.Object) *ObjectAttrs { Metageneration: o.Metageneration, StorageClass: o.StorageClass, CustomerKeySHA256: sha256, + KMSKeyName: o.KmsKeyName, Created: convertTime(o.TimeCreated), Deleted: convertTime(o.TimeDeleted), Updated: convertTime(o.Updated), diff --git a/vendor/cloud.google.com/go/storage/writer.go b/vendor/cloud.google.com/go/storage/writer.go index 28eb74a..93597c8 100644 --- a/vendor/cloud.google.com/go/storage/writer.go +++ b/vendor/cloud.google.com/go/storage/writer.go @@ -1,4 +1,4 @@ -// Copyright 2014 Google Inc. All Rights Reserved. +// Copyright 2014 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ import ( "errors" "fmt" "io" + "sync" "unicode/utf8" "golang.org/x/net/context" @@ -47,8 +48,11 @@ type Writer struct { // to the nearest multiple of 256K. If zero, chunking will be disabled and // the object will be uploaded in a single request. // - // ChunkSize will default to a reasonable value. Any custom configuration - // must be done before the first Write call. + // ChunkSize will default to a reasonable value. If you perform many concurrent + // writes of small objects, you may wish set ChunkSize to a value that matches + // your objects' sizes to avoid consuming large amounts of memory. + // + // ChunkSize must be set before the first Write call. ChunkSize int // ProgressFunc can be used to monitor the progress of a large write. @@ -68,8 +72,10 @@ type Writer struct { pw *io.PipeWriter donec chan struct{} // closed after err and obj are set. - err error obj *ObjectAttrs + + mu sync.Mutex + err error } func (w *Writer) open() error { @@ -82,6 +88,9 @@ func (w *Writer) open() error { if !utf8.ValidString(attrs.Name) { return fmt.Errorf("storage: object name %q is not valid UTF-8", attrs.Name) } + if attrs.KMSKeyName != "" && w.o.encryptionKey != nil { + return errors.New("storage: cannot use KMSKeyName with a customer-supplied encryption key") + } pr, pw := io.Pipe() w.pw = pw w.opened = true @@ -113,9 +122,14 @@ func (w *Writer) open() error { if w.ProgressFunc != nil { call.ProgressUpdater(func(n, _ int64) { w.ProgressFunc(n) }) } + if attrs.KMSKeyName != "" { + call.KmsKeyName(attrs.KMSKeyName) + } if err := setEncryptionHeaders(call.Header(), w.o.encryptionKey, false); err != nil { + w.mu.Lock() w.err = err - pr.CloseWithError(w.err) + w.mu.Unlock() + pr.CloseWithError(err) return } var resp *raw.Object @@ -142,8 +156,10 @@ func (w *Writer) open() error { } } if err != nil { + w.mu.Lock() w.err = err - pr.CloseWithError(w.err) + w.mu.Unlock() + pr.CloseWithError(err) return } w.obj = newObject(resp) @@ -158,8 +174,11 @@ func (w *Writer) open() error { // use the error returned from Writer.Close to determine if // the upload was successful. func (w *Writer) Write(p []byte) (n int, err error) { - if w.err != nil { - return 0, w.err + w.mu.Lock() + werr := w.err + w.mu.Unlock() + if werr != nil { + return 0, werr } if !w.opened { if err := w.open(); err != nil { @@ -182,11 +201,15 @@ func (w *Writer) Close() error { return err } <-w.donec + w.mu.Lock() + defer w.mu.Unlock() return w.err } // CloseWithError aborts the write operation with the provided error. // CloseWithError always returns nil. +// +// Deprecated: cancel the context passed to NewWriter instead. func (w *Writer) CloseWithError(err error) error { if !w.opened { return nil |