From 7b320119ba532fd409ec7dade7ad02011c309599 Mon Sep 17 00:00:00 2001 From: Niall Sheridan Date: Wed, 18 Oct 2017 13:15:14 +0100 Subject: Update dependencies --- .../go/internal/optional/optional.go | 14 + .../go/internal/version/version.go | 2 +- vendor/cloud.google.com/go/storage/acl.go | 95 +++-- vendor/cloud.google.com/go/storage/bucket.go | 444 ++++++++++++++++++++- vendor/cloud.google.com/go/storage/copy.go | 24 +- vendor/cloud.google.com/go/storage/go110.go | 30 ++ vendor/cloud.google.com/go/storage/go17.go | 26 ++ vendor/cloud.google.com/go/storage/iam.go | 17 +- vendor/cloud.google.com/go/storage/invoke.go | 9 +- vendor/cloud.google.com/go/storage/not_go110.go | 40 ++ vendor/cloud.google.com/go/storage/not_go17.go | 26 ++ vendor/cloud.google.com/go/storage/reader.go | 6 + vendor/cloud.google.com/go/storage/storage.go | 143 ++++--- vendor/cloud.google.com/go/storage/writer.go | 41 +- 14 files changed, 771 insertions(+), 146 deletions(-) create mode 100644 vendor/cloud.google.com/go/storage/go110.go create mode 100644 vendor/cloud.google.com/go/storage/go17.go create mode 100644 vendor/cloud.google.com/go/storage/not_go110.go create mode 100644 vendor/cloud.google.com/go/storage/not_go17.go (limited to 'vendor/cloud.google.com') diff --git a/vendor/cloud.google.com/go/internal/optional/optional.go b/vendor/cloud.google.com/go/internal/optional/optional.go index f9102f3..4c15410 100644 --- a/vendor/cloud.google.com/go/internal/optional/optional.go +++ b/vendor/cloud.google.com/go/internal/optional/optional.go @@ -20,6 +20,7 @@ package optional import ( "fmt" "strings" + "time" ) type ( @@ -37,6 +38,9 @@ type ( // Float64 is either a float64 or nil. Float64 interface{} + + // Duration is either a time.Duration or nil. + Duration interface{} ) // ToBool returns its argument as a bool. @@ -89,6 +93,16 @@ func ToFloat64(v Float64) float64 { return x } +// ToDuration returns its argument as a time.Duration. +// It panics if its argument is nil or not a time.Duration. +func ToDuration(v Duration) time.Duration { + x, ok := v.(time.Duration) + if !ok { + doPanic("Duration", v) + } + return x +} + func doPanic(capType string, v interface{}) { panic(fmt.Sprintf("optional.%s value should be %s, got %T", capType, strings.ToLower(capType), v)) } diff --git a/vendor/cloud.google.com/go/internal/version/version.go b/vendor/cloud.google.com/go/internal/version/version.go index 33f1cdb..513afa4 100644 --- a/vendor/cloud.google.com/go/internal/version/version.go +++ b/vendor/cloud.google.com/go/internal/version/version.go @@ -26,7 +26,7 @@ import ( // Repo is the current version of the client libraries in this // repo. It should be a date in YYYYMMDD format. -const Repo = "20170404" +const Repo = "20170928" // Go returns the Go runtime version. The returned string // has no whitespace. diff --git a/vendor/cloud.google.com/go/storage/acl.go b/vendor/cloud.google.com/go/storage/acl.go index 041e541..a1b2b6d 100644 --- a/vendor/cloud.google.com/go/storage/acl.go +++ b/vendor/cloud.google.com/go/storage/acl.go @@ -16,8 +16,11 @@ package storage import ( "fmt" + "net/http" + "reflect" "golang.org/x/net/context" + "google.golang.org/api/googleapi" raw "google.golang.org/api/storage/v1" ) @@ -53,10 +56,11 @@ type ACLRule struct { // ACLHandle provides operations on an access control list for a Google Cloud Storage bucket or object. type ACLHandle struct { - c *Client - bucket string - object string - isDefault bool + c *Client + bucket string + object string + isDefault bool + userProject string // for requester-pays buckets } // Delete permanently deletes the ACL entry for the given entity. @@ -73,10 +77,10 @@ func (a *ACLHandle) Delete(ctx context.Context, entity ACLEntity) error { // Set sets the permission level for the given entity. func (a *ACLHandle) Set(ctx context.Context, entity ACLEntity, role ACLRole) error { if a.object != "" { - return a.objectSet(ctx, entity, role) + return a.objectSet(ctx, entity, role, false) } if a.isDefault { - return a.bucketDefaultSet(ctx, entity, role) + return a.objectSet(ctx, entity, role, true) } return a.bucketSet(ctx, entity, role) } @@ -96,8 +100,8 @@ func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) { var acls *raw.ObjectAccessControls var err error err = runWithRetry(ctx, func() error { - req := a.c.raw.DefaultObjectAccessControls.List(a.bucket).Context(ctx) - setClientHeader(req.Header()) + req := a.c.raw.DefaultObjectAccessControls.List(a.bucket) + a.configureCall(req, ctx) acls, err = req.Do() return err }) @@ -107,28 +111,10 @@ func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) { return toACLRules(acls.Items), nil } -func (a *ACLHandle) bucketDefaultSet(ctx context.Context, entity ACLEntity, role ACLRole) error { - acl := &raw.ObjectAccessControl{ - Bucket: a.bucket, - Entity: string(entity), - Role: string(role), - } - err := runWithRetry(ctx, func() error { - req := a.c.raw.DefaultObjectAccessControls.Update(a.bucket, string(entity), acl).Context(ctx) - setClientHeader(req.Header()) - _, err := req.Do() - return err - }) - if err != nil { - return fmt.Errorf("storage: error updating default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err) - } - return nil -} - func (a *ACLHandle) bucketDefaultDelete(ctx context.Context, entity ACLEntity) error { err := runWithRetry(ctx, func() error { - req := a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity)).Context(ctx) - setClientHeader(req.Header()) + req := a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity)) + a.configureCall(req, ctx) return req.Do() }) if err != nil { @@ -141,8 +127,8 @@ func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) { var acls *raw.BucketAccessControls var err error err = runWithRetry(ctx, func() error { - req := a.c.raw.BucketAccessControls.List(a.bucket).Context(ctx) - setClientHeader(req.Header()) + req := a.c.raw.BucketAccessControls.List(a.bucket) + a.configureCall(req, ctx) acls, err = req.Do() return err }) @@ -164,8 +150,8 @@ func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRol Role: string(role), } err := runWithRetry(ctx, func() error { - req := a.c.raw.BucketAccessControls.Update(a.bucket, string(entity), acl).Context(ctx) - setClientHeader(req.Header()) + req := a.c.raw.BucketAccessControls.Update(a.bucket, string(entity), acl) + a.configureCall(req, ctx) _, err := req.Do() return err }) @@ -177,8 +163,8 @@ func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRol func (a *ACLHandle) bucketDelete(ctx context.Context, entity ACLEntity) error { err := runWithRetry(ctx, func() error { - req := a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity)).Context(ctx) - setClientHeader(req.Header()) + req := a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity)) + a.configureCall(req, ctx) return req.Do() }) if err != nil { @@ -191,8 +177,8 @@ func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) { var acls *raw.ObjectAccessControls var err error err = runWithRetry(ctx, func() error { - req := a.c.raw.ObjectAccessControls.List(a.bucket, a.object).Context(ctx) - setClientHeader(req.Header()) + req := a.c.raw.ObjectAccessControls.List(a.bucket, a.object) + a.configureCall(req, ctx) acls, err = req.Do() return err }) @@ -202,28 +188,42 @@ func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) { return toACLRules(acls.Items), nil } -func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRole) error { +func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRole, isBucketDefault bool) error { + type setRequest interface { + Do(opts ...googleapi.CallOption) (*raw.ObjectAccessControl, error) + Header() http.Header + } + acl := &raw.ObjectAccessControl{ Bucket: a.bucket, Entity: string(entity), Role: string(role), } + var req setRequest + if isBucketDefault { + req = a.c.raw.DefaultObjectAccessControls.Update(a.bucket, string(entity), acl) + } else { + req = a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl) + } + a.configureCall(req, ctx) err := runWithRetry(ctx, func() error { - req := a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl).Context(ctx) - setClientHeader(req.Header()) _, err := req.Do() return err }) if err != nil { - return fmt.Errorf("storage: error updating object ACL entry for bucket %q, file %q, entity %q: %v", a.bucket, a.object, entity, err) + if isBucketDefault { + return fmt.Errorf("storage: error updating default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err) + } else { + return fmt.Errorf("storage: error updating object ACL entry for bucket %q, object %q, entity %q: %v", a.bucket, a.object, entity, err) + } } return nil } func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error { err := runWithRetry(ctx, func() error { - req := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity)).Context(ctx) - setClientHeader(req.Header()) + req := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity)) + a.configureCall(req, ctx) return req.Do() }) if err != nil { @@ -232,6 +232,17 @@ func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error { return nil } +func (a *ACLHandle) configureCall(call interface { + Header() http.Header +}, ctx context.Context) { + vc := reflect.ValueOf(call) + vc.MethodByName("Context").Call([]reflect.Value{reflect.ValueOf(ctx)}) + if a.userProject != "" { + vc.MethodByName("UserProject").Call([]reflect.Value{reflect.ValueOf(a.userProject)}) + } + setClientHeader(call.Header()) +} + func toACLRules(items []*raw.ObjectAccessControl) []ACLRule { r := make([]ACLRule, 0, len(items)) for _, item := range items { diff --git a/vendor/cloud.google.com/go/storage/bucket.go b/vendor/cloud.google.com/go/storage/bucket.go index deed5b2..07852a5 100644 --- a/vendor/cloud.google.com/go/storage/bucket.go +++ b/vendor/cloud.google.com/go/storage/bucket.go @@ -1,4 +1,4 @@ -// Copyright 2014 Google Inc. All Rights Reserved. +// Copyright 2014 Google Inc. LiveAndArchived Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,15 +15,52 @@ package storage import ( + "fmt" "net/http" + "reflect" "time" + "cloud.google.com/go/internal/optional" "golang.org/x/net/context" "google.golang.org/api/googleapi" "google.golang.org/api/iterator" raw "google.golang.org/api/storage/v1" ) +// BucketHandle provides operations on a Google Cloud Storage bucket. +// Use Client.Bucket to get a handle. +type BucketHandle struct { + c *Client + name string + acl ACLHandle + defaultObjectACL ACLHandle + conds *BucketConditions + userProject string // project for requester-pays buckets +} + +// Bucket returns a BucketHandle, which provides operations on the named bucket. +// This call does not perform any network operations. +// +// The supplied name must contain only lowercase letters, numbers, dashes, +// underscores, and dots. The full specification for valid bucket names can be +// found at: +// https://cloud.google.com/storage/docs/bucket-naming +func (c *Client) Bucket(name string) *BucketHandle { + return &BucketHandle{ + c: c, + name: name, + acl: ACLHandle{ + c: c, + bucket: name, + }, + defaultObjectACL: ACLHandle{ + c: c, + bucket: name, + isDefault: true, + }, + } +} + // Create creates the Bucket in the project. // If attrs is nil the API defaults will be used. func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *BucketAttrs) error { @@ -34,6 +71,11 @@ func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *Buck bkt = &raw.Bucket{} } bkt.Name = b.name + // If there is lifecycle information but no location, explicitly set + // the location. This is a GCS quirk/bug. + if bkt.Location == "" && bkt.Lifecycle != nil { + bkt.Location = "US" + } req := b.c.raw.Buckets.Insert(projectID, bkt) setClientHeader(req.Header()) return runWithRetry(ctx, func() error { _, err := req.Context(ctx).Do(); return err }) @@ -41,9 +83,23 @@ func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *Buck // Delete deletes the Bucket. func (b *BucketHandle) Delete(ctx context.Context) error { + req, err := b.newDeleteCall() + if err != nil { + return err + } + return runWithRetry(ctx, func() error { return req.Context(ctx).Do() }) +} + +func (b *BucketHandle) newDeleteCall() (*raw.BucketsDeleteCall, error) { req := b.c.raw.Buckets.Delete(b.name) setClientHeader(req.Header()) - return runWithRetry(ctx, func() error { return req.Context(ctx).Do() }) + if err := applyBucketConds("BucketHandle.Delete", b.conds, req); err != nil { + return nil, err + } + if b.userProject != "" { + req.UserProject(b.userProject) + } + return req, nil } // ACL returns an ACLHandle, which provides access to the bucket's access control list. @@ -72,20 +128,23 @@ func (b *BucketHandle) Object(name string) *ObjectHandle { bucket: b.name, object: name, acl: ACLHandle{ - c: b.c, - bucket: b.name, - object: name, + c: b.c, + bucket: b.name, + object: name, + userProject: b.userProject, }, - gen: -1, + gen: -1, + userProject: b.userProject, } } // Attrs returns the metadata for the bucket. func (b *BucketHandle) Attrs(ctx context.Context) (*BucketAttrs, error) { - req := b.c.raw.Buckets.Get(b.name).Projection("full") - setClientHeader(req.Header()) + req, err := b.newGetCall() + if err != nil { + return nil, err + } var resp *raw.Bucket - var err error err = runWithRetry(ctx, func() error { resp, err = req.Context(ctx).Do() return err @@ -99,9 +158,49 @@ func (b *BucketHandle) Attrs(ctx context.Context) (*BucketAttrs, error) { return newBucket(resp), nil } +func (b *BucketHandle) newGetCall() (*raw.BucketsGetCall, error) { + req := b.c.raw.Buckets.Get(b.name).Projection("full") + setClientHeader(req.Header()) + if err := applyBucketConds("BucketHandle.Attrs", b.conds, req); err != nil { + return nil, err + } + if b.userProject != "" { + req.UserProject(b.userProject) + } + return req, nil +} + +func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (*BucketAttrs, error) { + req, err := b.newPatchCall(&uattrs) + if err != nil { + return nil, err + } + // TODO(jba): retry iff metagen is set? + rb, err := req.Context(ctx).Do() + if err != nil { + return nil, err + } + return newBucket(rb), nil +} + +func (b *BucketHandle) newPatchCall(uattrs *BucketAttrsToUpdate) (*raw.BucketsPatchCall, error) { + rb := uattrs.toRawBucket() + req := b.c.raw.Buckets.Patch(b.name, rb).Projection("full") + setClientHeader(req.Header()) + if err := applyBucketConds("BucketHandle.Update", b.conds, req); err != nil { + return nil, err + } + if b.userProject != "" { + req.UserProject(b.userProject) + } + return req, nil +} + // BucketAttrs represents the metadata for a Google Cloud Storage bucket. +// Read-only fields are ignored by BucketHandle.Create. type BucketAttrs struct { // Name is the name of the bucket. + // This field is read-only. Name string // ACL is the list of access control rules on the bucket. @@ -115,6 +214,7 @@ type BucketAttrs struct { Location string // MetaGeneration is the metadata generation of the bucket. + // This field is read-only. MetaGeneration int64 // StorageClass is the default storage class of the bucket. This defines @@ -127,11 +227,109 @@ type BucketAttrs struct { StorageClass string // Created is the creation time of the bucket. + // This field is read-only. Created time.Time // VersioningEnabled reports whether this bucket has versioning enabled. - // This field is read-only. VersioningEnabled bool + + // Labels are the bucket's labels. + Labels map[string]string + + // RequesterPays reports whether the bucket is a Requester Pays bucket. + RequesterPays bool + // Lifecycle is the lifecycle configuration for objects in the bucket. + Lifecycle Lifecycle +} + +// Lifecycle is the lifecycle configuration for objects in the bucket. +type Lifecycle struct { + Rules []LifecycleRule +} + +const ( + // RFC3339 date with only the date segment, used for CreatedBefore in LifecycleRule. + rfc3339Date = "2006-01-02" + + // DeleteAction is a lifecycle action that deletes a live and/or archived + // objects. Takes precendence over SetStorageClass actions. + DeleteAction = "Delete" + + // SetStorageClassAction changes the storage class of live and/or archived + // objects. + SetStorageClassAction = "SetStorageClass" +) + +// LifecycleRule is a lifecycle configuration rule. +// +// When all the configured conditions are met by an object in the bucket, the +// configured action will automatically be taken on that object. +type LifecycleRule struct { + // Action is the action to take when all of the associated conditions are + // met. + Action LifecycleAction + + // Condition is the set of conditions that must be met for the associated + // action to be taken. + Condition LifecycleCondition +} + +// LifecycleAction is a lifecycle configuration action. +type LifecycleAction struct { + // Type is the type of action to take on matching objects. + // + // Acceptable values are "Delete" to delete matching objects and + // "SetStorageClass" to set the storage class defined in StorageClass on + // matching objects. + Type string + + // StorageClass is the storage class to set on matching objects if the Action + // is "SetStorageClass". + StorageClass string +} + +// Liveness specifies whether the object is live or not. +type Liveness int + +const ( + // LiveAndArchived includes both live and archived objects. + LiveAndArchived Liveness = iota + // Live specifies that the object is still live. + Live + // Archived specifies that the object is archived. + Archived +) + +// LifecycleCondition is a set of conditions used to match objects and take an +// action automatically. +// +// All configured conditions must be met for the associated action to be taken. +type LifecycleCondition struct { + // AgeInDays is the age of the object in days. + AgeInDays int64 + + // CreatedBefore is the time the object was created. + // + // This condition is satisfied when an object is created before midnight of + // the specified date in UTC. + CreatedBefore time.Time + + // Liveness specifies the object's liveness. Relevant only for versioned objects + Liveness Liveness + + // MatchesStorageClasses is the condition matching the object's storage + // class. + // + // Values include "MULTI_REGIONAL", "REGIONAL", "NEARLINE", "COLDLINE", + // "STANDARD", and "DURABLE_REDUCED_AVAILABILITY". + MatchesStorageClasses []string + + // NumNewerVersions is the condition matching objects with a number of newer versions. + // + // If the value is N, this condition is satisfied when there are at least N + // versions (including the live version) newer than this version of the + // object. + NumNewerVersions int64 } func newBucket(b *raw.Bucket) *BucketAttrs { @@ -145,6 +343,9 @@ func newBucket(b *raw.Bucket) *BucketAttrs { StorageClass: b.StorageClass, Created: convertTime(b.TimeCreated), VersioningEnabled: b.Versioning != nil && b.Versioning.Enabled, + Labels: b.Labels, + RequesterPays: b.Billing != nil && b.Billing.RequesterPays, + Lifecycle: toLifecycle(b.Lifecycle), } acl := make([]ACLRule, len(b.Acl)) for i, rule := range b.Acl { @@ -178,13 +379,233 @@ func (b *BucketAttrs) toRawBucket() *raw.Bucket { } } dACL := toRawObjectACL(b.DefaultObjectACL) + // Copy label map. + var labels map[string]string + if len(b.Labels) > 0 { + labels = make(map[string]string, len(b.Labels)) + for k, v := range b.Labels { + labels[k] = v + } + } + // Ignore VersioningEnabled if it is false. This is OK because + // we only call this method when creating a bucket, and by default + // new buckets have versioning off. + var v *raw.BucketVersioning + if b.VersioningEnabled { + v = &raw.BucketVersioning{Enabled: true} + } + var bb *raw.BucketBilling + if b.RequesterPays { + bb = &raw.BucketBilling{RequesterPays: true} + } return &raw.Bucket{ Name: b.Name, DefaultObjectAcl: dACL, Location: b.Location, StorageClass: b.StorageClass, Acl: acl, + Versioning: v, + Labels: labels, + Billing: bb, + Lifecycle: toRawLifecycle(b.Lifecycle), + } +} + +type BucketAttrsToUpdate struct { + // VersioningEnabled, if set, updates whether the bucket uses versioning. + VersioningEnabled optional.Bool + + // RequesterPays, if set, updates whether the bucket is a Requester Pays bucket. + RequesterPays optional.Bool + + setLabels map[string]string + deleteLabels map[string]bool +} + +// SetLabel causes a label to be added or modified when ua is used +// in a call to Bucket.Update. +func (ua *BucketAttrsToUpdate) SetLabel(name, value string) { + if ua.setLabels == nil { + ua.setLabels = map[string]string{} + } + ua.setLabels[name] = value +} + +// DeleteLabel causes a label to be deleted when ua is used in a +// call to Bucket.Update. +func (ua *BucketAttrsToUpdate) DeleteLabel(name string) { + if ua.deleteLabels == nil { + ua.deleteLabels = map[string]bool{} + } + ua.deleteLabels[name] = true +} + +func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket { + rb := &raw.Bucket{} + if ua.VersioningEnabled != nil { + rb.Versioning = &raw.BucketVersioning{ + Enabled: optional.ToBool(ua.VersioningEnabled), + ForceSendFields: []string{"Enabled"}, + } + } + if ua.RequesterPays != nil { + rb.Billing = &raw.BucketBilling{ + RequesterPays: optional.ToBool(ua.RequesterPays), + ForceSendFields: []string{"RequesterPays"}, + } + } + if ua.setLabels != nil || ua.deleteLabels != nil { + rb.Labels = map[string]string{} + for k, v := range ua.setLabels { + rb.Labels[k] = v + } + if len(rb.Labels) == 0 && len(ua.deleteLabels) > 0 { + rb.ForceSendFields = append(rb.ForceSendFields, "Labels") + } + for l := range ua.deleteLabels { + rb.NullFields = append(rb.NullFields, "Labels."+l) + } + } + return rb +} + +// If returns a new BucketHandle that applies a set of preconditions. +// Preconditions already set on the BucketHandle are ignored. +// Operations on the new handle will only occur if the preconditions are +// satisfied. The only valid preconditions for buckets are MetagenerationMatch +// and MetagenerationNotMatch. +func (b *BucketHandle) If(conds BucketConditions) *BucketHandle { + b2 := *b + b2.conds = &conds + return &b2 +} + +// BucketConditions constrain bucket methods to act on specific metagenerations. +// +// The zero value is an empty set of constraints. +type BucketConditions struct { + // MetagenerationMatch specifies that the bucket must have the given + // metageneration for the operation to occur. + // If MetagenerationMatch is zero, it has no effect. + MetagenerationMatch int64 + + // MetagenerationNotMatch specifies that the bucket must not have the given + // metageneration for the operation to occur. + // If MetagenerationNotMatch is zero, it has no effect. + MetagenerationNotMatch int64 +} + +func (c *BucketConditions) validate(method string) error { + if *c == (BucketConditions{}) { + return fmt.Errorf("storage: %s: empty conditions", method) + } + if c.MetagenerationMatch != 0 && c.MetagenerationNotMatch != 0 { + return fmt.Errorf("storage: %s: multiple conditions specified for metageneration", method) + } + return nil +} + +// UserProject returns a new BucketHandle that passes the project ID as the user +// project for all subsequent calls. A user project is required for all operations +// on requester-pays buckets. +func (b *BucketHandle) UserProject(projectID string) *BucketHandle { + b2 := *b + b2.userProject = projectID + b2.acl.userProject = projectID + b2.defaultObjectACL.userProject = projectID + return &b2 +} + +// applyBucketConds modifies the provided call using the conditions in conds. +// call is something that quacks like a *raw.WhateverCall. +func applyBucketConds(method string, conds *BucketConditions, call interface{}) error { + if conds == nil { + return nil + } + if err := conds.validate(method); err != nil { + return err + } + cval := reflect.ValueOf(call) + switch { + case conds.MetagenerationMatch != 0: + if !setConditionField(cval, "IfMetagenerationMatch", conds.MetagenerationMatch) { + return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method) + } + case conds.MetagenerationNotMatch != 0: + if !setConditionField(cval, "IfMetagenerationNotMatch", conds.MetagenerationNotMatch) { + return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method) + } + } + return nil +} + +func toRawLifecycle(l Lifecycle) *raw.BucketLifecycle { + var rl raw.BucketLifecycle + if len(l.Rules) == 0 { + return nil + } + for _, r := range l.Rules { + rr := &raw.BucketLifecycleRule{ + Action: &raw.BucketLifecycleRuleAction{ + Type: r.Action.Type, + StorageClass: r.Action.StorageClass, + }, + Condition: &raw.BucketLifecycleRuleCondition{ + Age: r.Condition.AgeInDays, + MatchesStorageClass: r.Condition.MatchesStorageClasses, + NumNewerVersions: r.Condition.NumNewerVersions, + }, + } + + switch r.Condition.Liveness { + case LiveAndArchived: + rr.Condition.IsLive = nil + case Live: + rr.Condition.IsLive = googleapi.Bool(true) + case Archived: + rr.Condition.IsLive = googleapi.Bool(false) + } + + if !r.Condition.CreatedBefore.IsZero() { + rr.Condition.CreatedBefore = r.Condition.CreatedBefore.Format(rfc3339Date) + } + rl.Rule = append(rl.Rule, rr) + } + return &rl +} + +func toLifecycle(rl *raw.BucketLifecycle) Lifecycle { + var l Lifecycle + if rl == nil { + return l } + for _, rr := range rl.Rule { + r := LifecycleRule{ + Action: LifecycleAction{ + Type: rr.Action.Type, + StorageClass: rr.Action.StorageClass, + }, + Condition: LifecycleCondition{ + AgeInDays: rr.Condition.Age, + MatchesStorageClasses: rr.Condition.MatchesStorageClass, + NumNewerVersions: rr.Condition.NumNewerVersions, + }, + } + + switch { + case rr.Condition.IsLive == nil: + r.Condition.Liveness = LiveAndArchived + case *rr.Condition.IsLive == true: + r.Condition.Liveness = Live + case *rr.Condition.IsLive == false: + r.Condition.Liveness = Archived + } + + if rr.Condition.CreatedBefore != "" { + r.Condition.CreatedBefore, _ = time.Parse(rfc3339Date, rr.Condition.CreatedBefore) + } + } + return l } // Objects returns an iterator over the objects in the bucket that match the Query q. @@ -241,6 +662,9 @@ func (it *ObjectIterator) fetch(pageSize int, pageToken string) (string, error) req.Prefix(it.query.Prefix) req.Versions(it.query.Versions) req.PageToken(pageToken) + if it.bucket.userProject != "" { + req.UserProject(it.bucket.userProject) + } if pageSize > 0 { req.MaxResults(int64(pageSize)) } diff --git a/vendor/cloud.google.com/go/storage/copy.go b/vendor/cloud.google.com/go/storage/copy.go index 16e28e3..d0a999c 100644 --- a/vendor/cloud.google.com/go/storage/copy.go +++ b/vendor/cloud.google.com/go/storage/copy.go @@ -25,6 +25,9 @@ import ( // CopierFrom creates a Copier that can copy src to dst. // You can immediately call Run on the returned Copier, or // you can configure it first. +// +// For Requester Pays buckets, the user project of dst is billed, unless it is empty, +// in which case the user project of src is billed. func (dst *ObjectHandle) CopierFrom(src *ObjectHandle) *Copier { return &Copier{dst: dst, src: src} } @@ -42,7 +45,7 @@ type Copier struct { RewriteToken string // ProgressFunc can be used to monitor the progress of a multi-RPC copy - // operation. If ProgressFunc is not nil and CopyFrom requires multiple + // operation. If ProgressFunc is not nil and copying requires multiple // calls to the underlying service (see // https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite), then // ProgressFunc will be invoked after each call with the number of bytes of @@ -73,22 +76,21 @@ func (c *Copier) Run(ctx context.Context) (*ObjectAttrs, error) { // does not cause any problems. rawObject := c.ObjectAttrs.toRawObject("") for { - res, err := c.callRewrite(ctx, c.src, rawObject) + res, err := c.callRewrite(ctx, rawObject) if err != nil { return nil, err } if c.ProgressFunc != nil { - c.ProgressFunc(res.TotalBytesRewritten, res.ObjectSize) + c.ProgressFunc(uint64(res.TotalBytesRewritten), uint64(res.ObjectSize)) } if res.Done { // Finished successfully. return newObject(res.Resource), nil } } - return nil, nil } -func (c *Copier) callRewrite(ctx context.Context, src *ObjectHandle, rawObj *raw.Object) (*raw.RewriteResponse, error) { - call := c.dst.c.raw.Objects.Rewrite(src.bucket, src.object, c.dst.bucket, c.dst.object, rawObj) +func (c *Copier) callRewrite(ctx context.Context, rawObj *raw.Object) (*raw.RewriteResponse, error) { + call := c.dst.c.raw.Objects.Rewrite(c.src.bucket, c.src.object, c.dst.bucket, c.dst.object, rawObj) call.Context(ctx).Projection("full") if c.RewriteToken != "" { @@ -97,6 +99,11 @@ func (c *Copier) callRewrite(ctx context.Context, src *ObjectHandle, rawObj *raw if err := applyConds("Copy destination", c.dst.gen, c.dst.conds, call); err != nil { return nil, err } + if c.dst.userProject != "" { + call.UserProject(c.dst.userProject) + } else if c.src.userProject != "" { + call.UserProject(c.src.userProject) + } if err := applySourceConds(c.src.gen, c.src.conds, call); err != nil { return nil, err } @@ -129,6 +136,8 @@ func (dst *ObjectHandle) ComposerFrom(srcs ...*ObjectHandle) *Composer { } // A Composer composes source objects into a destination object. +// +// For Requester Pays buckets, the user project of dst is billed. type Composer struct { // ObjectAttrs are optional attributes to set on the destination object. // Any attributes must be initialized before any calls on the Composer. Nil @@ -175,6 +184,9 @@ func (c *Composer) Run(ctx context.Context) (*ObjectAttrs, error) { if err := applyConds("ComposeFrom destination", c.dst.gen, c.dst.conds, call); err != nil { return nil, err } + if c.dst.userProject != "" { + call.UserProject(c.dst.userProject) + } if err := setEncryptionHeaders(call.Header(), c.dst.encryptionKey, false); err != nil { return nil, err } diff --git a/vendor/cloud.google.com/go/storage/go110.go b/vendor/cloud.google.com/go/storage/go110.go new file mode 100644 index 0000000..b85e8c3 --- /dev/null +++ b/vendor/cloud.google.com/go/storage/go110.go @@ -0,0 +1,30 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build go1.10 + +package storage + +import "google.golang.org/api/googleapi" + +func shouldRetry(err error) bool { + switch e := err.(type) { + case *googleapi.Error: + // Retry on 429 and 5xx, according to + // https://cloud.google.com/storage/docs/exponential-backoff. + return e.Code == 429 || (e.Code >= 500 && e.Code < 600) + default: + return false + } +} diff --git a/vendor/cloud.google.com/go/storage/go17.go b/vendor/cloud.google.com/go/storage/go17.go new file mode 100644 index 0000000..982db4e --- /dev/null +++ b/vendor/cloud.google.com/go/storage/go17.go @@ -0,0 +1,26 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build go1.7 + +package storage + +import ( + "context" + "net/http" +) + +func withContext(r *http.Request, ctx context.Context) *http.Request { + return r.WithContext(ctx) +} diff --git a/vendor/cloud.google.com/go/storage/iam.go b/vendor/cloud.google.com/go/storage/iam.go index 0cd9501..6607d8c 100644 --- a/vendor/cloud.google.com/go/storage/iam.go +++ b/vendor/cloud.google.com/go/storage/iam.go @@ -15,8 +15,6 @@ package storage import ( - "errors" - "cloud.google.com/go/iam" "golang.org/x/net/context" raw "google.golang.org/api/storage/v1" @@ -58,8 +56,19 @@ func (c *iamClient) Set(ctx context.Context, resource string, p *iampb.Policy) e }) } -func (c *iamClient) Test(context.Context, string, []string) ([]string, error) { - return nil, errors.New("TestPermissions is unimplemented") +func (c *iamClient) Test(ctx context.Context, resource string, perms []string) ([]string, error) { + req := c.raw.Buckets.TestIamPermissions(resource, perms) + setClientHeader(req.Header()) + var res *raw.TestIamPermissionsResponse + var err error + err = runWithRetry(ctx, func() error { + res, err = req.Context(ctx).Do() + return err + }) + if err != nil { + return nil, err + } + return res.Permissions, nil } func iamToStoragePolicy(ip *iampb.Policy) *raw.Policy { diff --git a/vendor/cloud.google.com/go/storage/invoke.go b/vendor/cloud.google.com/go/storage/invoke.go index e8fc924..46423a8 100644 --- a/vendor/cloud.google.com/go/storage/invoke.go +++ b/vendor/cloud.google.com/go/storage/invoke.go @@ -18,7 +18,6 @@ import ( "cloud.google.com/go/internal" gax "github.com/googleapis/gax-go" "golang.org/x/net/context" - "google.golang.org/api/googleapi" ) // runWithRetry calls the function until it returns nil or a non-retryable error, or @@ -29,13 +28,7 @@ func runWithRetry(ctx context.Context, call func() error) error { if err == nil { return true, nil } - e, ok := err.(*googleapi.Error) - if !ok { - return true, err - } - // Retry on 429 and 5xx, according to - // https://cloud.google.com/storage/docs/exponential-backoff. - if e.Code == 429 || (e.Code >= 500 && e.Code < 600) { + if shouldRetry(err) { return false, nil } return true, err diff --git a/vendor/cloud.google.com/go/storage/not_go110.go b/vendor/cloud.google.com/go/storage/not_go110.go new file mode 100644 index 0000000..c354e74 --- /dev/null +++ b/vendor/cloud.google.com/go/storage/not_go110.go @@ -0,0 +1,40 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !go1.10 + +package storage + +import ( + "net/url" + "strings" + + "google.golang.org/api/googleapi" +) + +func shouldRetry(err error) bool { + switch e := err.(type) { + case *googleapi.Error: + // Retry on 429 and 5xx, according to + // https://cloud.google.com/storage/docs/exponential-backoff. + return e.Code == 429 || (e.Code >= 500 && e.Code < 600) + case *url.Error: + // Retry on REFUSED_STREAM. + // Unfortunately the error type is unexported, so we resort to string + // matching. + return strings.Contains(e.Error(), "REFUSED_STREAM") + default: + return false + } +} diff --git a/vendor/cloud.google.com/go/storage/not_go17.go b/vendor/cloud.google.com/go/storage/not_go17.go new file mode 100644 index 0000000..1f6f7ae --- /dev/null +++ b/vendor/cloud.google.com/go/storage/not_go17.go @@ -0,0 +1,26 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !go1.7 + +package storage + +import ( + "net/http" +) + +func withContext(r *http.Request, _ interface{}) *http.Request { + // In Go 1.6 and below, ignore the context. + return r +} diff --git a/vendor/cloud.google.com/go/storage/reader.go b/vendor/cloud.google.com/go/storage/reader.go index aa103c1..c96ca8a 100644 --- a/vendor/cloud.google.com/go/storage/reader.go +++ b/vendor/cloud.google.com/go/storage/reader.go @@ -28,6 +28,7 @@ type Reader struct { body io.ReadCloser remain, size int64 contentType string + cacheControl string checkCRC bool // should we check the CRC? wantCRC uint32 // the CRC32c value the server sent in the header gotCRC uint32 // running crc @@ -72,3 +73,8 @@ func (r *Reader) Remain() int64 { func (r *Reader) ContentType() string { return r.contentType } + +// CacheControl returns the cache control of the object. +func (r *Reader) CacheControl() string { + return r.cacheControl +} diff --git a/vendor/cloud.google.com/go/storage/storage.go b/vendor/cloud.google.com/go/storage/storage.go index 6b82fa3..8ffa845 100644 --- a/vendor/cloud.google.com/go/storage/storage.go +++ b/vendor/cloud.google.com/go/storage/storage.go @@ -36,7 +36,7 @@ import ( "unicode/utf8" "google.golang.org/api/option" - "google.golang.org/api/transport" + htransport "google.golang.org/api/transport/http" "cloud.google.com/go/internal/optional" "cloud.google.com/go/internal/version" @@ -89,7 +89,7 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error option.WithUserAgent(userAgent), } opts = append(o, opts...) - hc, ep, err := transport.NewHTTPClient(ctx, opts...) + hc, ep, err := htransport.NewClient(ctx, opts...) if err != nil { return nil, fmt.Errorf("dialing: %v", err) } @@ -110,43 +110,13 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error // // Close need not be called at program exit. func (c *Client) Close() error { + // Set fields to nil so that subsequent uses + // will panic. c.hc = nil + c.raw = nil return nil } -// BucketHandle provides operations on a Google Cloud Storage bucket. -// Use Client.Bucket to get a handle. -type BucketHandle struct { - acl ACLHandle - defaultObjectACL ACLHandle - - c *Client - name string -} - -// Bucket returns a BucketHandle, which provides operations on the named bucket. -// This call does not perform any network operations. -// -// The supplied name must contain only lowercase letters, numbers, dashes, -// underscores, and dots. The full specification for valid bucket names can be -// found at: -// https://cloud.google.com/storage/docs/bucket-naming -func (c *Client) Bucket(name string) *BucketHandle { - return &BucketHandle{ - c: c, - name: name, - acl: ACLHandle{ - c: c, - bucket: name, - }, - defaultObjectACL: ACLHandle{ - c: c, - bucket: name, - isDefault: true, - }, - } -} - // SignedURLOptions allows you to restrict the access to the signed URL. type SignedURLOptions struct { // GoogleAccessID represents the authorizer of the signed URL generation. @@ -200,7 +170,7 @@ type SignedURLOptions struct { // Optional. ContentType string - // Headers is a list of extention headers the client must provide + // Headers is a list of extension headers the client must provide // in order to use the generated signed URL. // Optional. Headers []string @@ -295,6 +265,7 @@ type ObjectHandle struct { gen int64 // a negative value indicates latest conds *Conditions encryptionKey []byte // AES-256 key + userProject string // for requester-pays buckets } // ACL provides access to the object's access control list. @@ -347,6 +318,9 @@ func (o *ObjectHandle) Attrs(ctx context.Context) (*ObjectAttrs, error) { if err := applyConds("Attrs", o.gen, o.conds, call); err != nil { return nil, err } + if o.userProject != "" { + call.UserProject(o.userProject) + } if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil { return nil, err } @@ -375,11 +349,17 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) ( var forceSendFields, nullFields []string if uattrs.ContentType != nil { attrs.ContentType = optional.ToString(uattrs.ContentType) - forceSendFields = append(forceSendFields, "ContentType") + // For ContentType, sending the empty string is a no-op. + // Instead we send a null. + if attrs.ContentType == "" { + nullFields = append(nullFields, "ContentType") + } else { + forceSendFields = append(forceSendFields, "ContentType") + } } if uattrs.ContentLanguage != nil { attrs.ContentLanguage = optional.ToString(uattrs.ContentLanguage) - // For ContentLanguage It's an error to send the empty string. + // For ContentLanguage it's an error to send the empty string. // Instead we send a null. if attrs.ContentLanguage == "" { nullFields = append(nullFields, "ContentLanguage") @@ -389,7 +369,7 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) ( } if uattrs.ContentEncoding != nil { attrs.ContentEncoding = optional.ToString(uattrs.ContentEncoding) - forceSendFields = append(forceSendFields, "ContentType") + forceSendFields = append(forceSendFields, "ContentEncoding") } if uattrs.ContentDisposition != nil { attrs.ContentDisposition = optional.ToString(uattrs.ContentDisposition) @@ -421,6 +401,9 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) ( if err := applyConds("Update", o.gen, o.conds, call); err != nil { return nil, err } + if o.userProject != "" { + call.UserProject(o.userProject) + } if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil { return nil, err } @@ -467,6 +450,10 @@ func (o *ObjectHandle) Delete(ctx context.Context) error { if err := applyConds("Delete", o.gen, o.conds, call); err != nil { return err } + if o.userProject != "" { + call.UserProject(o.userProject) + } + // Encryption doesn't apply to Delete. setClientHeader(call.Header()) err := runWithRetry(ctx, func() error { return call.Do() }) switch e := err.(type) { @@ -518,11 +505,15 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) if err != nil { return nil, err } + req = withContext(req, ctx) if length < 0 && offset > 0 { req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) } else if length > 0 { req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) } + if o.userProject != "" { + req.Header.Set("X-Goog-User-Project", o.userProject) + } if err := setEncryptionHeaders(req.Header, o.encryptionKey, false); err != nil { return nil, err } @@ -585,12 +576,13 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) crc, checkCRC = parseCRC32c(res) } return &Reader{ - body: body, - size: size, - remain: remain, - contentType: res.Header.Get("Content-Type"), - wantCRC: crc, - checkCRC: checkCRC, + body: body, + size: size, + remain: remain, + contentType: res.Header.Get("Content-Type"), + cacheControl: res.Header.Get("Cache-Control"), + wantCRC: crc, + checkCRC: checkCRC, }, nil } @@ -646,11 +638,10 @@ func (o *ObjectHandle) validate() error { return nil } -// parseKey converts the binary contents of a private key file -// to an *rsa.PrivateKey. It detects whether the private key is in a -// PEM container or not. If so, it extracts the the private key -// from PEM container before conversion. It only supports PEM -// containers with no passphrase. +// parseKey converts the binary contents of a private key file to an +// *rsa.PrivateKey. It detects whether the private key is in a PEM container or +// not. If so, it extracts the private key from PEM container before +// conversion. It only supports PEM containers with no passphrase. func parseKey(key []byte) (*rsa.PrivateKey, error) { if block, _ := pem.Decode(key); block != nil { key = block.Bytes @@ -738,11 +729,16 @@ type ObjectAttrs struct { // sent in the response headers. ContentDisposition string - // MD5 is the MD5 hash of the object's content. This field is read-only. + // MD5 is the MD5 hash of the object's content. This field is read-only, + // except when used from a Writer. If set on a Writer, the uploaded + // data is rejected if its MD5 hash does not match this field. MD5 []byte // CRC32C is the CRC32 checksum of the object's content using - // the Castagnoli93 polynomial. This field is read-only. + // the Castagnoli93 polynomial. This field is read-only, except when + // used from a Writer. If set on a Writer and Writer.SendCRC32C + // is true, the uploaded data is rejected if its CRC32c hash does not + // match this field. CRC32C uint32 // MediaLink is an URL to the object's content. This field is read-only. @@ -830,26 +826,27 @@ func newObject(o *raw.Object) *ObjectAttrs { sha256 = o.CustomerEncryption.KeySha256 } return &ObjectAttrs{ - Bucket: o.Bucket, - Name: o.Name, - ContentType: o.ContentType, - ContentLanguage: o.ContentLanguage, - CacheControl: o.CacheControl, - ACL: acl, - Owner: owner, - ContentEncoding: o.ContentEncoding, - Size: int64(o.Size), - MD5: md5, - CRC32C: crc32c, - MediaLink: o.MediaLink, - Metadata: o.Metadata, - Generation: o.Generation, - Metageneration: o.Metageneration, - StorageClass: o.StorageClass, - CustomerKeySHA256: sha256, - Created: convertTime(o.TimeCreated), - Deleted: convertTime(o.TimeDeleted), - Updated: convertTime(o.Updated), + Bucket: o.Bucket, + Name: o.Name, + ContentType: o.ContentType, + ContentLanguage: o.ContentLanguage, + CacheControl: o.CacheControl, + ACL: acl, + Owner: owner, + ContentEncoding: o.ContentEncoding, + ContentDisposition: o.ContentDisposition, + Size: int64(o.Size), + MD5: md5, + CRC32C: crc32c, + MediaLink: o.MediaLink, + Metadata: o.Metadata, + Generation: o.Generation, + Metageneration: o.Metageneration, + StorageClass: o.StorageClass, + CustomerKeySHA256: sha256, + Created: convertTime(o.TimeCreated), + Deleted: convertTime(o.TimeDeleted), + Updated: convertTime(o.Updated), } } @@ -904,7 +901,7 @@ func (c *contentTyper) ContentType() string { } // Conditions constrain methods to act on specific generations of -// resources. +// objects. // // The zero value is an empty set of constraints. Not all conditions or // combinations of conditions are applicable to all methods. diff --git a/vendor/cloud.google.com/go/storage/writer.go b/vendor/cloud.google.com/go/storage/writer.go index 21ffdb9..28eb74a 100644 --- a/vendor/cloud.google.com/go/storage/writer.go +++ b/vendor/cloud.google.com/go/storage/writer.go @@ -36,6 +36,8 @@ type Writer struct { // SendCRC specifies whether to transmit a CRC32C field. It should be set // to true in addition to setting the Writer's CRC32C field, because zero // is a valid CRC and normally a zero would not be transmitted. + // If a CRC32C is sent, and the data written does not match the checksum, + // the write will be rejected. SendCRC32C bool // ChunkSize controls the maximum number of bytes of the object that the @@ -49,6 +51,16 @@ type Writer struct { // must be done before the first Write call. ChunkSize int + // ProgressFunc can be used to monitor the progress of a large write. + // operation. If ProgressFunc is not nil and writing requires multiple + // calls to the underlying service (see + // https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload), + // then ProgressFunc will be invoked after each call with the number of bytes of + // content copied so far. + // + // ProgressFunc should return quickly without blocking. + ProgressFunc func(int64) + ctx context.Context o *ObjectHandle @@ -75,7 +87,7 @@ func (w *Writer) open() error { w.opened = true if w.ChunkSize < 0 { - return errors.New("storage: Writer.ChunkSize must non-negative") + return errors.New("storage: Writer.ChunkSize must be non-negative") } mediaOpts := []googleapi.MediaOption{ googleapi.ChunkSize(w.ChunkSize), @@ -98,6 +110,9 @@ func (w *Writer) open() error { Media(pr, mediaOpts...). Projection("full"). Context(w.ctx) + if w.ProgressFunc != nil { + call.ProgressUpdater(func(n, _ int64) { w.ProgressFunc(n) }) + } if err := setEncryptionHeaders(call.Header(), w.o.encryptionKey, false); err != nil { w.err = err pr.CloseWithError(w.err) @@ -106,8 +121,25 @@ func (w *Writer) open() error { var resp *raw.Object err := applyConds("NewWriter", w.o.gen, w.o.conds, call) if err == nil { + if w.o.userProject != "" { + call.UserProject(w.o.userProject) + } setClientHeader(call.Header()) - resp, err = call.Do() + // If the chunk size is zero, then no chunking is done on the Reader, + // which means we cannot retry: the first call will read the data, and if + // it fails, there is no way to re-read. + if w.ChunkSize == 0 { + resp, err = call.Do() + } else { + // We will only retry here if the initial POST, which obtains a URI for + // the resumable upload, fails with a retryable error. The upload itself + // has its own retry logic. + err = runWithRetry(w.ctx, func() error { + var err2 error + resp, err2 = call.Do() + return err2 + }) + } } if err != nil { w.err = err @@ -120,6 +152,11 @@ func (w *Writer) open() error { } // Write appends to w. It implements the io.Writer interface. +// +// Since writes happen asynchronously, Write may return a nil +// error even though the write failed (or will fail). Always +// use the error returned from Writer.Close to determine if +// the upload was successful. func (w *Writer) Write(p []byte) (n int, err error) { if w.err != nil { return 0, w.err -- cgit v1.2.3