aboutsummaryrefslogtreecommitdiff
path: root/vendor/cloud.google.com/go/storage
diff options
context:
space:
mode:
authorNiall Sheridan <nsheridan@gmail.com>2017-10-18 13:15:14 +0100
committerNiall Sheridan <niall@intercom.io>2017-10-18 13:25:46 +0100
commit7b320119ba532fd409ec7dade7ad02011c309599 (patch)
treea39860f35b55e6cc499f8f5bfa969138c5dd6b73 /vendor/cloud.google.com/go/storage
parent7c99874c7a3e7a89716f3ee0cdf696532e35ae35 (diff)
Update dependencies
Diffstat (limited to 'vendor/cloud.google.com/go/storage')
-rw-r--r--vendor/cloud.google.com/go/storage/acl.go95
-rw-r--r--vendor/cloud.google.com/go/storage/bucket.go444
-rw-r--r--vendor/cloud.google.com/go/storage/copy.go24
-rw-r--r--vendor/cloud.google.com/go/storage/go110.go30
-rw-r--r--vendor/cloud.google.com/go/storage/go17.go26
-rw-r--r--vendor/cloud.google.com/go/storage/iam.go17
-rw-r--r--vendor/cloud.google.com/go/storage/invoke.go9
-rw-r--r--vendor/cloud.google.com/go/storage/not_go110.go40
-rw-r--r--vendor/cloud.google.com/go/storage/not_go17.go26
-rw-r--r--vendor/cloud.google.com/go/storage/reader.go6
-rw-r--r--vendor/cloud.google.com/go/storage/storage.go143
-rw-r--r--vendor/cloud.google.com/go/storage/writer.go41
12 files changed, 756 insertions, 145 deletions
diff --git a/vendor/cloud.google.com/go/storage/acl.go b/vendor/cloud.google.com/go/storage/acl.go
index 041e541..a1b2b6d 100644
--- a/vendor/cloud.google.com/go/storage/acl.go
+++ b/vendor/cloud.google.com/go/storage/acl.go
@@ -16,8 +16,11 @@ package storage
import (
"fmt"
+ "net/http"
+ "reflect"
"golang.org/x/net/context"
+ "google.golang.org/api/googleapi"
raw "google.golang.org/api/storage/v1"
)
@@ -53,10 +56,11 @@ type ACLRule struct {
// ACLHandle provides operations on an access control list for a Google Cloud Storage bucket or object.
type ACLHandle struct {
- c *Client
- bucket string
- object string
- isDefault bool
+ c *Client
+ bucket string
+ object string
+ isDefault bool
+ userProject string // for requester-pays buckets
}
// Delete permanently deletes the ACL entry for the given entity.
@@ -73,10 +77,10 @@ func (a *ACLHandle) Delete(ctx context.Context, entity ACLEntity) error {
// Set sets the permission level for the given entity.
func (a *ACLHandle) Set(ctx context.Context, entity ACLEntity, role ACLRole) error {
if a.object != "" {
- return a.objectSet(ctx, entity, role)
+ return a.objectSet(ctx, entity, role, false)
}
if a.isDefault {
- return a.bucketDefaultSet(ctx, entity, role)
+ return a.objectSet(ctx, entity, role, true)
}
return a.bucketSet(ctx, entity, role)
}
@@ -96,8 +100,8 @@ func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) {
var acls *raw.ObjectAccessControls
var err error
err = runWithRetry(ctx, func() error {
- req := a.c.raw.DefaultObjectAccessControls.List(a.bucket).Context(ctx)
- setClientHeader(req.Header())
+ req := a.c.raw.DefaultObjectAccessControls.List(a.bucket)
+ a.configureCall(req, ctx)
acls, err = req.Do()
return err
})
@@ -107,28 +111,10 @@ func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) {
return toACLRules(acls.Items), nil
}
-func (a *ACLHandle) bucketDefaultSet(ctx context.Context, entity ACLEntity, role ACLRole) error {
- acl := &raw.ObjectAccessControl{
- Bucket: a.bucket,
- Entity: string(entity),
- Role: string(role),
- }
- err := runWithRetry(ctx, func() error {
- req := a.c.raw.DefaultObjectAccessControls.Update(a.bucket, string(entity), acl).Context(ctx)
- setClientHeader(req.Header())
- _, err := req.Do()
- return err
- })
- if err != nil {
- return fmt.Errorf("storage: error updating default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
- }
- return nil
-}
-
func (a *ACLHandle) bucketDefaultDelete(ctx context.Context, entity ACLEntity) error {
err := runWithRetry(ctx, func() error {
- req := a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity)).Context(ctx)
- setClientHeader(req.Header())
+ req := a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity))
+ a.configureCall(req, ctx)
return req.Do()
})
if err != nil {
@@ -141,8 +127,8 @@ func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) {
var acls *raw.BucketAccessControls
var err error
err = runWithRetry(ctx, func() error {
- req := a.c.raw.BucketAccessControls.List(a.bucket).Context(ctx)
- setClientHeader(req.Header())
+ req := a.c.raw.BucketAccessControls.List(a.bucket)
+ a.configureCall(req, ctx)
acls, err = req.Do()
return err
})
@@ -164,8 +150,8 @@ func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRol
Role: string(role),
}
err := runWithRetry(ctx, func() error {
- req := a.c.raw.BucketAccessControls.Update(a.bucket, string(entity), acl).Context(ctx)
- setClientHeader(req.Header())
+ req := a.c.raw.BucketAccessControls.Update(a.bucket, string(entity), acl)
+ a.configureCall(req, ctx)
_, err := req.Do()
return err
})
@@ -177,8 +163,8 @@ func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRol
func (a *ACLHandle) bucketDelete(ctx context.Context, entity ACLEntity) error {
err := runWithRetry(ctx, func() error {
- req := a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity)).Context(ctx)
- setClientHeader(req.Header())
+ req := a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity))
+ a.configureCall(req, ctx)
return req.Do()
})
if err != nil {
@@ -191,8 +177,8 @@ func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) {
var acls *raw.ObjectAccessControls
var err error
err = runWithRetry(ctx, func() error {
- req := a.c.raw.ObjectAccessControls.List(a.bucket, a.object).Context(ctx)
- setClientHeader(req.Header())
+ req := a.c.raw.ObjectAccessControls.List(a.bucket, a.object)
+ a.configureCall(req, ctx)
acls, err = req.Do()
return err
})
@@ -202,28 +188,42 @@ func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) {
return toACLRules(acls.Items), nil
}
-func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRole) error {
+func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRole, isBucketDefault bool) error {
+ type setRequest interface {
+ Do(opts ...googleapi.CallOption) (*raw.ObjectAccessControl, error)
+ Header() http.Header
+ }
+
acl := &raw.ObjectAccessControl{
Bucket: a.bucket,
Entity: string(entity),
Role: string(role),
}
+ var req setRequest
+ if isBucketDefault {
+ req = a.c.raw.DefaultObjectAccessControls.Update(a.bucket, string(entity), acl)
+ } else {
+ req = a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl)
+ }
+ a.configureCall(req, ctx)
err := runWithRetry(ctx, func() error {
- req := a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl).Context(ctx)
- setClientHeader(req.Header())
_, err := req.Do()
return err
})
if err != nil {
- return fmt.Errorf("storage: error updating object ACL entry for bucket %q, file %q, entity %q: %v", a.bucket, a.object, entity, err)
+ if isBucketDefault {
+ return fmt.Errorf("storage: error updating default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err)
+ } else {
+ return fmt.Errorf("storage: error updating object ACL entry for bucket %q, object %q, entity %q: %v", a.bucket, a.object, entity, err)
+ }
}
return nil
}
func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error {
err := runWithRetry(ctx, func() error {
- req := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity)).Context(ctx)
- setClientHeader(req.Header())
+ req := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity))
+ a.configureCall(req, ctx)
return req.Do()
})
if err != nil {
@@ -232,6 +232,17 @@ func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error {
return nil
}
+func (a *ACLHandle) configureCall(call interface {
+ Header() http.Header
+}, ctx context.Context) {
+ vc := reflect.ValueOf(call)
+ vc.MethodByName("Context").Call([]reflect.Value{reflect.ValueOf(ctx)})
+ if a.userProject != "" {
+ vc.MethodByName("UserProject").Call([]reflect.Value{reflect.ValueOf(a.userProject)})
+ }
+ setClientHeader(call.Header())
+}
+
func toACLRules(items []*raw.ObjectAccessControl) []ACLRule {
r := make([]ACLRule, 0, len(items))
for _, item := range items {
diff --git a/vendor/cloud.google.com/go/storage/bucket.go b/vendor/cloud.google.com/go/storage/bucket.go
index deed5b2..07852a5 100644
--- a/vendor/cloud.google.com/go/storage/bucket.go
+++ b/vendor/cloud.google.com/go/storage/bucket.go
@@ -1,4 +1,4 @@
-// Copyright 2014 Google Inc. All Rights Reserved.
+// Copyright 2014 Google Inc. LiveAndArchived Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -15,15 +15,52 @@
package storage
import (
+ "fmt"
"net/http"
+ "reflect"
"time"
+ "cloud.google.com/go/internal/optional"
"golang.org/x/net/context"
"google.golang.org/api/googleapi"
"google.golang.org/api/iterator"
raw "google.golang.org/api/storage/v1"
)
+// BucketHandle provides operations on a Google Cloud Storage bucket.
+// Use Client.Bucket to get a handle.
+type BucketHandle struct {
+ c *Client
+ name string
+ acl ACLHandle
+ defaultObjectACL ACLHandle
+ conds *BucketConditions
+ userProject string // project for requester-pays buckets
+}
+
+// Bucket returns a BucketHandle, which provides operations on the named bucket.
+// This call does not perform any network operations.
+//
+// The supplied name must contain only lowercase letters, numbers, dashes,
+// underscores, and dots. The full specification for valid bucket names can be
+// found at:
+// https://cloud.google.com/storage/docs/bucket-naming
+func (c *Client) Bucket(name string) *BucketHandle {
+ return &BucketHandle{
+ c: c,
+ name: name,
+ acl: ACLHandle{
+ c: c,
+ bucket: name,
+ },
+ defaultObjectACL: ACLHandle{
+ c: c,
+ bucket: name,
+ isDefault: true,
+ },
+ }
+}
+
// Create creates the Bucket in the project.
// If attrs is nil the API defaults will be used.
func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *BucketAttrs) error {
@@ -34,6 +71,11 @@ func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *Buck
bkt = &raw.Bucket{}
}
bkt.Name = b.name
+ // If there is lifecycle information but no location, explicitly set
+ // the location. This is a GCS quirk/bug.
+ if bkt.Location == "" && bkt.Lifecycle != nil {
+ bkt.Location = "US"
+ }
req := b.c.raw.Buckets.Insert(projectID, bkt)
setClientHeader(req.Header())
return runWithRetry(ctx, func() error { _, err := req.Context(ctx).Do(); return err })
@@ -41,9 +83,23 @@ func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *Buck
// Delete deletes the Bucket.
func (b *BucketHandle) Delete(ctx context.Context) error {
+ req, err := b.newDeleteCall()
+ if err != nil {
+ return err
+ }
+ return runWithRetry(ctx, func() error { return req.Context(ctx).Do() })
+}
+
+func (b *BucketHandle) newDeleteCall() (*raw.BucketsDeleteCall, error) {
req := b.c.raw.Buckets.Delete(b.name)
setClientHeader(req.Header())
- return runWithRetry(ctx, func() error { return req.Context(ctx).Do() })
+ if err := applyBucketConds("BucketHandle.Delete", b.conds, req); err != nil {
+ return nil, err
+ }
+ if b.userProject != "" {
+ req.UserProject(b.userProject)
+ }
+ return req, nil
}
// ACL returns an ACLHandle, which provides access to the bucket's access control list.
@@ -72,20 +128,23 @@ func (b *BucketHandle) Object(name string) *ObjectHandle {
bucket: b.name,
object: name,
acl: ACLHandle{
- c: b.c,
- bucket: b.name,
- object: name,
+ c: b.c,
+ bucket: b.name,
+ object: name,
+ userProject: b.userProject,
},
- gen: -1,
+ gen: -1,
+ userProject: b.userProject,
}
}
// Attrs returns the metadata for the bucket.
func (b *BucketHandle) Attrs(ctx context.Context) (*BucketAttrs, error) {
- req := b.c.raw.Buckets.Get(b.name).Projection("full")
- setClientHeader(req.Header())
+ req, err := b.newGetCall()
+ if err != nil {
+ return nil, err
+ }
var resp *raw.Bucket
- var err error
err = runWithRetry(ctx, func() error {
resp, err = req.Context(ctx).Do()
return err
@@ -99,9 +158,49 @@ func (b *BucketHandle) Attrs(ctx context.Context) (*BucketAttrs, error) {
return newBucket(resp), nil
}
+func (b *BucketHandle) newGetCall() (*raw.BucketsGetCall, error) {
+ req := b.c.raw.Buckets.Get(b.name).Projection("full")
+ setClientHeader(req.Header())
+ if err := applyBucketConds("BucketHandle.Attrs", b.conds, req); err != nil {
+ return nil, err
+ }
+ if b.userProject != "" {
+ req.UserProject(b.userProject)
+ }
+ return req, nil
+}
+
+func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (*BucketAttrs, error) {
+ req, err := b.newPatchCall(&uattrs)
+ if err != nil {
+ return nil, err
+ }
+ // TODO(jba): retry iff metagen is set?
+ rb, err := req.Context(ctx).Do()
+ if err != nil {
+ return nil, err
+ }
+ return newBucket(rb), nil
+}
+
+func (b *BucketHandle) newPatchCall(uattrs *BucketAttrsToUpdate) (*raw.BucketsPatchCall, error) {
+ rb := uattrs.toRawBucket()
+ req := b.c.raw.Buckets.Patch(b.name, rb).Projection("full")
+ setClientHeader(req.Header())
+ if err := applyBucketConds("BucketHandle.Update", b.conds, req); err != nil {
+ return nil, err
+ }
+ if b.userProject != "" {
+ req.UserProject(b.userProject)
+ }
+ return req, nil
+}
+
// BucketAttrs represents the metadata for a Google Cloud Storage bucket.
+// Read-only fields are ignored by BucketHandle.Create.
type BucketAttrs struct {
// Name is the name of the bucket.
+ // This field is read-only.
Name string
// ACL is the list of access control rules on the bucket.
@@ -115,6 +214,7 @@ type BucketAttrs struct {
Location string
// MetaGeneration is the metadata generation of the bucket.
+ // This field is read-only.
MetaGeneration int64
// StorageClass is the default storage class of the bucket. This defines
@@ -127,11 +227,109 @@ type BucketAttrs struct {
StorageClass string
// Created is the creation time of the bucket.
+ // This field is read-only.
Created time.Time
// VersioningEnabled reports whether this bucket has versioning enabled.
- // This field is read-only.
VersioningEnabled bool
+
+ // Labels are the bucket's labels.
+ Labels map[string]string
+
+ // RequesterPays reports whether the bucket is a Requester Pays bucket.
+ RequesterPays bool
+ // Lifecycle is the lifecycle configuration for objects in the bucket.
+ Lifecycle Lifecycle
+}
+
+// Lifecycle is the lifecycle configuration for objects in the bucket.
+type Lifecycle struct {
+ Rules []LifecycleRule
+}
+
+const (
+ // RFC3339 date with only the date segment, used for CreatedBefore in LifecycleRule.
+ rfc3339Date = "2006-01-02"
+
+ // DeleteAction is a lifecycle action that deletes a live and/or archived
+ // objects. Takes precendence over SetStorageClass actions.
+ DeleteAction = "Delete"
+
+ // SetStorageClassAction changes the storage class of live and/or archived
+ // objects.
+ SetStorageClassAction = "SetStorageClass"
+)
+
+// LifecycleRule is a lifecycle configuration rule.
+//
+// When all the configured conditions are met by an object in the bucket, the
+// configured action will automatically be taken on that object.
+type LifecycleRule struct {
+ // Action is the action to take when all of the associated conditions are
+ // met.
+ Action LifecycleAction
+
+ // Condition is the set of conditions that must be met for the associated
+ // action to be taken.
+ Condition LifecycleCondition
+}
+
+// LifecycleAction is a lifecycle configuration action.
+type LifecycleAction struct {
+ // Type is the type of action to take on matching objects.
+ //
+ // Acceptable values are "Delete" to delete matching objects and
+ // "SetStorageClass" to set the storage class defined in StorageClass on
+ // matching objects.
+ Type string
+
+ // StorageClass is the storage class to set on matching objects if the Action
+ // is "SetStorageClass".
+ StorageClass string
+}
+
+// Liveness specifies whether the object is live or not.
+type Liveness int
+
+const (
+ // LiveAndArchived includes both live and archived objects.
+ LiveAndArchived Liveness = iota
+ // Live specifies that the object is still live.
+ Live
+ // Archived specifies that the object is archived.
+ Archived
+)
+
+// LifecycleCondition is a set of conditions used to match objects and take an
+// action automatically.
+//
+// All configured conditions must be met for the associated action to be taken.
+type LifecycleCondition struct {
+ // AgeInDays is the age of the object in days.
+ AgeInDays int64
+
+ // CreatedBefore is the time the object was created.
+ //
+ // This condition is satisfied when an object is created before midnight of
+ // the specified date in UTC.
+ CreatedBefore time.Time
+
+ // Liveness specifies the object's liveness. Relevant only for versioned objects
+ Liveness Liveness
+
+ // MatchesStorageClasses is the condition matching the object's storage
+ // class.
+ //
+ // Values include "MULTI_REGIONAL", "REGIONAL", "NEARLINE", "COLDLINE",
+ // "STANDARD", and "DURABLE_REDUCED_AVAILABILITY".
+ MatchesStorageClasses []string
+
+ // NumNewerVersions is the condition matching objects with a number of newer versions.
+ //
+ // If the value is N, this condition is satisfied when there are at least N
+ // versions (including the live version) newer than this version of the
+ // object.
+ NumNewerVersions int64
}
func newBucket(b *raw.Bucket) *BucketAttrs {
@@ -145,6 +343,9 @@ func newBucket(b *raw.Bucket) *BucketAttrs {
StorageClass: b.StorageClass,
Created: convertTime(b.TimeCreated),
VersioningEnabled: b.Versioning != nil && b.Versioning.Enabled,
+ Labels: b.Labels,
+ RequesterPays: b.Billing != nil && b.Billing.RequesterPays,
+ Lifecycle: toLifecycle(b.Lifecycle),
}
acl := make([]ACLRule, len(b.Acl))
for i, rule := range b.Acl {
@@ -178,13 +379,233 @@ func (b *BucketAttrs) toRawBucket() *raw.Bucket {
}
}
dACL := toRawObjectACL(b.DefaultObjectACL)
+ // Copy label map.
+ var labels map[string]string
+ if len(b.Labels) > 0 {
+ labels = make(map[string]string, len(b.Labels))
+ for k, v := range b.Labels {
+ labels[k] = v
+ }
+ }
+ // Ignore VersioningEnabled if it is false. This is OK because
+ // we only call this method when creating a bucket, and by default
+ // new buckets have versioning off.
+ var v *raw.BucketVersioning
+ if b.VersioningEnabled {
+ v = &raw.BucketVersioning{Enabled: true}
+ }
+ var bb *raw.BucketBilling
+ if b.RequesterPays {
+ bb = &raw.BucketBilling{RequesterPays: true}
+ }
return &raw.Bucket{
Name: b.Name,
DefaultObjectAcl: dACL,
Location: b.Location,
StorageClass: b.StorageClass,
Acl: acl,
+ Versioning: v,
+ Labels: labels,
+ Billing: bb,
+ Lifecycle: toRawLifecycle(b.Lifecycle),
+ }
+}
+
+type BucketAttrsToUpdate struct {
+ // VersioningEnabled, if set, updates whether the bucket uses versioning.
+ VersioningEnabled optional.Bool
+
+ // RequesterPays, if set, updates whether the bucket is a Requester Pays bucket.
+ RequesterPays optional.Bool
+
+ setLabels map[string]string
+ deleteLabels map[string]bool
+}
+
+// SetLabel causes a label to be added or modified when ua is used
+// in a call to Bucket.Update.
+func (ua *BucketAttrsToUpdate) SetLabel(name, value string) {
+ if ua.setLabels == nil {
+ ua.setLabels = map[string]string{}
+ }
+ ua.setLabels[name] = value
+}
+
+// DeleteLabel causes a label to be deleted when ua is used in a
+// call to Bucket.Update.
+func (ua *BucketAttrsToUpdate) DeleteLabel(name string) {
+ if ua.deleteLabels == nil {
+ ua.deleteLabels = map[string]bool{}
+ }
+ ua.deleteLabels[name] = true
+}
+
+func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket {
+ rb := &raw.Bucket{}
+ if ua.VersioningEnabled != nil {
+ rb.Versioning = &raw.BucketVersioning{
+ Enabled: optional.ToBool(ua.VersioningEnabled),
+ ForceSendFields: []string{"Enabled"},
+ }
+ }
+ if ua.RequesterPays != nil {
+ rb.Billing = &raw.BucketBilling{
+ RequesterPays: optional.ToBool(ua.RequesterPays),
+ ForceSendFields: []string{"RequesterPays"},
+ }
+ }
+ if ua.setLabels != nil || ua.deleteLabels != nil {
+ rb.Labels = map[string]string{}
+ for k, v := range ua.setLabels {
+ rb.Labels[k] = v
+ }
+ if len(rb.Labels) == 0 && len(ua.deleteLabels) > 0 {
+ rb.ForceSendFields = append(rb.ForceSendFields, "Labels")
+ }
+ for l := range ua.deleteLabels {
+ rb.NullFields = append(rb.NullFields, "Labels."+l)
+ }
+ }
+ return rb
+}
+
+// If returns a new BucketHandle that applies a set of preconditions.
+// Preconditions already set on the BucketHandle are ignored.
+// Operations on the new handle will only occur if the preconditions are
+// satisfied. The only valid preconditions for buckets are MetagenerationMatch
+// and MetagenerationNotMatch.
+func (b *BucketHandle) If(conds BucketConditions) *BucketHandle {
+ b2 := *b
+ b2.conds = &conds
+ return &b2
+}
+
+// BucketConditions constrain bucket methods to act on specific metagenerations.
+//
+// The zero value is an empty set of constraints.
+type BucketConditions struct {
+ // MetagenerationMatch specifies that the bucket must have the given
+ // metageneration for the operation to occur.
+ // If MetagenerationMatch is zero, it has no effect.
+ MetagenerationMatch int64
+
+ // MetagenerationNotMatch specifies that the bucket must not have the given
+ // metageneration for the operation to occur.
+ // If MetagenerationNotMatch is zero, it has no effect.
+ MetagenerationNotMatch int64
+}
+
+func (c *BucketConditions) validate(method string) error {
+ if *c == (BucketConditions{}) {
+ return fmt.Errorf("storage: %s: empty conditions", method)
+ }
+ if c.MetagenerationMatch != 0 && c.MetagenerationNotMatch != 0 {
+ return fmt.Errorf("storage: %s: multiple conditions specified for metageneration", method)
+ }
+ return nil
+}
+
+// UserProject returns a new BucketHandle that passes the project ID as the user
+// project for all subsequent calls. A user project is required for all operations
+// on requester-pays buckets.
+func (b *BucketHandle) UserProject(projectID string) *BucketHandle {
+ b2 := *b
+ b2.userProject = projectID
+ b2.acl.userProject = projectID
+ b2.defaultObjectACL.userProject = projectID
+ return &b2
+}
+
+// applyBucketConds modifies the provided call using the conditions in conds.
+// call is something that quacks like a *raw.WhateverCall.
+func applyBucketConds(method string, conds *BucketConditions, call interface{}) error {
+ if conds == nil {
+ return nil
+ }
+ if err := conds.validate(method); err != nil {
+ return err
+ }
+ cval := reflect.ValueOf(call)
+ switch {
+ case conds.MetagenerationMatch != 0:
+ if !setConditionField(cval, "IfMetagenerationMatch", conds.MetagenerationMatch) {
+ return fmt.Errorf("storage: %s: ifMetagenerationMatch not supported", method)
+ }
+ case conds.MetagenerationNotMatch != 0:
+ if !setConditionField(cval, "IfMetagenerationNotMatch", conds.MetagenerationNotMatch) {
+ return fmt.Errorf("storage: %s: ifMetagenerationNotMatch not supported", method)
+ }
+ }
+ return nil
+}
+
+func toRawLifecycle(l Lifecycle) *raw.BucketLifecycle {
+ var rl raw.BucketLifecycle
+ if len(l.Rules) == 0 {
+ return nil
+ }
+ for _, r := range l.Rules {
+ rr := &raw.BucketLifecycleRule{
+ Action: &raw.BucketLifecycleRuleAction{
+ Type: r.Action.Type,
+ StorageClass: r.Action.StorageClass,
+ },
+ Condition: &raw.BucketLifecycleRuleCondition{
+ Age: r.Condition.AgeInDays,
+ MatchesStorageClass: r.Condition.MatchesStorageClasses,
+ NumNewerVersions: r.Condition.NumNewerVersions,
+ },
+ }
+
+ switch r.Condition.Liveness {
+ case LiveAndArchived:
+ rr.Condition.IsLive = nil
+ case Live:
+ rr.Condition.IsLive = googleapi.Bool(true)
+ case Archived:
+ rr.Condition.IsLive = googleapi.Bool(false)
+ }
+
+ if !r.Condition.CreatedBefore.IsZero() {
+ rr.Condition.CreatedBefore = r.Condition.CreatedBefore.Format(rfc3339Date)
+ }
+ rl.Rule = append(rl.Rule, rr)
+ }
+ return &rl
+}
+
+func toLifecycle(rl *raw.BucketLifecycle) Lifecycle {
+ var l Lifecycle
+ if rl == nil {
+ return l
}
+ for _, rr := range rl.Rule {
+ r := LifecycleRule{
+ Action: LifecycleAction{
+ Type: rr.Action.Type,
+ StorageClass: rr.Action.StorageClass,
+ },
+ Condition: LifecycleCondition{
+ AgeInDays: rr.Condition.Age,
+ MatchesStorageClasses: rr.Condition.MatchesStorageClass,
+ NumNewerVersions: rr.Condition.NumNewerVersions,
+ },
+ }
+
+ switch {
+ case rr.Condition.IsLive == nil:
+ r.Condition.Liveness = LiveAndArchived
+ case *rr.Condition.IsLive == true:
+ r.Condition.Liveness = Live
+ case *rr.Condition.IsLive == false:
+ r.Condition.Liveness = Archived
+ }
+
+ if rr.Condition.CreatedBefore != "" {
+ r.Condition.CreatedBefore, _ = time.Parse(rfc3339Date, rr.Condition.CreatedBefore)
+ }
+ }
+ return l
}
// Objects returns an iterator over the objects in the bucket that match the Query q.
@@ -241,6 +662,9 @@ func (it *ObjectIterator) fetch(pageSize int, pageToken string) (string, error)
req.Prefix(it.query.Prefix)
req.Versions(it.query.Versions)
req.PageToken(pageToken)
+ if it.bucket.userProject != "" {
+ req.UserProject(it.bucket.userProject)
+ }
if pageSize > 0 {
req.MaxResults(int64(pageSize))
}
diff --git a/vendor/cloud.google.com/go/storage/copy.go b/vendor/cloud.google.com/go/storage/copy.go
index 16e28e3..d0a999c 100644
--- a/vendor/cloud.google.com/go/storage/copy.go
+++ b/vendor/cloud.google.com/go/storage/copy.go
@@ -25,6 +25,9 @@ import (
// CopierFrom creates a Copier that can copy src to dst.
// You can immediately call Run on the returned Copier, or
// you can configure it first.
+//
+// For Requester Pays buckets, the user project of dst is billed, unless it is empty,
+// in which case the user project of src is billed.
func (dst *ObjectHandle) CopierFrom(src *ObjectHandle) *Copier {
return &Copier{dst: dst, src: src}
}
@@ -42,7 +45,7 @@ type Copier struct {
RewriteToken string
// ProgressFunc can be used to monitor the progress of a multi-RPC copy
- // operation. If ProgressFunc is not nil and CopyFrom requires multiple
+ // operation. If ProgressFunc is not nil and copying requires multiple
// calls to the underlying service (see
// https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite), then
// ProgressFunc will be invoked after each call with the number of bytes of
@@ -73,22 +76,21 @@ func (c *Copier) Run(ctx context.Context) (*ObjectAttrs, error) {
// does not cause any problems.
rawObject := c.ObjectAttrs.toRawObject("")
for {
- res, err := c.callRewrite(ctx, c.src, rawObject)
+ res, err := c.callRewrite(ctx, rawObject)
if err != nil {
return nil, err
}
if c.ProgressFunc != nil {
- c.ProgressFunc(res.TotalBytesRewritten, res.ObjectSize)
+ c.ProgressFunc(uint64(res.TotalBytesRewritten), uint64(res.ObjectSize))
}
if res.Done { // Finished successfully.
return newObject(res.Resource), nil
}
}
- return nil, nil
}
-func (c *Copier) callRewrite(ctx context.Context, src *ObjectHandle, rawObj *raw.Object) (*raw.RewriteResponse, error) {
- call := c.dst.c.raw.Objects.Rewrite(src.bucket, src.object, c.dst.bucket, c.dst.object, rawObj)
+func (c *Copier) callRewrite(ctx context.Context, rawObj *raw.Object) (*raw.RewriteResponse, error) {
+ call := c.dst.c.raw.Objects.Rewrite(c.src.bucket, c.src.object, c.dst.bucket, c.dst.object, rawObj)
call.Context(ctx).Projection("full")
if c.RewriteToken != "" {
@@ -97,6 +99,11 @@ func (c *Copier) callRewrite(ctx context.Context, src *ObjectHandle, rawObj *raw
if err := applyConds("Copy destination", c.dst.gen, c.dst.conds, call); err != nil {
return nil, err
}
+ if c.dst.userProject != "" {
+ call.UserProject(c.dst.userProject)
+ } else if c.src.userProject != "" {
+ call.UserProject(c.src.userProject)
+ }
if err := applySourceConds(c.src.gen, c.src.conds, call); err != nil {
return nil, err
}
@@ -129,6 +136,8 @@ func (dst *ObjectHandle) ComposerFrom(srcs ...*ObjectHandle) *Composer {
}
// A Composer composes source objects into a destination object.
+//
+// For Requester Pays buckets, the user project of dst is billed.
type Composer struct {
// ObjectAttrs are optional attributes to set on the destination object.
// Any attributes must be initialized before any calls on the Composer. Nil
@@ -175,6 +184,9 @@ func (c *Composer) Run(ctx context.Context) (*ObjectAttrs, error) {
if err := applyConds("ComposeFrom destination", c.dst.gen, c.dst.conds, call); err != nil {
return nil, err
}
+ if c.dst.userProject != "" {
+ call.UserProject(c.dst.userProject)
+ }
if err := setEncryptionHeaders(call.Header(), c.dst.encryptionKey, false); err != nil {
return nil, err
}
diff --git a/vendor/cloud.google.com/go/storage/go110.go b/vendor/cloud.google.com/go/storage/go110.go
new file mode 100644
index 0000000..b85e8c3
--- /dev/null
+++ b/vendor/cloud.google.com/go/storage/go110.go
@@ -0,0 +1,30 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.10
+
+package storage
+
+import "google.golang.org/api/googleapi"
+
+func shouldRetry(err error) bool {
+ switch e := err.(type) {
+ case *googleapi.Error:
+ // Retry on 429 and 5xx, according to
+ // https://cloud.google.com/storage/docs/exponential-backoff.
+ return e.Code == 429 || (e.Code >= 500 && e.Code < 600)
+ default:
+ return false
+ }
+}
diff --git a/vendor/cloud.google.com/go/storage/go17.go b/vendor/cloud.google.com/go/storage/go17.go
new file mode 100644
index 0000000..982db4e
--- /dev/null
+++ b/vendor/cloud.google.com/go/storage/go17.go
@@ -0,0 +1,26 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.7
+
+package storage
+
+import (
+ "context"
+ "net/http"
+)
+
+func withContext(r *http.Request, ctx context.Context) *http.Request {
+ return r.WithContext(ctx)
+}
diff --git a/vendor/cloud.google.com/go/storage/iam.go b/vendor/cloud.google.com/go/storage/iam.go
index 0cd9501..6607d8c 100644
--- a/vendor/cloud.google.com/go/storage/iam.go
+++ b/vendor/cloud.google.com/go/storage/iam.go
@@ -15,8 +15,6 @@
package storage
import (
- "errors"
-
"cloud.google.com/go/iam"
"golang.org/x/net/context"
raw "google.golang.org/api/storage/v1"
@@ -58,8 +56,19 @@ func (c *iamClient) Set(ctx context.Context, resource string, p *iampb.Policy) e
})
}
-func (c *iamClient) Test(context.Context, string, []string) ([]string, error) {
- return nil, errors.New("TestPermissions is unimplemented")
+func (c *iamClient) Test(ctx context.Context, resource string, perms []string) ([]string, error) {
+ req := c.raw.Buckets.TestIamPermissions(resource, perms)
+ setClientHeader(req.Header())
+ var res *raw.TestIamPermissionsResponse
+ var err error
+ err = runWithRetry(ctx, func() error {
+ res, err = req.Context(ctx).Do()
+ return err
+ })
+ if err != nil {
+ return nil, err
+ }
+ return res.Permissions, nil
}
func iamToStoragePolicy(ip *iampb.Policy) *raw.Policy {
diff --git a/vendor/cloud.google.com/go/storage/invoke.go b/vendor/cloud.google.com/go/storage/invoke.go
index e8fc924..46423a8 100644
--- a/vendor/cloud.google.com/go/storage/invoke.go
+++ b/vendor/cloud.google.com/go/storage/invoke.go
@@ -18,7 +18,6 @@ import (
"cloud.google.com/go/internal"
gax "github.com/googleapis/gax-go"
"golang.org/x/net/context"
- "google.golang.org/api/googleapi"
)
// runWithRetry calls the function until it returns nil or a non-retryable error, or
@@ -29,13 +28,7 @@ func runWithRetry(ctx context.Context, call func() error) error {
if err == nil {
return true, nil
}
- e, ok := err.(*googleapi.Error)
- if !ok {
- return true, err
- }
- // Retry on 429 and 5xx, according to
- // https://cloud.google.com/storage/docs/exponential-backoff.
- if e.Code == 429 || (e.Code >= 500 && e.Code < 600) {
+ if shouldRetry(err) {
return false, nil
}
return true, err
diff --git a/vendor/cloud.google.com/go/storage/not_go110.go b/vendor/cloud.google.com/go/storage/not_go110.go
new file mode 100644
index 0000000..c354e74
--- /dev/null
+++ b/vendor/cloud.google.com/go/storage/not_go110.go
@@ -0,0 +1,40 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !go1.10
+
+package storage
+
+import (
+ "net/url"
+ "strings"
+
+ "google.golang.org/api/googleapi"
+)
+
+func shouldRetry(err error) bool {
+ switch e := err.(type) {
+ case *googleapi.Error:
+ // Retry on 429 and 5xx, according to
+ // https://cloud.google.com/storage/docs/exponential-backoff.
+ return e.Code == 429 || (e.Code >= 500 && e.Code < 600)
+ case *url.Error:
+ // Retry on REFUSED_STREAM.
+ // Unfortunately the error type is unexported, so we resort to string
+ // matching.
+ return strings.Contains(e.Error(), "REFUSED_STREAM")
+ default:
+ return false
+ }
+}
diff --git a/vendor/cloud.google.com/go/storage/not_go17.go b/vendor/cloud.google.com/go/storage/not_go17.go
new file mode 100644
index 0000000..1f6f7ae
--- /dev/null
+++ b/vendor/cloud.google.com/go/storage/not_go17.go
@@ -0,0 +1,26 @@
+// Copyright 2017 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !go1.7
+
+package storage
+
+import (
+ "net/http"
+)
+
+func withContext(r *http.Request, _ interface{}) *http.Request {
+ // In Go 1.6 and below, ignore the context.
+ return r
+}
diff --git a/vendor/cloud.google.com/go/storage/reader.go b/vendor/cloud.google.com/go/storage/reader.go
index aa103c1..c96ca8a 100644
--- a/vendor/cloud.google.com/go/storage/reader.go
+++ b/vendor/cloud.google.com/go/storage/reader.go
@@ -28,6 +28,7 @@ type Reader struct {
body io.ReadCloser
remain, size int64
contentType string
+ cacheControl string
checkCRC bool // should we check the CRC?
wantCRC uint32 // the CRC32c value the server sent in the header
gotCRC uint32 // running crc
@@ -72,3 +73,8 @@ func (r *Reader) Remain() int64 {
func (r *Reader) ContentType() string {
return r.contentType
}
+
+// CacheControl returns the cache control of the object.
+func (r *Reader) CacheControl() string {
+ return r.cacheControl
+}
diff --git a/vendor/cloud.google.com/go/storage/storage.go b/vendor/cloud.google.com/go/storage/storage.go
index 6b82fa3..8ffa845 100644
--- a/vendor/cloud.google.com/go/storage/storage.go
+++ b/vendor/cloud.google.com/go/storage/storage.go
@@ -36,7 +36,7 @@ import (
"unicode/utf8"
"google.golang.org/api/option"
- "google.golang.org/api/transport"
+ htransport "google.golang.org/api/transport/http"
"cloud.google.com/go/internal/optional"
"cloud.google.com/go/internal/version"
@@ -89,7 +89,7 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error
option.WithUserAgent(userAgent),
}
opts = append(o, opts...)
- hc, ep, err := transport.NewHTTPClient(ctx, opts...)
+ hc, ep, err := htransport.NewClient(ctx, opts...)
if err != nil {
return nil, fmt.Errorf("dialing: %v", err)
}
@@ -110,43 +110,13 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error
//
// Close need not be called at program exit.
func (c *Client) Close() error {
+ // Set fields to nil so that subsequent uses
+ // will panic.
c.hc = nil
+ c.raw = nil
return nil
}
-// BucketHandle provides operations on a Google Cloud Storage bucket.
-// Use Client.Bucket to get a handle.
-type BucketHandle struct {
- acl ACLHandle
- defaultObjectACL ACLHandle
-
- c *Client
- name string
-}
-
-// Bucket returns a BucketHandle, which provides operations on the named bucket.
-// This call does not perform any network operations.
-//
-// The supplied name must contain only lowercase letters, numbers, dashes,
-// underscores, and dots. The full specification for valid bucket names can be
-// found at:
-// https://cloud.google.com/storage/docs/bucket-naming
-func (c *Client) Bucket(name string) *BucketHandle {
- return &BucketHandle{
- c: c,
- name: name,
- acl: ACLHandle{
- c: c,
- bucket: name,
- },
- defaultObjectACL: ACLHandle{
- c: c,
- bucket: name,
- isDefault: true,
- },
- }
-}
-
// SignedURLOptions allows you to restrict the access to the signed URL.
type SignedURLOptions struct {
// GoogleAccessID represents the authorizer of the signed URL generation.
@@ -200,7 +170,7 @@ type SignedURLOptions struct {
// Optional.
ContentType string
- // Headers is a list of extention headers the client must provide
+ // Headers is a list of extension headers the client must provide
// in order to use the generated signed URL.
// Optional.
Headers []string
@@ -295,6 +265,7 @@ type ObjectHandle struct {
gen int64 // a negative value indicates latest
conds *Conditions
encryptionKey []byte // AES-256 key
+ userProject string // for requester-pays buckets
}
// ACL provides access to the object's access control list.
@@ -347,6 +318,9 @@ func (o *ObjectHandle) Attrs(ctx context.Context) (*ObjectAttrs, error) {
if err := applyConds("Attrs", o.gen, o.conds, call); err != nil {
return nil, err
}
+ if o.userProject != "" {
+ call.UserProject(o.userProject)
+ }
if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil {
return nil, err
}
@@ -375,11 +349,17 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (
var forceSendFields, nullFields []string
if uattrs.ContentType != nil {
attrs.ContentType = optional.ToString(uattrs.ContentType)
- forceSendFields = append(forceSendFields, "ContentType")
+ // For ContentType, sending the empty string is a no-op.
+ // Instead we send a null.
+ if attrs.ContentType == "" {
+ nullFields = append(nullFields, "ContentType")
+ } else {
+ forceSendFields = append(forceSendFields, "ContentType")
+ }
}
if uattrs.ContentLanguage != nil {
attrs.ContentLanguage = optional.ToString(uattrs.ContentLanguage)
- // For ContentLanguage It's an error to send the empty string.
+ // For ContentLanguage it's an error to send the empty string.
// Instead we send a null.
if attrs.ContentLanguage == "" {
nullFields = append(nullFields, "ContentLanguage")
@@ -389,7 +369,7 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (
}
if uattrs.ContentEncoding != nil {
attrs.ContentEncoding = optional.ToString(uattrs.ContentEncoding)
- forceSendFields = append(forceSendFields, "ContentType")
+ forceSendFields = append(forceSendFields, "ContentEncoding")
}
if uattrs.ContentDisposition != nil {
attrs.ContentDisposition = optional.ToString(uattrs.ContentDisposition)
@@ -421,6 +401,9 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (
if err := applyConds("Update", o.gen, o.conds, call); err != nil {
return nil, err
}
+ if o.userProject != "" {
+ call.UserProject(o.userProject)
+ }
if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil {
return nil, err
}
@@ -467,6 +450,10 @@ func (o *ObjectHandle) Delete(ctx context.Context) error {
if err := applyConds("Delete", o.gen, o.conds, call); err != nil {
return err
}
+ if o.userProject != "" {
+ call.UserProject(o.userProject)
+ }
+ // Encryption doesn't apply to Delete.
setClientHeader(call.Header())
err := runWithRetry(ctx, func() error { return call.Do() })
switch e := err.(type) {
@@ -518,11 +505,15 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64)
if err != nil {
return nil, err
}
+ req = withContext(req, ctx)
if length < 0 && offset > 0 {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
} else if length > 0 {
req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1))
}
+ if o.userProject != "" {
+ req.Header.Set("X-Goog-User-Project", o.userProject)
+ }
if err := setEncryptionHeaders(req.Header, o.encryptionKey, false); err != nil {
return nil, err
}
@@ -585,12 +576,13 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64)
crc, checkCRC = parseCRC32c(res)
}
return &Reader{
- body: body,
- size: size,
- remain: remain,
- contentType: res.Header.Get("Content-Type"),
- wantCRC: crc,
- checkCRC: checkCRC,
+ body: body,
+ size: size,
+ remain: remain,
+ contentType: res.Header.Get("Content-Type"),
+ cacheControl: res.Header.Get("Cache-Control"),
+ wantCRC: crc,
+ checkCRC: checkCRC,
}, nil
}
@@ -646,11 +638,10 @@ func (o *ObjectHandle) validate() error {
return nil
}
-// parseKey converts the binary contents of a private key file
-// to an *rsa.PrivateKey. It detects whether the private key is in a
-// PEM container or not. If so, it extracts the the private key
-// from PEM container before conversion. It only supports PEM
-// containers with no passphrase.
+// parseKey converts the binary contents of a private key file to an
+// *rsa.PrivateKey. It detects whether the private key is in a PEM container or
+// not. If so, it extracts the private key from PEM container before
+// conversion. It only supports PEM containers with no passphrase.
func parseKey(key []byte) (*rsa.PrivateKey, error) {
if block, _ := pem.Decode(key); block != nil {
key = block.Bytes
@@ -738,11 +729,16 @@ type ObjectAttrs struct {
// sent in the response headers.
ContentDisposition string
- // MD5 is the MD5 hash of the object's content. This field is read-only.
+ // MD5 is the MD5 hash of the object's content. This field is read-only,
+ // except when used from a Writer. If set on a Writer, the uploaded
+ // data is rejected if its MD5 hash does not match this field.
MD5 []byte
// CRC32C is the CRC32 checksum of the object's content using
- // the Castagnoli93 polynomial. This field is read-only.
+ // the Castagnoli93 polynomial. This field is read-only, except when
+ // used from a Writer. If set on a Writer and Writer.SendCRC32C
+ // is true, the uploaded data is rejected if its CRC32c hash does not
+ // match this field.
CRC32C uint32
// MediaLink is an URL to the object's content. This field is read-only.
@@ -830,26 +826,27 @@ func newObject(o *raw.Object) *ObjectAttrs {
sha256 = o.CustomerEncryption.KeySha256
}
return &ObjectAttrs{
- Bucket: o.Bucket,
- Name: o.Name,
- ContentType: o.ContentType,
- ContentLanguage: o.ContentLanguage,
- CacheControl: o.CacheControl,
- ACL: acl,
- Owner: owner,
- ContentEncoding: o.ContentEncoding,
- Size: int64(o.Size),
- MD5: md5,
- CRC32C: crc32c,
- MediaLink: o.MediaLink,
- Metadata: o.Metadata,
- Generation: o.Generation,
- Metageneration: o.Metageneration,
- StorageClass: o.StorageClass,
- CustomerKeySHA256: sha256,
- Created: convertTime(o.TimeCreated),
- Deleted: convertTime(o.TimeDeleted),
- Updated: convertTime(o.Updated),
+ Bucket: o.Bucket,
+ Name: o.Name,
+ ContentType: o.ContentType,
+ ContentLanguage: o.ContentLanguage,
+ CacheControl: o.CacheControl,
+ ACL: acl,
+ Owner: owner,
+ ContentEncoding: o.ContentEncoding,
+ ContentDisposition: o.ContentDisposition,
+ Size: int64(o.Size),
+ MD5: md5,
+ CRC32C: crc32c,
+ MediaLink: o.MediaLink,
+ Metadata: o.Metadata,
+ Generation: o.Generation,
+ Metageneration: o.Metageneration,
+ StorageClass: o.StorageClass,
+ CustomerKeySHA256: sha256,
+ Created: convertTime(o.TimeCreated),
+ Deleted: convertTime(o.TimeDeleted),
+ Updated: convertTime(o.Updated),
}
}
@@ -904,7 +901,7 @@ func (c *contentTyper) ContentType() string {
}
// Conditions constrain methods to act on specific generations of
-// resources.
+// objects.
//
// The zero value is an empty set of constraints. Not all conditions or
// combinations of conditions are applicable to all methods.
diff --git a/vendor/cloud.google.com/go/storage/writer.go b/vendor/cloud.google.com/go/storage/writer.go
index 21ffdb9..28eb74a 100644
--- a/vendor/cloud.google.com/go/storage/writer.go
+++ b/vendor/cloud.google.com/go/storage/writer.go
@@ -36,6 +36,8 @@ type Writer struct {
// SendCRC specifies whether to transmit a CRC32C field. It should be set
// to true in addition to setting the Writer's CRC32C field, because zero
// is a valid CRC and normally a zero would not be transmitted.
+ // If a CRC32C is sent, and the data written does not match the checksum,
+ // the write will be rejected.
SendCRC32C bool
// ChunkSize controls the maximum number of bytes of the object that the
@@ -49,6 +51,16 @@ type Writer struct {
// must be done before the first Write call.
ChunkSize int
+ // ProgressFunc can be used to monitor the progress of a large write.
+ // operation. If ProgressFunc is not nil and writing requires multiple
+ // calls to the underlying service (see
+ // https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload),
+ // then ProgressFunc will be invoked after each call with the number of bytes of
+ // content copied so far.
+ //
+ // ProgressFunc should return quickly without blocking.
+ ProgressFunc func(int64)
+
ctx context.Context
o *ObjectHandle
@@ -75,7 +87,7 @@ func (w *Writer) open() error {
w.opened = true
if w.ChunkSize < 0 {
- return errors.New("storage: Writer.ChunkSize must non-negative")
+ return errors.New("storage: Writer.ChunkSize must be non-negative")
}
mediaOpts := []googleapi.MediaOption{
googleapi.ChunkSize(w.ChunkSize),
@@ -98,6 +110,9 @@ func (w *Writer) open() error {
Media(pr, mediaOpts...).
Projection("full").
Context(w.ctx)
+ if w.ProgressFunc != nil {
+ call.ProgressUpdater(func(n, _ int64) { w.ProgressFunc(n) })
+ }
if err := setEncryptionHeaders(call.Header(), w.o.encryptionKey, false); err != nil {
w.err = err
pr.CloseWithError(w.err)
@@ -106,8 +121,25 @@ func (w *Writer) open() error {
var resp *raw.Object
err := applyConds("NewWriter", w.o.gen, w.o.conds, call)
if err == nil {
+ if w.o.userProject != "" {
+ call.UserProject(w.o.userProject)
+ }
setClientHeader(call.Header())
- resp, err = call.Do()
+ // If the chunk size is zero, then no chunking is done on the Reader,
+ // which means we cannot retry: the first call will read the data, and if
+ // it fails, there is no way to re-read.
+ if w.ChunkSize == 0 {
+ resp, err = call.Do()
+ } else {
+ // We will only retry here if the initial POST, which obtains a URI for
+ // the resumable upload, fails with a retryable error. The upload itself
+ // has its own retry logic.
+ err = runWithRetry(w.ctx, func() error {
+ var err2 error
+ resp, err2 = call.Do()
+ return err2
+ })
+ }
}
if err != nil {
w.err = err
@@ -120,6 +152,11 @@ func (w *Writer) open() error {
}
// Write appends to w. It implements the io.Writer interface.
+//
+// Since writes happen asynchronously, Write may return a nil
+// error even though the write failed (or will fail). Always
+// use the error returned from Writer.Close to determine if
+// the upload was successful.
func (w *Writer) Write(p []byte) (n int, err error) {
if w.err != nil {
return 0, w.err