From 30802e07b2d84fbc213b490d3402707dffe60096 Mon Sep 17 00:00:00 2001 From: Niall Sheridan Date: Mon, 10 Apr 2017 21:18:42 +0100 Subject: update dependencies --- .../go/compute/metadata/metadata.go | 35 ++- vendor/cloud.google.com/go/iam/iam.go | 256 +++++++++++++++++++++ vendor/cloud.google.com/go/internal/cloud.go | 64 ------ vendor/cloud.google.com/go/internal/retry.go | 3 +- .../go/internal/version/update_version.sh | 6 + .../go/internal/version/version.go | 71 ++++++ vendor/cloud.google.com/go/storage/acl.go | 36 ++- vendor/cloud.google.com/go/storage/bucket.go | 10 +- vendor/cloud.google.com/go/storage/copy.go | 22 +- vendor/cloud.google.com/go/storage/doc.go | 5 + vendor/cloud.google.com/go/storage/iam.go | 99 ++++++++ vendor/cloud.google.com/go/storage/reader.go | 17 ++ vendor/cloud.google.com/go/storage/storage.go | 84 +++++-- vendor/cloud.google.com/go/storage/writer.go | 18 +- 14 files changed, 599 insertions(+), 127 deletions(-) create mode 100644 vendor/cloud.google.com/go/iam/iam.go delete mode 100644 vendor/cloud.google.com/go/internal/cloud.go create mode 100755 vendor/cloud.google.com/go/internal/version/update_version.sh create mode 100644 vendor/cloud.google.com/go/internal/version/version.go create mode 100644 vendor/cloud.google.com/go/storage/iam.go (limited to 'vendor/cloud.google.com') diff --git a/vendor/cloud.google.com/go/compute/metadata/metadata.go b/vendor/cloud.google.com/go/compute/metadata/metadata.go index f9d2bef..e708c03 100644 --- a/vendor/cloud.google.com/go/compute/metadata/metadata.go +++ b/vendor/cloud.google.com/go/compute/metadata/metadata.go @@ -34,8 +34,6 @@ import ( "golang.org/x/net/context" "golang.org/x/net/context/ctxhttp" - - "cloud.google.com/go/internal" ) const ( @@ -48,6 +46,8 @@ const ( // This is variable name is not defined by any spec, as far as // I know; it was made up for the Go package. metadataHostEnv = "GCE_METADATA_HOST" + + userAgent = "gcloud-golang/0.1" ) type cachedValue struct { @@ -65,24 +65,20 @@ var ( var ( metaClient = &http.Client{ - Transport: &internal.Transport{ - Base: &http.Transport{ - Dial: (&net.Dialer{ - Timeout: 2 * time.Second, - KeepAlive: 30 * time.Second, - }).Dial, - ResponseHeaderTimeout: 2 * time.Second, - }, + Transport: &http.Transport{ + Dial: (&net.Dialer{ + Timeout: 2 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial, + ResponseHeaderTimeout: 2 * time.Second, }, } subscribeClient = &http.Client{ - Transport: &internal.Transport{ - Base: &http.Transport{ - Dial: (&net.Dialer{ - Timeout: 2 * time.Second, - KeepAlive: 30 * time.Second, - }).Dial, - }, + Transport: &http.Transport{ + Dial: (&net.Dialer{ + Timeout: 2 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial, }, } ) @@ -132,6 +128,7 @@ func getETag(client *http.Client, suffix string) (value, etag string, err error) url := "http://" + host + "/computeMetadata/v1/" + suffix req, _ := http.NewRequest("GET", url, nil) req.Header.Set("Metadata-Flavor", "Google") + req.Header.Set("User-Agent", userAgent) res, err := client.Do(req) if err != nil { return "", "", err @@ -202,7 +199,9 @@ func testOnGCE() bool { // Try two strategies in parallel. // See https://github.com/GoogleCloudPlatform/google-cloud-go/issues/194 go func() { - res, err := ctxhttp.Get(ctx, metaClient, "http://"+metadataIP) + req, _ := http.NewRequest("GET", "http://"+metadataIP, nil) + req.Header.Set("User-Agent", userAgent) + res, err := ctxhttp.Do(ctx, metaClient, req) if err != nil { resc <- false return diff --git a/vendor/cloud.google.com/go/iam/iam.go b/vendor/cloud.google.com/go/iam/iam.go new file mode 100644 index 0000000..8722ee8 --- /dev/null +++ b/vendor/cloud.google.com/go/iam/iam.go @@ -0,0 +1,256 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package iam supports the resource-specific operations of Google Cloud +// IAM (Identity and Access Management) for the Google Cloud Libraries. +// See https://cloud.google.com/iam for more about IAM. +// +// Users of the Google Cloud Libraries will typically not use this package +// directly. Instead they will begin with some resource that supports IAM, like +// a pubsub topic, and call its IAM method to get a Handle for that resource. +package iam + +import ( + "golang.org/x/net/context" + pb "google.golang.org/genproto/googleapis/iam/v1" + "google.golang.org/grpc" +) + +// client abstracts the IAMPolicy API to allow multiple implementations. +type client interface { + Get(ctx context.Context, resource string) (*pb.Policy, error) + Set(ctx context.Context, resource string, p *pb.Policy) error + Test(ctx context.Context, resource string, perms []string) ([]string, error) +} + +// grpcClient implements client for the standard gRPC-based IAMPolicy service. +type grpcClient struct { + c pb.IAMPolicyClient +} + +func (g *grpcClient) Get(ctx context.Context, resource string) (*pb.Policy, error) { + proto, err := g.c.GetIamPolicy(ctx, &pb.GetIamPolicyRequest{Resource: resource}) + if err != nil { + return nil, err + } + return proto, nil +} +func (g *grpcClient) Set(ctx context.Context, resource string, p *pb.Policy) error { + _, err := g.c.SetIamPolicy(ctx, &pb.SetIamPolicyRequest{ + Resource: resource, + Policy: p, + }) + return err +} + +func (g *grpcClient) Test(ctx context.Context, resource string, perms []string) ([]string, error) { + res, err := g.c.TestIamPermissions(ctx, &pb.TestIamPermissionsRequest{ + Resource: resource, + Permissions: perms, + }) + if err != nil { + return nil, err + } + return res.Permissions, nil +} + +// A Handle provides IAM operations for a resource. +type Handle struct { + c client + resource string +} + +// InternalNewHandle is for use by the Google Cloud Libraries only. +// +// InternalNewHandle returns a Handle for resource. +// The conn parameter refers to a server that must support the IAMPolicy service. +func InternalNewHandle(conn *grpc.ClientConn, resource string) *Handle { + return InternalNewHandleClient(&grpcClient{c: pb.NewIAMPolicyClient(conn)}, resource) +} + +// InternalNewHandleClient is for use by the Google Cloud Libraries only. +// +// InternalNewHandleClient returns a Handle for resource using the given +// client implementation. +func InternalNewHandleClient(c client, resource string) *Handle { + return &Handle{ + c: c, + resource: resource, + } +} + +// Policy retrieves the IAM policy for the resource. +func (h *Handle) Policy(ctx context.Context) (*Policy, error) { + proto, err := h.c.Get(ctx, h.resource) + if err != nil { + return nil, err + } + return &Policy{InternalProto: proto}, nil +} + +// SetPolicy replaces the resource's current policy with the supplied Policy. +// +// If policy was created from a prior call to Get, then the modification will +// only succeed if the policy has not changed since the Get. +func (h *Handle) SetPolicy(ctx context.Context, policy *Policy) error { + return h.c.Set(ctx, h.resource, policy.InternalProto) +} + +// TestPermissions returns the subset of permissions that the caller has on the resource. +func (h *Handle) TestPermissions(ctx context.Context, permissions []string) ([]string, error) { + return h.c.Test(ctx, h.resource, permissions) +} + +// A RoleName is a name representing a collection of permissions. +type RoleName string + +// Common role names. +const ( + Owner RoleName = "roles/owner" + Editor RoleName = "roles/editor" + Viewer RoleName = "roles/viewer" +) + +const ( + // AllUsers is a special member that denotes all users, even unauthenticated ones. + AllUsers = "allUsers" + + // AllAuthenticatedUsers is a special member that denotes all authenticated users. + AllAuthenticatedUsers = "allAuthenticatedUsers" +) + +// A Policy is a list of Bindings representing roles +// granted to members. +// +// The zero Policy is a valid policy with no bindings. +type Policy struct { + // TODO(jba): when type aliases are available, put Policy into an internal package + // and provide an exported alias here. + + // This field is exported for use by the Google Cloud Libraries only. + // It may become unexported in a future release. + InternalProto *pb.Policy +} + +// Members returns the list of members with the supplied role. +// The return value should not be modified. Use Add and Remove +// to modify the members of a role. +func (p *Policy) Members(r RoleName) []string { + b := p.binding(r) + if b == nil { + return nil + } + return b.Members +} + +// HasRole reports whether member has role r. +func (p *Policy) HasRole(member string, r RoleName) bool { + return memberIndex(member, p.binding(r)) >= 0 +} + +// Add adds member member to role r if it is not already present. +// A new binding is created if there is no binding for the role. +func (p *Policy) Add(member string, r RoleName) { + b := p.binding(r) + if b == nil { + if p.InternalProto == nil { + p.InternalProto = &pb.Policy{} + } + p.InternalProto.Bindings = append(p.InternalProto.Bindings, &pb.Binding{ + Role: string(r), + Members: []string{member}, + }) + return + } + if memberIndex(member, b) < 0 { + b.Members = append(b.Members, member) + return + } +} + +// Remove removes member from role r if it is present. +func (p *Policy) Remove(member string, r RoleName) { + bi := p.bindingIndex(r) + if bi < 0 { + return + } + bindings := p.InternalProto.Bindings + b := bindings[bi] + mi := memberIndex(member, b) + if mi < 0 { + return + } + // Order doesn't matter for bindings or members, so to remove, move the last item + // into the removed spot and shrink the slice. + if len(b.Members) == 1 { + // Remove binding. + last := len(bindings) - 1 + bindings[bi] = bindings[last] + bindings[last] = nil + p.InternalProto.Bindings = bindings[:last] + return + } + // Remove member. + // TODO(jba): worry about multiple copies of m? + last := len(b.Members) - 1 + b.Members[mi] = b.Members[last] + b.Members[last] = "" + b.Members = b.Members[:last] +} + +// Roles returns the names of all the roles that appear in the Policy. +func (p *Policy) Roles() []RoleName { + if p.InternalProto == nil { + return nil + } + var rns []RoleName + for _, b := range p.InternalProto.Bindings { + rns = append(rns, RoleName(b.Role)) + } + return rns +} + +// binding returns the Binding for the suppied role, or nil if there isn't one. +func (p *Policy) binding(r RoleName) *pb.Binding { + i := p.bindingIndex(r) + if i < 0 { + return nil + } + return p.InternalProto.Bindings[i] +} + +func (p *Policy) bindingIndex(r RoleName) int { + if p.InternalProto == nil { + return -1 + } + for i, b := range p.InternalProto.Bindings { + if b.Role == string(r) { + return i + } + } + return -1 +} + +// memberIndex returns the index of m in b's Members, or -1 if not found. +func memberIndex(m string, b *pb.Binding) int { + if b == nil { + return -1 + } + for i, mm := range b.Members { + if mm == m { + return i + } + } + return -1 +} diff --git a/vendor/cloud.google.com/go/internal/cloud.go b/vendor/cloud.google.com/go/internal/cloud.go deleted file mode 100644 index 8e0c8f8..0000000 --- a/vendor/cloud.google.com/go/internal/cloud.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2014 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package internal provides support for the cloud packages. -// -// Users should not import this package directly. -package internal - -import ( - "fmt" - "net/http" -) - -const userAgent = "gcloud-golang/0.1" - -// Transport is an http.RoundTripper that appends Google Cloud client's -// user-agent to the original request's user-agent header. -type Transport struct { - // TODO(bradfitz): delete internal.Transport. It's too wrappy for what it does. - // Do User-Agent some other way. - - // Base is the actual http.RoundTripper - // requests will use. It must not be nil. - Base http.RoundTripper -} - -// RoundTrip appends a user-agent to the existing user-agent -// header and delegates the request to the base http.RoundTripper. -func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { - req = cloneRequest(req) - ua := req.Header.Get("User-Agent") - if ua == "" { - ua = userAgent - } else { - ua = fmt.Sprintf("%s %s", ua, userAgent) - } - req.Header.Set("User-Agent", ua) - return t.Base.RoundTrip(req) -} - -// cloneRequest returns a clone of the provided *http.Request. -// The clone is a shallow copy of the struct and its Header map. -func cloneRequest(r *http.Request) *http.Request { - // shallow copy of the struct - r2 := new(http.Request) - *r2 = *r - // deep copy of the Header - r2.Header = make(http.Header) - for k, s := range r.Header { - r2.Header[k] = s - } - return r2 -} diff --git a/vendor/cloud.google.com/go/internal/retry.go b/vendor/cloud.google.com/go/internal/retry.go index 79995be..f554fbf 100644 --- a/vendor/cloud.google.com/go/internal/retry.go +++ b/vendor/cloud.google.com/go/internal/retry.go @@ -27,7 +27,8 @@ import ( // backoff parameters. It returns when one of the following occurs: // When f's first return value is true, Retry immediately returns with f's second // return value. -// When the provided context is done, Retry returns with ctx.Err(). +// When the provided context is done, Retry returns with an error that +// includes both ctx.Error() and the last error returned by f. func Retry(ctx context.Context, bo gax.Backoff, f func() (stop bool, err error)) error { return retry(ctx, bo, f, gax.Sleep) } diff --git a/vendor/cloud.google.com/go/internal/version/update_version.sh b/vendor/cloud.google.com/go/internal/version/update_version.sh new file mode 100755 index 0000000..fecf1f0 --- /dev/null +++ b/vendor/cloud.google.com/go/internal/version/update_version.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +today=$(date +%Y%m%d) + +sed -i -r -e 's/const Repo = "([0-9]{8})"/const Repo = "'$today'"/' $GOFILE + diff --git a/vendor/cloud.google.com/go/internal/version/version.go b/vendor/cloud.google.com/go/internal/version/version.go new file mode 100644 index 0000000..33f1cdb --- /dev/null +++ b/vendor/cloud.google.com/go/internal/version/version.go @@ -0,0 +1,71 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate ./update_version.sh + +// Package version contains version information for Google Cloud Client +// Libraries for Go, as reported in request headers. +package version + +import ( + "runtime" + "strings" + "unicode" +) + +// Repo is the current version of the client libraries in this +// repo. It should be a date in YYYYMMDD format. +const Repo = "20170404" + +// Go returns the Go runtime version. The returned string +// has no whitespace. +func Go() string { + return goVersion +} + +var goVersion = goVer(runtime.Version()) + +const develPrefix = "devel +" + +func goVer(s string) string { + if strings.HasPrefix(s, develPrefix) { + s = s[len(develPrefix):] + if p := strings.IndexFunc(s, unicode.IsSpace); p >= 0 { + s = s[:p] + } + return s + } + + if strings.HasPrefix(s, "go1") { + s = s[2:] + var prerelease string + if p := strings.IndexFunc(s, notSemverRune); p >= 0 { + s, prerelease = s[:p], s[p:] + } + if strings.HasSuffix(s, ".") { + s += "0" + } else if strings.Count(s, ".") < 2 { + s += ".0" + } + if prerelease != "" { + s += "-" + prerelease + } + return s + } + return "" +} + +func notSemverRune(r rune) bool { + return strings.IndexRune("0123456789.", r) < 0 +} diff --git a/vendor/cloud.google.com/go/storage/acl.go b/vendor/cloud.google.com/go/storage/acl.go index 6c24394..041e541 100644 --- a/vendor/cloud.google.com/go/storage/acl.go +++ b/vendor/cloud.google.com/go/storage/acl.go @@ -96,7 +96,9 @@ func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) { var acls *raw.ObjectAccessControls var err error err = runWithRetry(ctx, func() error { - acls, err = a.c.raw.DefaultObjectAccessControls.List(a.bucket).Context(ctx).Do() + req := a.c.raw.DefaultObjectAccessControls.List(a.bucket).Context(ctx) + setClientHeader(req.Header()) + acls, err = req.Do() return err }) if err != nil { @@ -112,7 +114,9 @@ func (a *ACLHandle) bucketDefaultSet(ctx context.Context, entity ACLEntity, role Role: string(role), } err := runWithRetry(ctx, func() error { - _, err := a.c.raw.DefaultObjectAccessControls.Update(a.bucket, string(entity), acl).Context(ctx).Do() + req := a.c.raw.DefaultObjectAccessControls.Update(a.bucket, string(entity), acl).Context(ctx) + setClientHeader(req.Header()) + _, err := req.Do() return err }) if err != nil { @@ -123,7 +127,9 @@ func (a *ACLHandle) bucketDefaultSet(ctx context.Context, entity ACLEntity, role func (a *ACLHandle) bucketDefaultDelete(ctx context.Context, entity ACLEntity) error { err := runWithRetry(ctx, func() error { - return a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity)).Context(ctx).Do() + req := a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity)).Context(ctx) + setClientHeader(req.Header()) + return req.Do() }) if err != nil { return fmt.Errorf("storage: error deleting default ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err) @@ -135,7 +141,9 @@ func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) { var acls *raw.BucketAccessControls var err error err = runWithRetry(ctx, func() error { - acls, err = a.c.raw.BucketAccessControls.List(a.bucket).Context(ctx).Do() + req := a.c.raw.BucketAccessControls.List(a.bucket).Context(ctx) + setClientHeader(req.Header()) + acls, err = req.Do() return err }) if err != nil { @@ -156,7 +164,9 @@ func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRol Role: string(role), } err := runWithRetry(ctx, func() error { - _, err := a.c.raw.BucketAccessControls.Update(a.bucket, string(entity), acl).Context(ctx).Do() + req := a.c.raw.BucketAccessControls.Update(a.bucket, string(entity), acl).Context(ctx) + setClientHeader(req.Header()) + _, err := req.Do() return err }) if err != nil { @@ -167,7 +177,9 @@ func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRol func (a *ACLHandle) bucketDelete(ctx context.Context, entity ACLEntity) error { err := runWithRetry(ctx, func() error { - return a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity)).Context(ctx).Do() + req := a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity)).Context(ctx) + setClientHeader(req.Header()) + return req.Do() }) if err != nil { return fmt.Errorf("storage: error deleting bucket ACL entry for bucket %q, entity %q: %v", a.bucket, entity, err) @@ -179,7 +191,9 @@ func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) { var acls *raw.ObjectAccessControls var err error err = runWithRetry(ctx, func() error { - acls, err = a.c.raw.ObjectAccessControls.List(a.bucket, a.object).Context(ctx).Do() + req := a.c.raw.ObjectAccessControls.List(a.bucket, a.object).Context(ctx) + setClientHeader(req.Header()) + acls, err = req.Do() return err }) if err != nil { @@ -195,7 +209,9 @@ func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRol Role: string(role), } err := runWithRetry(ctx, func() error { - _, err := a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl).Context(ctx).Do() + req := a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl).Context(ctx) + setClientHeader(req.Header()) + _, err := req.Do() return err }) if err != nil { @@ -206,7 +222,9 @@ func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRol func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error { err := runWithRetry(ctx, func() error { - return a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity)).Context(ctx).Do() + req := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity)).Context(ctx) + setClientHeader(req.Header()) + return req.Do() }) if err != nil { return fmt.Errorf("storage: error deleting object ACL entry for bucket %q, file %q, entity %q: %v", a.bucket, a.object, entity, err) diff --git a/vendor/cloud.google.com/go/storage/bucket.go b/vendor/cloud.google.com/go/storage/bucket.go index f87fe33..deed5b2 100644 --- a/vendor/cloud.google.com/go/storage/bucket.go +++ b/vendor/cloud.google.com/go/storage/bucket.go @@ -35,12 +35,14 @@ func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *Buck } bkt.Name = b.name req := b.c.raw.Buckets.Insert(projectID, bkt) + setClientHeader(req.Header()) return runWithRetry(ctx, func() error { _, err := req.Context(ctx).Do(); return err }) } // Delete deletes the Bucket. func (b *BucketHandle) Delete(ctx context.Context) error { req := b.c.raw.Buckets.Delete(b.name) + setClientHeader(req.Header()) return runWithRetry(ctx, func() error { return req.Context(ctx).Do() }) } @@ -80,10 +82,12 @@ func (b *BucketHandle) Object(name string) *ObjectHandle { // Attrs returns the metadata for the bucket. func (b *BucketHandle) Attrs(ctx context.Context) (*BucketAttrs, error) { + req := b.c.raw.Buckets.Get(b.name).Projection("full") + setClientHeader(req.Header()) var resp *raw.Bucket var err error err = runWithRetry(ctx, func() error { - resp, err = b.c.raw.Buckets.Get(b.name).Projection("full").Context(ctx).Do() + resp, err = req.Context(ctx).Do() return err }) if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { @@ -113,7 +117,7 @@ type BucketAttrs struct { // MetaGeneration is the metadata generation of the bucket. MetaGeneration int64 - // StorageClass is the storage class of the bucket. This defines + // StorageClass is the default storage class of the bucket. This defines // how objects in the bucket are stored and determines the SLA // and the cost of storage. Typical values are "MULTI_REGIONAL", // "REGIONAL", "NEARLINE", "COLDLINE", "STANDARD" and @@ -231,6 +235,7 @@ func (it *ObjectIterator) Next() (*ObjectAttrs, error) { func (it *ObjectIterator) fetch(pageSize int, pageToken string) (string, error) { req := it.bucket.c.raw.Objects.List(it.bucket.name) + setClientHeader(req.Header()) req.Projection("full") req.Delimiter(it.query.Delimiter) req.Prefix(it.query.Prefix) @@ -309,6 +314,7 @@ func (it *BucketIterator) PageInfo() *iterator.PageInfo { return it.pageInfo } func (it *BucketIterator) fetch(pageSize int, pageToken string) (string, error) { req := it.client.raw.Buckets.List(it.projectID) + setClientHeader(req.Header()) req.Projection("full") req.Prefix(it.Prefix) req.PageToken(pageToken) diff --git a/vendor/cloud.google.com/go/storage/copy.go b/vendor/cloud.google.com/go/storage/copy.go index 1f28455..16e28e3 100644 --- a/vendor/cloud.google.com/go/storage/copy.go +++ b/vendor/cloud.google.com/go/storage/copy.go @@ -12,15 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package storage contains a Google Cloud Storage client. -// -// This package is experimental and may make backwards-incompatible changes. package storage import ( "errors" "fmt" - "reflect" "golang.org/x/net/context" raw "google.golang.org/api/storage/v1" @@ -71,17 +67,11 @@ func (c *Copier) Run(ctx context.Context) (*ObjectAttrs, error) { if err := c.dst.validate(); err != nil { return nil, err } - var rawObject *raw.Object - // If any attribute was set, then we make sure the name matches the destination - // name, and we check that ContentType is non-empty so we can provide a better - // error message than the service. - if !reflect.DeepEqual(c.ObjectAttrs, ObjectAttrs{}) { - c.ObjectAttrs.Name = c.dst.object - if c.ObjectAttrs.ContentType == "" { - return nil, errors.New("storage: Copier.ContentType must be non-empty") - } - rawObject = c.ObjectAttrs.toRawObject(c.dst.bucket) - } + // Convert destination attributes to raw form, omitting the bucket. + // If the bucket is included but name or content-type aren't, the service + // returns a 400 with "Required" as the only message. Omitting the bucket + // does not cause any problems. + rawObject := c.ObjectAttrs.toRawObject("") for { res, err := c.callRewrite(ctx, c.src, rawObject) if err != nil { @@ -118,6 +108,7 @@ func (c *Copier) callRewrite(ctx context.Context, src *ObjectHandle, rawObj *raw } var res *raw.RewriteResponse var err error + setClientHeader(call.Header()) err = runWithRetry(ctx, func() error { res, err = call.Do(); return err }) if err != nil { return nil, err @@ -189,6 +180,7 @@ func (c *Composer) Run(ctx context.Context) (*ObjectAttrs, error) { } var obj *raw.Object var err error + setClientHeader(call.Header()) err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) if err != nil { return nil, err diff --git a/vendor/cloud.google.com/go/storage/doc.go b/vendor/cloud.google.com/go/storage/doc.go index cf6496b..951391f 100644 --- a/vendor/cloud.google.com/go/storage/doc.go +++ b/vendor/cloud.google.com/go/storage/doc.go @@ -152,5 +152,10 @@ SignedURL for details. // TODO: Handle error. } fmt.Println(url) + +Authentication + +See examples of authorization and authentication at +https://godoc.org/cloud.google.com/go#pkg-examples. */ package storage // import "cloud.google.com/go/storage" diff --git a/vendor/cloud.google.com/go/storage/iam.go b/vendor/cloud.google.com/go/storage/iam.go new file mode 100644 index 0000000..0cd9501 --- /dev/null +++ b/vendor/cloud.google.com/go/storage/iam.go @@ -0,0 +1,99 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "errors" + + "cloud.google.com/go/iam" + "golang.org/x/net/context" + raw "google.golang.org/api/storage/v1" + iampb "google.golang.org/genproto/googleapis/iam/v1" +) + +// IAM provides access to IAM access control for the bucket. +func (b *BucketHandle) IAM() *iam.Handle { + return iam.InternalNewHandleClient(&iamClient{raw: b.c.raw}, b.name) +} + +// iamClient implements the iam.client interface. +type iamClient struct { + raw *raw.Service +} + +func (c *iamClient) Get(ctx context.Context, resource string) (*iampb.Policy, error) { + req := c.raw.Buckets.GetIamPolicy(resource) + setClientHeader(req.Header()) + var rp *raw.Policy + var err error + err = runWithRetry(ctx, func() error { + rp, err = req.Context(ctx).Do() + return err + }) + if err != nil { + return nil, err + } + return iamFromStoragePolicy(rp), nil +} + +func (c *iamClient) Set(ctx context.Context, resource string, p *iampb.Policy) error { + rp := iamToStoragePolicy(p) + req := c.raw.Buckets.SetIamPolicy(resource, rp) + setClientHeader(req.Header()) + return runWithRetry(ctx, func() error { + _, err := req.Context(ctx).Do() + return err + }) +} + +func (c *iamClient) Test(context.Context, string, []string) ([]string, error) { + return nil, errors.New("TestPermissions is unimplemented") +} + +func iamToStoragePolicy(ip *iampb.Policy) *raw.Policy { + return &raw.Policy{ + Bindings: iamToStorageBindings(ip.Bindings), + Etag: string(ip.Etag), + } +} + +func iamToStorageBindings(ibs []*iampb.Binding) []*raw.PolicyBindings { + var rbs []*raw.PolicyBindings + for _, ib := range ibs { + rbs = append(rbs, &raw.PolicyBindings{ + Role: ib.Role, + Members: ib.Members, + }) + } + return rbs +} + +func iamFromStoragePolicy(rp *raw.Policy) *iampb.Policy { + return &iampb.Policy{ + Bindings: iamFromStorageBindings(rp.Bindings), + Etag: []byte(rp.Etag), + } +} + +func iamFromStorageBindings(rbs []*raw.PolicyBindings) []*iampb.Binding { + var ibs []*iampb.Binding + for _, rb := range rbs { + ibs = append(ibs, &iampb.Binding{ + Role: rb.Role, + Members: rb.Members, + }) + } + return ibs +} diff --git a/vendor/cloud.google.com/go/storage/reader.go b/vendor/cloud.google.com/go/storage/reader.go index 329a5f3..aa103c1 100644 --- a/vendor/cloud.google.com/go/storage/reader.go +++ b/vendor/cloud.google.com/go/storage/reader.go @@ -15,15 +15,22 @@ package storage import ( + "fmt" + "hash/crc32" "io" ) +var crc32cTable = crc32.MakeTable(crc32.Castagnoli) + // Reader reads a Cloud Storage object. // It implements io.Reader. type Reader struct { body io.ReadCloser remain, size int64 contentType string + checkCRC bool // should we check the CRC? + wantCRC uint32 // the CRC32c value the server sent in the header + gotCRC uint32 // running crc } // Close closes the Reader. It must be called when done reading. @@ -36,6 +43,16 @@ func (r *Reader) Read(p []byte) (int, error) { if r.remain != -1 { r.remain -= int64(n) } + if r.checkCRC { + r.gotCRC = crc32.Update(r.gotCRC, crc32cTable, p[:n]) + // Check CRC here. It would be natural to check it in Close, but + // everybody defers Close on the assumption that it doesn't return + // anything worth looking at. + if r.remain == 0 && r.gotCRC != r.wantCRC { + return n, fmt.Errorf("storage: bad CRC on read: got %d, want %d", + r.gotCRC, r.wantCRC) + } + } return n, err } diff --git a/vendor/cloud.google.com/go/storage/storage.go b/vendor/cloud.google.com/go/storage/storage.go index 53fc029..6b82fa3 100644 --- a/vendor/cloud.google.com/go/storage/storage.go +++ b/vendor/cloud.google.com/go/storage/storage.go @@ -39,6 +39,7 @@ import ( "google.golang.org/api/transport" "cloud.google.com/go/internal/optional" + "cloud.google.com/go/internal/version" "golang.org/x/net/context" "google.golang.org/api/googleapi" raw "google.golang.org/api/storage/v1" @@ -65,6 +66,12 @@ const ( ScopeReadWrite = raw.DevstorageReadWriteScope ) +var xGoogHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), version.Repo) + +func setClientHeader(headers http.Header) { + headers.Set("x-goog-api-client", xGoogHeader) +} + // Client is a client for interacting with Google Cloud Storage. // // Clients should be reused instead of created as needed. @@ -202,7 +209,7 @@ type SignedURLOptions struct { // If provided, the client should provide the exact value on the request // header in order to use the signed URL. // Optional. - MD5 []byte + MD5 string } // SignedURL returns a URL for the specified object. Signed URLs allow @@ -225,6 +232,12 @@ func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) { if opts.Expires.IsZero() { return "", errors.New("storage: missing required expires option") } + if opts.MD5 != "" { + md5, err := base64.StdEncoding.DecodeString(opts.MD5) + if err != nil || len(md5) != 16 { + return "", errors.New("storage: invalid MD5 checksum") + } + } signBytes := opts.SignBytes if opts.PrivateKey != nil { @@ -241,8 +254,6 @@ func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) { sum[:], ) } - } else { - signBytes = opts.SignBytes } u := &url.URL{ @@ -254,7 +265,9 @@ func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) { fmt.Fprintf(buf, "%s\n", opts.MD5) fmt.Fprintf(buf, "%s\n", opts.ContentType) fmt.Fprintf(buf, "%d\n", opts.Expires.Unix()) - fmt.Fprintf(buf, "%s", strings.Join(opts.Headers, "\n")) + if len(opts.Headers) > 0 { + fmt.Fprintf(buf, "%s\n", strings.Join(opts.Headers, "\n")) + } fmt.Fprintf(buf, "%s", u.String()) b, err := signBytes(buf.Bytes()) @@ -339,6 +352,7 @@ func (o *ObjectHandle) Attrs(ctx context.Context) (*ObjectAttrs, error) { } var obj *raw.Object var err error + setClientHeader(call.Header()) err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { return nil, ErrObjectNotExist @@ -412,6 +426,7 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) ( } var obj *raw.Object var err error + setClientHeader(call.Header()) err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { return nil, ErrObjectNotExist @@ -452,6 +467,7 @@ func (o *ObjectHandle) Delete(ctx context.Context) error { if err := applyConds("Delete", o.gen, o.conds, call); err != nil { return err } + setClientHeader(call.Header()) err := runWithRetry(ctx, func() error { return call.Do() }) switch e := err.(type) { case nil: @@ -560,15 +576,37 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) body.Close() body = emptyBody } - + var ( + checkCRC bool + crc uint32 + ) + // Even if there is a CRC header, we can't compute the hash on partial data. + if remain == size { + crc, checkCRC = parseCRC32c(res) + } return &Reader{ body: body, size: size, remain: remain, contentType: res.Header.Get("Content-Type"), + wantCRC: crc, + checkCRC: checkCRC, }, nil } +func parseCRC32c(res *http.Response) (uint32, bool) { + const prefix = "crc32c=" + for _, spec := range res.Header["X-Goog-Hash"] { + if strings.HasPrefix(spec, prefix) { + c, err := decodeUint32(spec[len(prefix):]) + if err == nil { + return c, true + } + } + } + return 0, false +} + var emptyBody = ioutil.NopCloser(strings.NewReader("")) // NewWriter returns a storage Writer that writes to the GCS object @@ -656,6 +694,7 @@ func (o *ObjectAttrs) toRawObject(bucket string) *raw.Object { ContentLanguage: o.ContentLanguage, CacheControl: o.CacheControl, ContentDisposition: o.ContentDisposition, + StorageClass: o.StorageClass, Acl: acl, Metadata: o.Metadata, } @@ -717,21 +756,20 @@ type ObjectAttrs struct { // This field is read-only. Generation int64 - // MetaGeneration is the version of the metadata for this + // Metageneration is the version of the metadata for this // object at this generation. This field is used for preconditions // and for detecting changes in metadata. A metageneration number // is only meaningful in the context of a particular generation // of a particular object. This field is read-only. - MetaGeneration int64 + Metageneration int64 - // StorageClass is the storage class of the bucket. + // StorageClass is the storage class of the object. // This value defines how objects in the bucket are stored and // determines the SLA and the cost of storage. Typical values are // "MULTI_REGIONAL", "REGIONAL", "NEARLINE", "COLDLINE", "STANDARD" // and "DURABLE_REDUCED_AVAILABILITY". // It defaults to "STANDARD", which is equivalent to "MULTI_REGIONAL" - // or "REGIONAL" depending on the bucket's location settings. This - // field is read-only. + // or "REGIONAL" depending on the bucket's location settings. StorageClass string // Created is the time the object was created. This field is read-only. @@ -786,11 +824,7 @@ func newObject(o *raw.Object) *ObjectAttrs { owner = o.Owner.Entity } md5, _ := base64.StdEncoding.DecodeString(o.Md5Hash) - var crc32c uint32 - d, err := base64.StdEncoding.DecodeString(o.Crc32c) - if err == nil && len(d) == 4 { - crc32c = uint32(d[0])<<24 + uint32(d[1])<<16 + uint32(d[2])<<8 + uint32(d[3]) - } + crc32c, _ := decodeUint32(o.Crc32c) var sha256 string if o.CustomerEncryption != nil { sha256 = o.CustomerEncryption.KeySha256 @@ -810,7 +844,7 @@ func newObject(o *raw.Object) *ObjectAttrs { MediaLink: o.MediaLink, Metadata: o.Metadata, Generation: o.Generation, - MetaGeneration: o.Metageneration, + Metageneration: o.Metageneration, StorageClass: o.StorageClass, CustomerKeySHA256: sha256, Created: convertTime(o.TimeCreated), @@ -819,6 +853,24 @@ func newObject(o *raw.Object) *ObjectAttrs { } } +// Decode a uint32 encoded in Base64 in big-endian byte order. +func decodeUint32(b64 string) (uint32, error) { + d, err := base64.StdEncoding.DecodeString(b64) + if err != nil { + return 0, err + } + if len(d) != 4 { + return 0, fmt.Errorf("storage: %q does not encode a 32-bit value", d) + } + return uint32(d[0])<<24 + uint32(d[1])<<16 + uint32(d[2])<<8 + uint32(d[3]), nil +} + +// Encode a uint32 as Base64 in big-endian byte order. +func encodeUint32(u uint32) string { + b := []byte{byte(u >> 24), byte(u >> 16), byte(u >> 8), byte(u)} + return base64.StdEncoding.EncodeToString(b) +} + // Query represents a query to filter objects from a bucket. type Query struct { // Delimiter returns results in a directory-like fashion. diff --git a/vendor/cloud.google.com/go/storage/writer.go b/vendor/cloud.google.com/go/storage/writer.go index a7a9329..21ffdb9 100644 --- a/vendor/cloud.google.com/go/storage/writer.go +++ b/vendor/cloud.google.com/go/storage/writer.go @@ -15,6 +15,7 @@ package storage import ( + "encoding/base64" "errors" "fmt" "io" @@ -32,6 +33,11 @@ type Writer struct { // attributes are ignored. ObjectAttrs + // SendCRC specifies whether to transmit a CRC32C field. It should be set + // to true in addition to setting the Writer's CRC32C field, because zero + // is a valid CRC and normally a zero would not be transmitted. + SendCRC32C bool + // ChunkSize controls the maximum number of bytes of the object that the // Writer will attempt to send to the server in a single request. Objects // smaller than the size will be sent in a single request, while larger @@ -81,7 +87,14 @@ func (w *Writer) open() error { go func() { defer close(w.donec) - call := w.o.c.raw.Objects.Insert(w.o.bucket, attrs.toRawObject(w.o.bucket)). + rawObj := attrs.toRawObject(w.o.bucket) + if w.SendCRC32C { + rawObj.Crc32c = encodeUint32(attrs.CRC32C) + } + if w.MD5 != nil { + rawObj.Md5Hash = base64.StdEncoding.EncodeToString(w.MD5) + } + call := w.o.c.raw.Objects.Insert(w.o.bucket, rawObj). Media(pr, mediaOpts...). Projection("full"). Context(w.ctx) @@ -93,6 +106,7 @@ func (w *Writer) open() error { var resp *raw.Object err := applyConds("NewWriter", w.o.gen, w.o.conds, call) if err == nil { + setClientHeader(call.Header()) resp, err = call.Do() } if err != nil { @@ -120,7 +134,7 @@ func (w *Writer) Write(p []byte) (n int, err error) { // Close completes the write operation and flushes any buffered data. // If Close doesn't return an error, metadata about the written object -// can be retrieved by calling Object. +// can be retrieved by calling Attrs. func (w *Writer) Close() error { if !w.opened { if err := w.open(); err != nil { -- cgit v1.2.3