From de6d2c524430287c699aaa898c1325da6afea539 Mon Sep 17 00:00:00 2001 From: Niall Sheridan Date: Wed, 20 Jun 2018 22:39:07 +0100 Subject: Update dependencies --- vendor/cloud.google.com/go/storage/storage.go | 236 ++++++++++---------------- 1 file changed, 90 insertions(+), 146 deletions(-) (limited to 'vendor/cloud.google.com/go/storage/storage.go') diff --git a/vendor/cloud.google.com/go/storage/storage.go b/vendor/cloud.google.com/go/storage/storage.go index 8ffa845..2681922 100644 --- a/vendor/cloud.google.com/go/storage/storage.go +++ b/vendor/cloud.google.com/go/storage/storage.go @@ -1,4 +1,4 @@ -// Copyright 2014 Google Inc. All Rights Reserved. +// Copyright 2014 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -26,15 +26,17 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "net/url" "reflect" + "regexp" + "sort" "strconv" "strings" "time" "unicode/utf8" + "cloud.google.com/go/internal/trace" "google.golang.org/api/option" htransport "google.golang.org/api/transport/http" @@ -182,6 +184,60 @@ type SignedURLOptions struct { MD5 string } +var ( + canonicalHeaderRegexp = regexp.MustCompile(`(?i)^(x-goog-[^:]+):(.*)?$`) + excludedCanonicalHeaders = map[string]bool{ + "x-goog-encryption-key": true, + "x-goog-encryption-key-sha256": true, + } +) + +// sanitizeHeaders applies the specifications for canonical extension headers at +// https://cloud.google.com/storage/docs/access-control/signed-urls#about-canonical-extension-headers. +func sanitizeHeaders(hdrs []string) []string { + headerMap := map[string][]string{} + for _, hdr := range hdrs { + // No leading or trailing whitespaces. + sanitizedHeader := strings.TrimSpace(hdr) + + // Only keep canonical headers, discard any others. + headerMatches := canonicalHeaderRegexp.FindStringSubmatch(sanitizedHeader) + if len(headerMatches) == 0 { + continue + } + + header := strings.ToLower(strings.TrimSpace(headerMatches[1])) + if excludedCanonicalHeaders[headerMatches[1]] { + // Do not keep any deliberately excluded canonical headers when signing. + continue + } + value := strings.TrimSpace(headerMatches[2]) + if len(value) > 0 { + // Remove duplicate headers by appending the values of duplicates + // in their order of appearance. + headerMap[header] = append(headerMap[header], value) + } + } + + var sanitizedHeaders []string + for header, values := range headerMap { + // There should be no spaces around the colon separating the + // header name from the header value or around the values + // themselves. The values should be separated by commas. + // NOTE: The semantics for headers without a value are not clear. + // However from specifications these should be edge-cases + // anyway and we should assume that there will be no + // canonical headers using empty values. Any such headers + // are discarded at the regexp stage above. + sanitizedHeaders = append( + sanitizedHeaders, + fmt.Sprintf("%s:%s", header, strings.Join(values, ",")), + ) + } + sort.Strings(sanitizedHeaders) + return sanitizedHeaders +} + // SignedURL returns a URL for the specified object. Signed URLs allow // the users access to a restricted resource for a limited time without having a // Google account or signing in. For more information about the signed @@ -208,6 +264,7 @@ func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) { return "", errors.New("storage: invalid MD5 checksum") } } + opts.Headers = sanitizeHeaders(opts.Headers) signBytes := opts.SignBytes if opts.PrivateKey != nil { @@ -258,14 +315,15 @@ func SignedURL(bucket, name string, opts *SignedURLOptions) (string, error) { // ObjectHandle provides operations on an object in a Google Cloud Storage bucket. // Use BucketHandle.Object to get a handle. type ObjectHandle struct { - c *Client - bucket string - object string - acl ACLHandle - gen int64 // a negative value indicates latest - conds *Conditions - encryptionKey []byte // AES-256 key - userProject string // for requester-pays buckets + c *Client + bucket string + object string + acl ACLHandle + gen int64 // a negative value indicates latest + conds *Conditions + encryptionKey []byte // AES-256 key + userProject string // for requester-pays buckets + readCompressed bool // Accept-Encoding: gzip } // ACL provides access to the object's access control list. @@ -310,7 +368,10 @@ func (o *ObjectHandle) Key(encryptionKey []byte) *ObjectHandle { // Attrs returns meta information about the object. // ErrObjectNotExist will be returned if the object is not found. -func (o *ObjectHandle) Attrs(ctx context.Context) (*ObjectAttrs, error) { +func (o *ObjectHandle) Attrs(ctx context.Context) (attrs *ObjectAttrs, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.Attrs") + defer func() { trace.EndSpan(ctx, err) }() + if err := o.validate(); err != nil { return nil, err } @@ -325,7 +386,6 @@ func (o *ObjectHandle) Attrs(ctx context.Context) (*ObjectAttrs, error) { return nil, err } var obj *raw.Object - var err error setClientHeader(call.Header()) err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { @@ -340,7 +400,10 @@ func (o *ObjectHandle) Attrs(ctx context.Context) (*ObjectAttrs, error) { // Update updates an object with the provided attributes. // All zero-value attributes are ignored. // ErrObjectNotExist will be returned if the object is not found. -func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (*ObjectAttrs, error) { +func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (oa *ObjectAttrs, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.Update") + defer func() { trace.EndSpan(ctx, err) }() + if err := o.validate(); err != nil { return nil, err } @@ -408,7 +471,6 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) ( return nil, err } var obj *raw.Object - var err error setClientHeader(call.Header()) err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { @@ -467,140 +529,13 @@ func (o *ObjectHandle) Delete(ctx context.Context) error { return err } -// NewReader creates a new Reader to read the contents of the -// object. -// ErrObjectNotExist will be returned if the object is not found. -// -// The caller must call Close on the returned Reader when done reading. -func (o *ObjectHandle) NewReader(ctx context.Context) (*Reader, error) { - return o.NewRangeReader(ctx, 0, -1) -} - -// NewRangeReader reads part of an object, reading at most length bytes -// starting at the given offset. If length is negative, the object is read -// until the end. -func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (*Reader, error) { - if err := o.validate(); err != nil { - return nil, err - } - if offset < 0 { - return nil, fmt.Errorf("storage: invalid offset %d < 0", offset) - } - if o.conds != nil { - if err := o.conds.validate("NewRangeReader"); err != nil { - return nil, err - } - } - u := &url.URL{ - Scheme: "https", - Host: "storage.googleapis.com", - Path: fmt.Sprintf("/%s/%s", o.bucket, o.object), - RawQuery: conditionsQuery(o.gen, o.conds), - } - verb := "GET" - if length == 0 { - verb = "HEAD" - } - req, err := http.NewRequest(verb, u.String(), nil) - if err != nil { - return nil, err - } - req = withContext(req, ctx) - if length < 0 && offset > 0 { - req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) - } else if length > 0 { - req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) - } - if o.userProject != "" { - req.Header.Set("X-Goog-User-Project", o.userProject) - } - if err := setEncryptionHeaders(req.Header, o.encryptionKey, false); err != nil { - return nil, err - } - var res *http.Response - err = runWithRetry(ctx, func() error { - res, err = o.c.hc.Do(req) - if err != nil { - return err - } - if res.StatusCode == http.StatusNotFound { - res.Body.Close() - return ErrObjectNotExist - } - if res.StatusCode < 200 || res.StatusCode > 299 { - body, _ := ioutil.ReadAll(res.Body) - res.Body.Close() - return &googleapi.Error{ - Code: res.StatusCode, - Header: res.Header, - Body: string(body), - } - } - if offset > 0 && length != 0 && res.StatusCode != http.StatusPartialContent { - res.Body.Close() - return errors.New("storage: partial request not satisfied") - } - return nil - }) - if err != nil { - return nil, err - } - - var size int64 // total size of object, even if a range was requested. - if res.StatusCode == http.StatusPartialContent { - cr := strings.TrimSpace(res.Header.Get("Content-Range")) - if !strings.HasPrefix(cr, "bytes ") || !strings.Contains(cr, "/") { - return nil, fmt.Errorf("storage: invalid Content-Range %q", cr) - } - size, err = strconv.ParseInt(cr[strings.LastIndex(cr, "/")+1:], 10, 64) - if err != nil { - return nil, fmt.Errorf("storage: invalid Content-Range %q", cr) - } - } else { - size = res.ContentLength - } - - remain := res.ContentLength - body := res.Body - if length == 0 { - remain = 0 - body.Close() - body = emptyBody - } - var ( - checkCRC bool - crc uint32 - ) - // Even if there is a CRC header, we can't compute the hash on partial data. - if remain == size { - crc, checkCRC = parseCRC32c(res) - } - return &Reader{ - body: body, - size: size, - remain: remain, - contentType: res.Header.Get("Content-Type"), - cacheControl: res.Header.Get("Cache-Control"), - wantCRC: crc, - checkCRC: checkCRC, - }, nil -} - -func parseCRC32c(res *http.Response) (uint32, bool) { - const prefix = "crc32c=" - for _, spec := range res.Header["X-Goog-Hash"] { - if strings.HasPrefix(spec, prefix) { - c, err := decodeUint32(spec[len(prefix):]) - if err == nil { - return c, true - } - } - } - return 0, false +// ReadCompressed when true causes the read to happen without decompressing. +func (o *ObjectHandle) ReadCompressed(compressed bool) *ObjectHandle { + o2 := *o + o2.readCompressed = compressed + return &o2 } -var emptyBody = ioutil.NopCloser(strings.NewReader("")) - // NewWriter returns a storage Writer that writes to the GCS object // associated with this ObjectHandle. // @@ -787,6 +722,14 @@ type ObjectAttrs struct { // encryption in Google Cloud Storage. CustomerKeySHA256 string + // Cloud KMS key name, in the form + // projects/P/locations/L/keyRings/R/cryptoKeys/K, used to encrypt this object, + // if the object is encrypted by such a key. + // + // Providing both a KMSKeyName and a customer-supplied encryption key (via + // ObjectHandle.Key) will result in an error when writing an object. + KMSKeyName string + // Prefix is set only for ObjectAttrs which represent synthetic "directory // entries" when iterating over buckets using Query.Delimiter. See // ObjectIterator.Next. When set, no other fields in ObjectAttrs will be @@ -844,6 +787,7 @@ func newObject(o *raw.Object) *ObjectAttrs { Metageneration: o.Metageneration, StorageClass: o.StorageClass, CustomerKeySHA256: sha256, + KMSKeyName: o.KmsKeyName, Created: convertTime(o.TimeCreated), Deleted: convertTime(o.TimeDeleted), Updated: convertTime(o.Updated), -- cgit v1.2.3