// Copyright 2011 Google Inc. All rights reserved. // Use of this source code is governed by the Apache 2.0 // license that can be found in the LICENSE file. // Package urlfetch provides an http.RoundTripper implementation // for fetching URLs via App Engine's urlfetch service. package urlfetch // import "google.golang.org/appengine/urlfetch" import ( "errors" "fmt" "io" "io/ioutil" "net/http" "net/url" "strconv" "strings" "time" "github.com/golang/protobuf/proto" "golang.org/x/net/context" "google.golang.org/appengine/internal" pb "google.golang.org/appengine/internal/urlfetch" ) // Transport is an implementation of http.RoundTripper for // App Engine. Users should generally create an http.Client using // this transport and use the Client rather than using this transport // directly. type Transport struct { Context context.Context // Controls whether the application checks the validity of SSL certificates // over HTTPS connections. A value of false (the default) instructs the // application to send a request to the server only if the certificate is // valid and signed by a trusted certificate authority (CA), and also // includes a hostname that matches the certificate. A value of true // instructs the application to perform no certificate validation. AllowInvalidServerCertificate bool } // Verify statically that *Transport implements http.RoundTripper. var _ http.RoundTripper = (*Transport)(nil) // Client returns an *http.Client using a default urlfetch Transport. This // client will have the default deadline of 5 seconds, and will check the // validity of SSL certificates. // // Any deadline of the provided context will be used for requests through this client; // if the client does not have a deadline then a 5 second default is used. func Client(ctx context.Context) *http.Client { return &http.Client{ Transport: &Transport{ Context: ctx, }, } } type bodyReader struct { content []byte truncated bool closed bool } // ErrTruncatedBody is the error returned after the final Read() from a // response's Body if the body has been truncated by App Engine's proxy. var ErrTruncatedBody = errors.New("urlfetch: truncated body") func statusCodeToText(code int) string { if t := http.StatusText(code); t != "" { return t } return strconv.Itoa(code) } func (br *bodyReader) Read(p []byte) (n int, err error) { if br.closed { if br.truncated { return 0, ErrTruncatedBody } return 0, io.EOF } n = copy(p, br.content) if n > 0 { br.content = br.content[n:] return } if br.truncated { br.closed = true return 0, ErrTruncatedBody } return 0, io.EOF } func (br *bodyReader) Close() error { br.closed = true br.content = nil return nil } // A map of the URL Fetch-accepted methods that take a request body. var methodAcceptsRequestBody = map[string]bool{ "POST": true, "PUT": true, "PATCH": true, } // urlString returns a valid string given a URL. This function is necessary because // the String method of URL doesn't correctly handle URLs with non-empty Opaque values. // See http://code.google.com/p/go/issues/detail?id=4860. func urlString(u *url.URL) string { if u.Opaque == "" || strings.HasPrefix(u.Opaque, "//") { return u.String() } aux := *u aux.Opaque = "//" + aux.Host + aux.Opaque return aux.String() } // RoundTrip issues a single HTTP request and returns its response. Per the // http.RoundTripper interface, RoundTrip only returns an error if there // was an unsupported request or the URL Fetch proxy fails. // Note that HTTP response codes such as 5xx, 403, 404, etc are not // errors as far as the transport is concerned and will be returned // with err set to nil. func (t *Transport) RoundTrip(req *http.Request) (res *http.Response, err error) { methNum, ok := pb.URLFetchRequest_RequestMethod_value[req.Method] if !ok { return nil, fmt.Errorf("urlfetch: unsupported HTTP method %q", req.Method) } method := pb.URLFetchRequest_RequestMethod(methNum) freq := &pb.URLFetchRequest{ Method: &method, Url: proto.String(urlString(req.URL)), FollowRedirects: proto.Bool(false), // http.Client's responsibility MustValidateServerCertificate: proto.Bool(!t.AllowInvalidServerCertificate), } if deadline, ok := t.Context.Deadline(); ok { freq.Deadline = proto.Float64(deadline.Sub(time.Now()).Seconds()) } for k, vals := range req.Header { for _, val := range vals { freq.Header = append(freq.Header, &pb.URLFetchRequest_Header{ Key: proto.String(k), Value: proto.String(val), }) } } if methodAcceptsRequestBody[req.Method] && req.Body != nil { // Avoid a []byte copy if req.Body has a Bytes method. switch b := req.Body.(type) { case interface { Bytes() []byte }: freq.Payload = b.Bytes() default: freq.Payload, err = ioutil.ReadAll(req.Body) if err != nil { return nil, err } } } fres := &pb.URLFetchResponse{} if err := internal.Call(t.Context, "urlfetch", "Fetch", freq, fres); err != nil { return nil, err } res = &http.Response{} res.StatusCode = int(*fres.StatusCode) res.Status = fmt.Sprintf("%d %s", res.StatusCode, statusCodeToText(res.StatusCode)) res.Header = make(http.Header) res.Request = req // Faked: res.ProtoMajor = 1 res.ProtoMinor = 1 res.Proto = "HTTP/1.1" res.Close = true for _, h := range fres.Header { hkey := http.CanonicalHeaderKey(*h.Key) hval := *h.Value if hkey == "Content-Length" { // Will get filled in below for all but HEAD requests. if req.Method == "HEAD" { res.ContentLength, _ = strconv.ParseInt(hval, 10, 64) } continue } res.Header.Add(hkey, hval) } if req.Method != "HEAD" { res.ContentLength = int64(len(fres.Content)) } truncated := fres.GetContentWasTruncated() res.Body = &bodyReader{content: fres.Content, truncated: truncated} return } func init() { internal.RegisterErrorCodeMap("urlfetch", pb.URLFetchServiceError_ErrorCode_name) internal.RegisterTimeoutErrorCode("urlfetch", int32(pb.URLFetchServiceError_DEADLINE_EXCEEDED)) }