net/http Client

  1// src/net/http/client.go#L29
  2// A Client is an HTTP client. Its zero value (DefaultClient) is a
  3// usable client that uses DefaultTransport.
  4//
  5// The Client's Transport typically has internal state (cached TCP
  6// connections), so Clients should be reused instead of created as
  7// needed. Clients are safe for concurrent use by multiple goroutines.
  8//
  9// A Client is higher-level than a RoundTripper (such as Transport)
 10// and additionally handles HTTP details such as cookies and
 11// redirects.
 12//
 13// When following redirects, the Client will forward all headers set on the
 14// initial Request except:
 15//
 16// • when forwarding sensitive headers like "Authorization",
 17// "WWW-Authenticate", and "Cookie" to untrusted targets.
 18// These headers will be ignored when following a redirect to a domain
 19// that is not a subdomain match or exact match of the initial domain.
 20// For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com"
 21// will forward the sensitive headers, but a redirect to "bar.com" will not.
 22//
 23// • when forwarding the "Cookie" header with a non-nil cookie Jar.
 24// Since each redirect may mutate the state of the cookie jar,
 25// a redirect may possibly alter a cookie set in the initial request.
 26// When forwarding the "Cookie" header, any mutated cookies will be omitted,
 27// with the expectation that the Jar will insert those mutated cookies
 28// with the updated values (assuming the origin matches).
 29// If Jar is nil, the initial cookies are forwarded without change.
 30type Client struct {
 31	// Transport specifies the mechanism by which individual
 32	// HTTP requests are made.
 33	// If nil, DefaultTransport is used.
 34	Transport RoundTripper
 35
 36	// CheckRedirect specifies the policy for handling redirects.
 37	// If CheckRedirect is not nil, the client calls it before
 38	// following an HTTP redirect. The arguments req and via are
 39	// the upcoming request and the requests made already, oldest
 40	// first. If CheckRedirect returns an error, the Client's Get
 41	// method returns both the previous Response (with its Body
 42	// closed) and CheckRedirect's error (wrapped in a url.Error)
 43	// instead of issuing the Request req.
 44	// As a special case, if CheckRedirect returns ErrUseLastResponse,
 45	// then the most recent response is returned with its body
 46	// unclosed, along with a nil error.
 47	//
 48	// If CheckRedirect is nil, the Client uses its default policy,
 49	// which is to stop after 10 consecutive requests.
 50	CheckRedirect func(req *Request, via []*Request) error
 51
 52	// Jar specifies the cookie jar.
 53	//
 54	// The Jar is used to insert relevant cookies into every
 55	// outbound Request and is updated with the cookie values
 56	// of every inbound Response. The Jar is consulted for every
 57	// redirect that the Client follows.
 58	//
 59	// If Jar is nil, cookies are only sent if they are explicitly
 60	// set on the Request.
 61	Jar CookieJar
 62
 63	// Timeout specifies a time limit for requests made by this
 64	// Client. The timeout includes connection time, any
 65	// redirects, and reading the response body. The timer remains
 66	// running after Get, Head, Post, or Do return and will
 67	// interrupt reading of the Response.Body.
 68	//
 69	// A Timeout of zero means no timeout.
 70	//
 71	// The Client cancels requests to the underlying Transport
 72	// as if the Request's Context ended.
 73	//
 74	// For compatibility, the Client will also use the deprecated
 75	// CancelRequest method on Transport if found. New
 76	// RoundTripper implementations should use the Request's Context
 77	// for cancellation instead of implementing CancelRequest.
 78	Timeout time.Duration
 79}
 80
 81// DefaultClient is the default Client and is used by Get, Head, and Post.
 82var DefaultClient = &Client{}
 83
 84// RoundTripper is an interface representing the ability to execute a
 85// single HTTP transaction, obtaining the Response for a given Request.
 86//
 87// A RoundTripper must be safe for concurrent use by multiple
 88// goroutines.
 89type RoundTripper interface {
 90	// RoundTrip executes a single HTTP transaction, returning
 91	// a Response for the provided Request.
 92	//
 93	// RoundTrip should not attempt to interpret the response. In
 94	// particular, RoundTrip must return err == nil if it obtained
 95	// a response, regardless of the response's HTTP status code.
 96	// A non-nil err should be reserved for failure to obtain a
 97	// response. Similarly, RoundTrip should not attempt to
 98	// handle higher-level protocol details such as redirects,
 99	// authentication, or cookies.
100	//
101	// RoundTrip should not modify the request, except for
102	// consuming and closing the Request's Body. RoundTrip may
103	// read fields of the request in a separate goroutine. Callers
104	// should not mutate or reuse the request until the Response's
105	// Body has been closed.
106	//
107	// RoundTrip must always close the body, including on errors,
108	// but depending on the implementation may do so in a separate
109	// goroutine even after RoundTrip returns. This means that
110	// callers wanting to reuse the body for subsequent requests
111	// must arrange to wait for the Close call before doing so.
112	//
113	// The Request's URL and Header fields must be initialized.
114	RoundTrip(*Request) (*Response, error)
115}

  1// src/net/http/request.go#L837
  2// NewRequest wraps NewRequestWithContext using context.Background.
  3func NewRequest(method, url string, body io.Reader) (*Request, error) {
  4	return NewRequestWithContext(context.Background(), method, url, body)
  5}
  6
  7// NewRequestWithContext returns a new Request given a method, URL, and
  8// optional body.
  9//
 10// If the provided body is also an io.Closer, the returned
 11// Request.Body is set to body and will be closed by the Client
 12// methods Do, Post, and PostForm, and Transport.RoundTrip.
 13//
 14// NewRequestWithContext returns a Request suitable for use with
 15// Client.Do or Transport.RoundTrip. To create a request for use with
 16// testing a Server Handler, either use the NewRequest function in the
 17// net/http/httptest package, use ReadRequest, or manually update the
 18// Request fields. For an outgoing client request, the context
 19// controls the entire lifetime of a request and its response:
 20// obtaining a connection, sending the request, and reading the
 21// response headers and body. See the Request type's documentation for
 22// the difference between inbound and outbound request fields.
 23//
 24// If body is of type *bytes.Buffer, *bytes.Reader, or
 25// *strings.Reader, the returned request's ContentLength is set to its
 26// exact value (instead of -1), GetBody is populated (so 307 and 308
 27// redirects can replay the body), and Body is set to NoBody if the
 28// ContentLength is 0.
 29func NewRequestWithContext(ctx context.Context, method, url string, body io.Reader) (*Request, error) {
 30	if method == "" {
 31		// We document that "" means "GET" for Request.Method, and people have
 32		// relied on that from NewRequest, so keep that working.
 33		// We still enforce validMethod for non-empty methods.
 34		method = "GET"
 35	}
 36	if !validMethod(method) {
 37		return nil, fmt.Errorf("net/http: invalid method %q", method)
 38	}
 39	if ctx == nil {
 40		return nil, errors.New("net/http: nil Context")
 41	}
 42	u, err := urlpkg.Parse(url)
 43	if err != nil {
 44		return nil, err
 45	}
 46	rc, ok := body.(io.ReadCloser)
 47	if !ok && body != nil {
 48		rc = io.NopCloser(body)
 49	}
 50	// The host's colon:port should be normalized. See Issue 14836.
 51	u.Host = removeEmptyPort(u.Host)
 52	req := &Request{
 53		ctx:        ctx,
 54		Method:     method,
 55		URL:        u,
 56		Proto:      "HTTP/1.1",
 57		ProtoMajor: 1,
 58		ProtoMinor: 1,
 59		Header:     make(Header),
 60		Body:       rc,
 61		Host:       u.Host,
 62	}
 63	if body != nil {
 64		switch v := body.(type) {
 65		case *bytes.Buffer:
 66			req.ContentLength = int64(v.Len())
 67			buf := v.Bytes()
 68			req.GetBody = func() (io.ReadCloser, error) {
 69				r := bytes.NewReader(buf)
 70				return io.NopCloser(r), nil
 71			}
 72		case *bytes.Reader:
 73			req.ContentLength = int64(v.Len())
 74			snapshot := *v
 75			req.GetBody = func() (io.ReadCloser, error) {
 76				r := snapshot
 77				return io.NopCloser(&r), nil
 78			}
 79		case *strings.Reader:
 80			req.ContentLength = int64(v.Len())
 81			snapshot := *v
 82			req.GetBody = func() (io.ReadCloser, error) {
 83				r := snapshot
 84				return io.NopCloser(&r), nil
 85			}
 86		default:
 87			// This is where we'd set it to -1 (at least
 88			// if body != NoBody) to mean unknown, but
 89			// that broke people during the Go 1.8 testing
 90			// period. People depend on it being 0 I
 91			// guess. Maybe retry later. See Issue 18117.
 92		}
 93		// For client requests, Request.ContentLength of 0
 94		// means either actually 0, or unknown. The only way
 95		// to explicitly say that the ContentLength is zero is
 96		// to set the Body to nil. But turns out too much code
 97		// depends on NewRequest returning a non-nil Body,
 98		// so we use a well-known ReadCloser variable instead
 99		// and have the http package also treat that sentinel
100		// variable to mean explicitly zero.
101		if req.GetBody != nil && req.ContentLength == 0 {
102			req.Body = NoBody
103			req.GetBody = func() (io.ReadCloser, error) { return NoBody, nil }
104		}
105	}
106
107	return req, nil
108}

  1// src/net/http/client.go#L544
  2// Do sends an HTTP request and returns an HTTP response, following
  3// policy (such as redirects, cookies, auth) as configured on the
  4// client.
  5//
  6// An error is returned if caused by client policy (such as
  7// CheckRedirect), or failure to speak HTTP (such as a network
  8// connectivity problem). A non-2xx status code doesn't cause an
  9// error.
 10//
 11// If the returned error is nil, the Response will contain a non-nil
 12// Body which the user is expected to close. If the Body is not both
 13// read to EOF and closed, the Client's underlying RoundTripper
 14// (typically Transport) may not be able to re-use a persistent TCP
 15// connection to the server for a subsequent "keep-alive" request.
 16//
 17// The request Body, if non-nil, will be closed by the underlying
 18// Transport, even on errors.
 19//
 20// On error, any Response can be ignored. A non-nil Response with a
 21// non-nil error only occurs when CheckRedirect fails, and even then
 22// the returned Response.Body is already closed.
 23//
 24// Generally Get, Post, or PostForm will be used instead of Do.
 25//
 26// If the server replies with a redirect, the Client first uses the
 27// CheckRedirect function to determine whether the redirect should be
 28// followed. If permitted, a 301, 302, or 303 redirect causes
 29// subsequent requests to use HTTP method GET
 30// (or HEAD if the original request was HEAD), with no body.
 31// A 307 or 308 redirect preserves the original HTTP method and body,
 32// provided that the Request.GetBody function is defined.
 33// The NewRequest function automatically sets GetBody for common
 34// standard library body types.
 35//
 36// Any returned error will be of type *url.Error. The url.Error
 37// value's Timeout method will report true if the request timed out.
 38func (c *Client) Do(req *Request) (*Response, error) {
 39	return c.do(req)
 40}
 41
 42var testHookClientDoResult func(retres *Response, reterr error)
 43
 44func (c *Client) do(req *Request) (retres *Response, reterr error) {
 45	if testHookClientDoResult != nil {
 46		defer func() { testHookClientDoResult(retres, reterr) }()
 47	}
 48	if req.URL == nil {
 49		req.closeBody()
 50		return nil, &url.Error{
 51			Op:  urlErrorOp(req.Method),
 52			Err: errors.New("http: nil Request.URL"),
 53		}
 54	}
 55
 56	var (
 57		deadline      = c.deadline()
 58		reqs          []*Request
 59		resp          *Response
 60		copyHeaders   = c.makeHeadersCopier(req)
 61		reqBodyClosed = false // have we closed the current req.Body?
 62
 63		// Redirect behavior:
 64		redirectMethod string
 65		includeBody    bool
 66	)
 67	uerr := func(err error) error {
 68		// the body may have been closed already by c.send()
 69		if !reqBodyClosed {
 70			req.closeBody()
 71		}
 72		var urlStr string
 73		if resp != nil && resp.Request != nil {
 74			urlStr = stripPassword(resp.Request.URL)
 75		} else {
 76			urlStr = stripPassword(req.URL)
 77		}
 78		return &url.Error{
 79			Op:  urlErrorOp(reqs[0].Method),
 80			URL: urlStr,
 81			Err: err,
 82		}
 83	}
 84	for {
 85		// For all but the first request, create the next
 86		// request hop and replace req.
 87		if len(reqs) > 0 {
 88			loc := resp.Header.Get("Location")
 89			if loc == "" {
 90				// While most 3xx responses include a Location, it is not
 91				// required and 3xx responses without a Location have been
 92				// observed in the wild. See issues #17773 and #49281.
 93				return resp, nil
 94			}
 95			u, err := req.URL.Parse(loc)
 96			if err != nil {
 97				resp.closeBody()
 98				return nil, uerr(fmt.Errorf("failed to parse Location header %q: %v", loc, err))
 99			}
100			host := ""
101			if req.Host != "" && req.Host != req.URL.Host {
102				// If the caller specified a custom Host header and the
103				// redirect location is relative, preserve the Host header
104				// through the redirect. See issue #22233.
105				if u, _ := url.Parse(loc); u != nil && !u.IsAbs() {
106					host = req.Host
107				}
108			}
109			ireq := reqs[0]
110			req = &Request{
111				Method:   redirectMethod,
112				Response: resp,
113				URL:      u,
114				Header:   make(Header),
115				Host:     host,
116				Cancel:   ireq.Cancel,
117				ctx:      ireq.ctx,
118			}
119			if includeBody && ireq.GetBody != nil {
120				req.Body, err = ireq.GetBody()
121				if err != nil {
122					resp.closeBody()
123					return nil, uerr(err)
124				}
125				req.ContentLength = ireq.ContentLength
126			}
127
128			// Copy original headers before setting the Referer,
129			// in case the user set Referer on their first request.
130			// If they really want to override, they can do it in
131			// their CheckRedirect func.
132			copyHeaders(req)
133
134			// Add the Referer header from the most recent
135			// request URL to the new one, if it's not https->http:
136			if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL); ref != "" {
137				req.Header.Set("Referer", ref)
138			}
139			err = c.checkRedirect(req, reqs)
140
141			// Sentinel error to let users select the
142			// previous response, without closing its
143			// body. See Issue 10069.
144			if err == ErrUseLastResponse {
145				return resp, nil
146			}
147
148			// Close the previous response's body. But
149			// read at least some of the body so if it's
150			// small the underlying TCP connection will be
151			// re-used. No need to check for errors: if it
152			// fails, the Transport won't reuse it anyway.
153			const maxBodySlurpSize = 2 << 10
154			if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize {
155				io.CopyN(io.Discard, resp.Body, maxBodySlurpSize)
156			}
157			resp.Body.Close()
158
159			if err != nil {
160				// Special case for Go 1 compatibility: return both the response
161				// and an error if the CheckRedirect function failed.
162				// See https://golang.org/issue/3795
163				// The resp.Body has already been closed.
164				ue := uerr(err)
165				ue.(*url.Error).URL = loc
166				return resp, ue
167			}
168		}
169
170		reqs = append(reqs, req)
171		var err error
172		var didTimeout func() bool
173		if resp, didTimeout, err = c.send(req, deadline); err != nil {
174			// c.send() always closes req.Body
175			reqBodyClosed = true
176			if !deadline.IsZero() && didTimeout() {
177				err = &httpError{
178					err:     err.Error() + " (Client.Timeout exceeded while awaiting headers)",
179					timeout: true,
180				}
181			}
182			return nil, uerr(err)
183		}
184
185		var shouldRedirect bool
186		redirectMethod, shouldRedirect, includeBody = redirectBehavior(req.Method, resp, reqs[0])
187		if !shouldRedirect {
188			return resp, nil
189		}
190
191		req.closeBody()
192	}
193}

  1// src/net/http/client.go#L168
  2// didTimeout is non-nil only if err != nil.
  3func (c *Client) send(req *Request, deadline time.Time) (resp *Response, didTimeout func() bool, err error) {
  4	if c.Jar != nil {
  5		for _, cookie := range c.Jar.Cookies(req.URL) {
  6			req.AddCookie(cookie)
  7		}
  8	}
  9	resp, didTimeout, err = send(req, c.transport(), deadline)
 10	if err != nil {
 11		return nil, didTimeout, err
 12	}
 13	if c.Jar != nil {
 14		if rc := resp.Cookies(); len(rc) > 0 {
 15			c.Jar.SetCookies(req.URL, rc)
 16		}
 17	}
 18	return resp, nil, nil
 19}
 20
 21func (c *Client) deadline() time.Time {
 22	if c.Timeout > 0 {
 23		return time.Now().Add(c.Timeout)
 24	}
 25	return time.Time{}
 26}
 27
 28func (c *Client) transport() RoundTripper {
 29	if c.Transport != nil {
 30		return c.Transport
 31	}
 32	return DefaultTransport
 33}
 34
 35// send issues an HTTP request.
 36// Caller should close resp.Body when done reading from it.
 37func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, didTimeout func() bool, err error) {
 38	req := ireq // req is either the original request, or a modified fork
 39
 40	if rt == nil {
 41		req.closeBody()
 42		return nil, alwaysFalse, errors.New("http: no Client.Transport or DefaultTransport")
 43	}
 44
 45	if req.URL == nil {
 46		req.closeBody()
 47		return nil, alwaysFalse, errors.New("http: nil Request.URL")
 48	}
 49
 50	if req.RequestURI != "" {
 51		req.closeBody()
 52		return nil, alwaysFalse, errors.New("http: Request.RequestURI can't be set in client requests")
 53	}
 54
 55	// forkReq forks req into a shallow clone of ireq the first
 56	// time it's called.
 57	forkReq := func() {
 58		if ireq == req {
 59			req = new(Request)
 60			*req = *ireq // shallow clone
 61		}
 62	}
 63
 64	// Most the callers of send (Get, Post, et al) don't need
 65	// Headers, leaving it uninitialized. We guarantee to the
 66	// Transport that this has been initialized, though.
 67	if req.Header == nil {
 68		forkReq()
 69		req.Header = make(Header)
 70	}
 71
 72	if u := req.URL.User; u != nil && req.Header.Get("Authorization") == "" {
 73		username := u.Username()
 74		password, _ := u.Password()
 75		forkReq()
 76		req.Header = cloneOrMakeHeader(ireq.Header)
 77		req.Header.Set("Authorization", "Basic "+basicAuth(username, password))
 78	}
 79
 80	if !deadline.IsZero() {
 81		forkReq()
 82	}
 83	stopTimer, didTimeout := setRequestCancel(req, rt, deadline)
 84
 85	resp, err = rt.RoundTrip(req)
 86	if err != nil {
 87		stopTimer()
 88		if resp != nil {
 89			log.Printf("RoundTripper returned a response & error; ignoring response")
 90		}
 91		if tlsErr, ok := err.(tls.RecordHeaderError); ok {
 92			// If we get a bad TLS record header, check to see if the
 93			// response looks like HTTP and give a more helpful error.
 94			// See golang.org/issue/11111.
 95			if string(tlsErr.RecordHeader[:]) == "HTTP/" {
 96				err = errors.New("http: server gave HTTP response to HTTPS client")
 97			}
 98		}
 99		return nil, didTimeout, err
100	}
101	if resp == nil {
102		return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a nil *Response with a nil error", rt)
103	}
104	if resp.Body == nil {
105		// The documentation on the Body field says “The http Client and Transport
106		// guarantee that Body is always non-nil, even on responses without a body
107		// or responses with a zero-length body.” Unfortunately, we didn't document
108		// that same constraint for arbitrary RoundTripper implementations, and
109		// RoundTripper implementations in the wild (mostly in tests) assume that
110		// they can use a nil Body to mean an empty one (similar to Request.Body).
111		// (See https://golang.org/issue/38095.)
112		//
113		// If the ContentLength allows the Body to be empty, fill in an empty one
114		// here to ensure that it is non-nil.
115		if resp.ContentLength > 0 && req.Method != "HEAD" {
116			return nil, didTimeout, fmt.Errorf("http: RoundTripper implementation (%T) returned a *Response with content length %d but a nil Body", rt, resp.ContentLength)
117		}
118		resp.Body = io.NopCloser(strings.NewReader(""))
119	}
120	if !deadline.IsZero() {
121		resp.Body = &cancelTimerBody{
122			stop:          stopTimer,
123			rc:            resp.Body,
124			reqDidTimeout: didTimeout,
125		}
126	}
127	return resp, nil, nil
128}