diff --git a/header.go b/header.go index f9c6c1f..40dc2d6 100644 --- a/header.go +++ b/header.go @@ -20,7 +20,9 @@ type ResponseHeader struct { // Response content length read from Content-Length header. // - // It may be negative on chunked response. + // It may be negative: + // -1 means Transfer-Encoding: chunked. + // -2 means Transfer-Encoding: identity. ContentLength int // Set to true if response contains 'Connection: close' header. @@ -48,7 +50,8 @@ type RequestHeader struct { // Request content length read from Content-Length header. // - // It may be negative on chunked request. + // It may be negative: + // -1 means Transfer-Encoding: chunked. ContentLength int // Set to true if request contains 'Connection: close' header. @@ -838,6 +841,7 @@ func (h *RequestHeader) parseFirstLine(buf []byte) (b []byte, err error) { } func (h *ResponseHeader) parseHeaders(buf []byte) ([]byte, error) { + // 'identity' content-length by default h.ContentLength = -2 var s headerScanner @@ -861,7 +865,7 @@ func (h *ResponseHeader) parseHeaders(buf []byte) ([]byte, error) { } } case bytes.Equal(s.key, strTransferEncoding): - if bytes.Equal(s.value, strChunked) { + if !bytes.Equal(s.value, strIdentity) { h.ContentLength = -1 } case bytes.Equal(s.key, strConnection): @@ -886,7 +890,8 @@ func (h *ResponseHeader) parseHeaders(buf []byte) ([]byte, error) { return nil, fmt.Errorf("missing required Content-Type header in %q", buf) } if h.ContentLength == -2 { - return nil, fmt.Errorf("missing both Content-Length and Transfer-Encoding: chunked in %q", buf) + // Close connection after 'identity' response. + h.ConnectionClose = true } return s.b, nil } @@ -917,7 +922,7 @@ func (h *RequestHeader) parseHeaders(buf []byte) ([]byte, error) { } } case bytes.Equal(s.key, strTransferEncoding): - if bytes.Equal(s.value, strChunked) { + if !bytes.Equal(s.value, strIdentity) { h.ContentLength = -1 } case bytes.Equal(s.key, strConnection): diff --git a/header_test.go b/header_test.go index df67e84..727550a 100644 --- a/header_test.go +++ b/header_test.go @@ -747,6 +747,16 @@ func TestResponseHeaderReadSuccess(t *testing.T) { // blank lines before the first line testResponseHeaderReadSuccess(t, h, "\r\nHTTP/1.1 200 OK\r\nContent-Type: aa\r\nContent-Length: 0\r\n\r\nsss", 200, 0, "aa", "sss") + if h.ConnectionClose { + t.Fatalf("unexpected connection: close") + } + + // no content-length (identity transfer-encoding) + testResponseHeaderReadSuccess(t, h, "HTTP/1.1 200 OK\r\nContent-Type: foo/bar\r\n\r\nabcdefg", + 200, -2, "foo/bar", "abcdefg") + if !h.ConnectionClose { + t.Fatalf("expecting connection: close for identity response") + } } func TestRequestHeaderReadSuccess(t *testing.T) { @@ -882,9 +892,6 @@ func TestResponseHeaderReadError(t *testing.T) { // no content-type testResponseHeaderReadError(t, h, "HTTP/1.1 200 OK\r\nContent-Length: 123\r\n\r\n") - - // no content-length - testResponseHeaderReadError(t, h, "HTTP/1.1 200 OK\r\nContent-Type: foo/bar\r\n\r\n") } func TestRequestHeaderReadError(t *testing.T) { diff --git a/http.go b/http.go index eeba836..67917d5 100644 --- a/http.go +++ b/http.go @@ -148,12 +148,11 @@ func (req *Request) Read(r *bufio.Reader) error { } if req.Header.IsMethodPost() { - body, err := readBody(r, req.Header.ContentLength, req.Body) + req.Body, err = readBody(r, req.Header.ContentLength, req.Body) if err != nil { req.Clear() return err } - req.Body = body req.Header.ContentLength = len(req.Body) } return nil @@ -168,12 +167,11 @@ func (resp *Response) Read(r *bufio.Reader) error { } if !isSkipResponseBody(resp.Header.StatusCode) && !resp.SkipBody { - body, err := readBody(r, resp.Header.ContentLength, resp.Body) + resp.Body, err = readBody(r, resp.Header.ContentLength, resp.Body) if err != nil { resp.Clear() return err } - resp.Body = body resp.Header.ContentLength = len(resp.Body) } return nil @@ -314,53 +312,77 @@ func writeChunk(w *bufio.Writer, b []byte) error { var copyBufPool sync.Pool -func readBody(r *bufio.Reader, contentLength int, b []byte) ([]byte, error) { - b = b[:0] +func readBody(r *bufio.Reader, contentLength int, dst []byte) ([]byte, error) { + dst = dst[:0] if contentLength >= 0 { - return readBodyFixedSize(r, contentLength, b) + return appendBodyFixedSize(r, dst, contentLength) } - return readBodyChunked(r, b) + if contentLength == -1 { + return readBodyChunked(r, dst) + } + return readBodyIdentity(r, dst) } -func readBodyFixedSize(r *bufio.Reader, n int, buf []byte) ([]byte, error) { +func readBodyIdentity(r *bufio.Reader, dst []byte) ([]byte, error) { + dst = dst[:cap(dst)] + if len(dst) == 0 { + dst = make([]byte, 1024) + } + offset := 0 + for { + nn, err := r.Read(dst[offset:]) + if nn <= 0 { + if err != nil { + if err == io.EOF { + return dst[:offset], nil + } + return dst[:offset], err + } + panic(fmt.Sprintf("BUG: bufio.Read() returned (%d, nil)", nn)) + } + offset += nn + if len(dst) == offset { + b := make([]byte, round2(2*offset)) + copy(b, dst) + dst = b + } + } +} + +func appendBodyFixedSize(r *bufio.Reader, dst []byte, n int) ([]byte, error) { if n == 0 { - return buf, nil + return dst, nil } - bufLen := len(buf) - bufCap := bufLen + n - if cap(buf) < bufCap { - b := make([]byte, bufLen, round2(bufCap)) - copy(b, buf) - buf = b + offset := len(dst) + dstLen := offset + n + if cap(dst) < dstLen { + b := make([]byte, round2(dstLen)) + copy(b, dst) + dst = b } - buf = buf[:bufCap] - b := buf[bufLen:] + dst = dst[:dstLen] for { - nn, err := r.Read(b) + nn, err := r.Read(dst[offset:]) if nn <= 0 { if err != nil { if err == io.EOF { err = io.ErrUnexpectedEOF } - return nil, err + return dst[:offset], err } - panic(fmt.Sprintf("BUF: bufio.Read() returned (%d, nil)", nn)) + panic(fmt.Sprintf("BUG: bufio.Read() returned (%d, nil)", nn)) } - if nn == n { - return buf, nil + offset += nn + if offset == dstLen { + return dst, nil } - if nn > n { - panic(fmt.Sprintf("BUF: read more than requested: %d vs %d", nn, n)) - } - n -= nn - b = b[nn:] } } -func readBodyChunked(r *bufio.Reader, b []byte) ([]byte, error) { - if len(b) > 0 { +func readBodyChunked(r *bufio.Reader, dst []byte) ([]byte, error) { + if len(dst) > 0 { panic("BUG: expected zero-length buffer") } @@ -368,18 +390,18 @@ func readBodyChunked(r *bufio.Reader, b []byte) ([]byte, error) { for { chunkSize, err := parseChunkSize(r) if err != nil { - return nil, err + return dst, err } - b, err = readBodyFixedSize(r, chunkSize+strCRLFLen, b) + dst, err = appendBodyFixedSize(r, dst, chunkSize+strCRLFLen) if err != nil { - return nil, err + return dst, err } - if !bytes.Equal(b[len(b)-strCRLFLen:], strCRLF) { - return nil, fmt.Errorf("cannot find crlf at the end of chunk") + if !bytes.Equal(dst[len(dst)-strCRLFLen:], strCRLF) { + return dst, fmt.Errorf("cannot find crlf at the end of chunk") } - b = b[:len(b)-strCRLFLen] + dst = dst[:len(dst)-strCRLFLen] if chunkSize == 0 { - return b, nil + return dst, nil } } } diff --git a/http_test.go b/http_test.go index ff69689..39fbe60 100644 --- a/http_test.go +++ b/http_test.go @@ -312,10 +312,27 @@ func TestResponseReadSuccess(t *testing.T) { testResponseReadSuccess(t, resp, "HTTP/1.1 300 OK\r\nContent-Length: 5\r\nContent-Type: bar\r\n\r\n56789aaa", 300, 5, "bar", "56789", "aaa") + // no conent-length ('identity' transfer-encoding) + testResponseReadSuccess(t, resp, "HTTP/1.1 200 OK\r\nContent-Type: foobar\r\n\r\nzxxc", + 200, 4, "foobar", "zxxc", "") + + // explicitly stated 'Transfer-Encoding: identity' + testResponseReadSuccess(t, resp, "HTTP/1.1 234 ss\r\nContent-Type: xxx\r\n\r\nxag", + 234, 3, "xxx", "xag", "") + + // big 'identity' response + body := string(createFixedBody(100500)) + testResponseReadSuccess(t, resp, "HTTP/1.1 200 OK\r\nContent-Type: aa\r\n\r\n"+body, + 200, 100500, "aa", body, "") + // chunked response testResponseReadSuccess(t, resp, "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nqwer\r\n2\r\nty\r\n0\r\n\r\nzzzzz", 200, 6, "text/html", "qwerty", "zzzzz") + // chunked response with non-chunked Transfer-Encoding. + testResponseReadSuccess(t, resp, "HTTP/1.1 230 OK\r\nContent-Type: text\r\nTransfer-Encoding: aaabbb\r\n\r\n2\r\ner\r\n2\r\nty\r\n0\r\n\r\nwe", + 230, 4, "text", "erty", "we") + // zero chunked response testResponseReadSuccess(t, resp, "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\nzzz", 200, 0, "text/html", "", "zzz") diff --git a/strings.go b/strings.go index 625c574..bf8f10e 100644 --- a/strings.go +++ b/strings.go @@ -40,5 +40,6 @@ var ( strClose = []byte("close") strChunked = []byte("chunked") + strIdentity = []byte("identity") strPostArgsContentType = []byte("application/x-www-form-urlencoded") )