diff --git a/client.go b/client.go
index a9b83fb..035c565 100644
--- a/client.go
+++ b/client.go
@@ -1178,7 +1178,9 @@ func (c *HostClient) doNonNilReqResp(req *Request, resp *Response) (bool, error)
 	if err = resp.ReadLimitBody(br, c.MaxResponseBodySize); err != nil {
 		c.releaseReader(br)
 		c.closeConn(cc)
-		return true, err
+		// Don't retry in case of ErrBodyTooLarge since we will just get the same again.
+		retry := err != ErrBodyTooLarge
+		return retry, err
 	}
 	c.releaseReader(br)