contrib/fasthttp
1
0
Fork 0
mirror of https://github.com/valyala/fasthttp.git synced 2026-06-25 17:45:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Brotli support in FS handler. (#880)

Browse Source 
* Add files via upload

* Update fs.go

* Add files via upload

* Update fs_test.go
This commit is contained in:
hex0x00
2020-09-28 10:14:28 -05:00
committed by GitHub
parent ae8b65fa62
commit 805af0ee73
2 changed files with 216 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fs.go
+149 -72
View File
@@ -16,6 +16,7 @@ import (
"sync"
"time"
"github.com/andybalholm/brotli"
"github.com/klauspost/compress/gzip"
"github.com/valyala/bytebufferpool"
)
@@ -105,6 +106,7 @@ var (
Root: "/",
GenerateIndexPages: true,
Compress: true,
CompressBrotli: true,
AcceptByteRange: true,
}
rootFSHandler RequestHandler
@@ -236,6 +238,13 @@ type FS struct {
// Transparent compression is disabled by default.
Compress bool
// Uses brotli encoding and fallbacks to gzip in responses if set to true, uses gzip if set to false.
//
// This value has sense only if Compress is set.
//
// Brotli encoding is disabled by default.
CompressBrotli bool
// Enables byte range requests if set to true.
//
// Byte range requests are disabled by default.
@@ -266,6 +275,13 @@ type FS struct {
// FSCompressedFileSuffix is used by default.
CompressedFileSuffix string
// Suffixes list to add to compressedFileSuffix depending on encoding
//
// This value has sense only if Compress is set.
//
// FSCompressedFileSuffixes is used by default.
CompressedFileSuffixes map[string]string
once sync.Once
h RequestHandler
}
@@ -275,6 +291,14 @@ type FS struct {
// See FS.Compress for details.
const FSCompressedFileSuffix = ".fasthttp.gz"
// FSCompressedFileSuffixes is the suffixes FS adds to the original file names depending on encoding
// when trying to store compressed file under the new file name.
// See FS.Compress for details.
var FSCompressedFileSuffixes = map[string]string{
"gzip": ".fasthttp.gz",
"br": ".fasthttp.br",
}
// FSHandlerCacheDuration is the default expiration duration for inactive
// file handlers opened by FS.
const FSHandlerCacheDuration = 10 * time.Second
@@ -345,23 +369,32 @@ func (fs *FS) initRequestHandler() {
if cacheDuration <= 0 {
cacheDuration = FSHandlerCacheDuration
}
compressedFileSuffix := fs.CompressedFileSuffix
if len(compressedFileSuffix) == 0 {
compressedFileSuffix = FSCompressedFileSuffix
compressedFileSuffixes := fs.CompressedFileSuffixes
if len(compressedFileSuffixes["br"]) == 0 || len(compressedFileSuffixes["gzip"]) == 0 ||
compressedFileSuffixes["br"] == compressedFileSuffixes["gzip"] {
compressedFileSuffixes = FSCompressedFileSuffixes
}
if len(fs.CompressedFileSuffix) > 0 {
compressedFileSuffixes["gzip"] = fs.CompressedFileSuffix
compressedFileSuffixes["br"] = FSCompressedFileSuffixes["br"]
}
h := &fsHandler{
root: root,
indexNames: fs.IndexNames,
pathRewrite: fs.PathRewrite,
generateIndexPages: fs.GenerateIndexPages,
compress: fs.Compress,
pathNotFound: fs.PathNotFound,
acceptByteRange: fs.AcceptByteRange,
cacheDuration: cacheDuration,
compressedFileSuffix: compressedFileSuffix,
cache: make(map[string]*fsFile),
compressedCache: make(map[string]*fsFile),
root: root,
indexNames: fs.IndexNames,
pathRewrite: fs.PathRewrite,
generateIndexPages: fs.GenerateIndexPages,
compress: fs.Compress,
compressBrotli: fs.CompressBrotli,
pathNotFound: fs.PathNotFound,
acceptByteRange: fs.AcceptByteRange,
cacheDuration: cacheDuration,
compressedFileSuffixes: compressedFileSuffixes,
cache: make(map[string]*fsFile),
cacheBrotli: make(map[string]*fsFile),
cacheGzip: make(map[string]*fsFile),
}
go func() {
@@ -376,19 +409,21 @@ func (fs *FS) initRequestHandler() {
}
type fsHandler struct {
root string
indexNames []string
pathRewrite PathRewriteFunc
pathNotFound RequestHandler
generateIndexPages bool
compress bool
acceptByteRange bool
cacheDuration time.Duration
compressedFileSuffix string
root string
indexNames []string
pathRewrite PathRewriteFunc
pathNotFound RequestHandler
generateIndexPages bool
compress bool
compressBrotli bool
acceptByteRange bool
cacheDuration time.Duration
compressedFileSuffixes map[string]string
cache map[string]*fsFile
compressedCache map[string]*fsFile
cacheLock sync.Mutex
cache map[string]*fsFile
cacheBrotli map[string]*fsFile
cacheGzip map[string]*fsFile
cacheLock sync.Mutex
smallFileReaderPool sync.Pool
}
@@ -652,7 +687,8 @@ func (h *fsHandler) cleanCache(pendingFiles []*fsFile) []*fsFile {
pendingFiles = remainingFiles
pendingFiles, filesToRelease = cleanCacheNolock(h.cache, pendingFiles, filesToRelease, h.cacheDuration)
pendingFiles, filesToRelease = cleanCacheNolock(h.compressedCache, pendingFiles, filesToRelease, h.cacheDuration)
pendingFiles, filesToRelease = cleanCacheNolock(h.cacheBrotli, pendingFiles, filesToRelease, h.cacheDuration)
pendingFiles, filesToRelease = cleanCacheNolock(h.cacheGzip, pendingFiles, filesToRelease, h.cacheDuration)
h.cacheLock.Unlock()
@@ -709,10 +745,18 @@ func (h *fsHandler) handleRequest(ctx *RequestCtx) {
mustCompress := false
fileCache := h.cache
fileEncoding := ""
byteRange := ctx.Request.Header.peek(strRange)
if len(byteRange) == 0 && h.compress && ctx.Request.Header.HasAcceptEncodingBytes(strGzip) {
mustCompress = true
fileCache = h.compressedCache
if len(byteRange) == 0 && h.compress {
if h.compressBrotli && ctx.Request.Header.HasAcceptEncodingBytes(strBr) {
mustCompress = true
fileCache = h.cacheBrotli
fileEncoding = "br"
} else if ctx.Request.Header.HasAcceptEncodingBytes(strGzip) {
mustCompress = true
fileCache = h.cacheGzip
fileEncoding = "gzip"
}
}
h.cacheLock.Lock()
@@ -726,19 +770,19 @@ func (h *fsHandler) handleRequest(ctx *RequestCtx) {
pathStr := string(path)
filePath := h.root + pathStr
var err error
ff, err = h.openFSFile(filePath, mustCompress)
ff, err = h.openFSFile(filePath, mustCompress, fileEncoding)
if mustCompress && err == errNoCreatePermission {
ctx.Logger().Printf("insufficient permissions for saving compressed file for %q. Serving uncompressed file. "+
"Allow write access to the directory with this file in order to improve fasthttp performance", filePath)
mustCompress = false
ff, err = h.openFSFile(filePath, mustCompress)
ff, err = h.openFSFile(filePath, mustCompress, fileEncoding)
}
if err == errDirIndexRequired {
if !hasTrailingSlash {
ctx.RedirectBytes(append(path, '/'), StatusFound)
return
}
ff, err = h.openIndexFile(ctx, filePath, mustCompress)
ff, err = h.openIndexFile(ctx, filePath, mustCompress, fileEncoding)
if err != nil {
ctx.Logger().Printf("cannot open dir index %q: %s", filePath, err)
ctx.Error("Directory index is forbidden", StatusForbidden)
@@ -789,7 +833,11 @@ func (h *fsHandler) handleRequest(ctx *RequestCtx) {
hdr := &ctx.Response.Header
if ff.compressed {
hdr.SetCanonical(strContentEncoding, strGzip)
if fileEncoding == "br" {
hdr.SetCanonical(strContentEncoding, strBr)
} else if fileEncoding == "gzip" {
hdr.SetCanonical(strContentEncoding, strGzip)
}
}
statusCode := StatusOK
@@ -900,10 +948,10 @@ func ParseByteRange(byteRange []byte, contentLength int) (startPos, endPos int,
return startPos, endPos, nil
}
func (h *fsHandler) openIndexFile(ctx *RequestCtx, dirPath string, mustCompress bool) (*fsFile, error) {
func (h *fsHandler) openIndexFile(ctx *RequestCtx, dirPath string, mustCompress bool, fileEncoding string) (*fsFile, error) {
for _, indexName := range h.indexNames {
indexFilePath := dirPath + "/" + indexName
ff, err := h.openFSFile(indexFilePath, mustCompress)
ff, err := h.openFSFile(indexFilePath, mustCompress, fileEncoding)
if err == nil {
return ff, nil
}
@@ -916,7 +964,7 @@ func (h *fsHandler) openIndexFile(ctx *RequestCtx, dirPath string, mustCompress
return nil, fmt.Errorf("cannot access directory without index page. Directory %q", dirPath)
}
return h.createDirIndex(ctx.URI(), dirPath, mustCompress)
return h.createDirIndex(ctx.URI(), dirPath, mustCompress, fileEncoding)
}
var (
@@ -924,7 +972,7 @@ var (
errNoCreatePermission = errors.New("no 'create file' permissions")
)
func (h *fsHandler) createDirIndex(base *URI, dirPath string, mustCompress bool) (*fsFile, error) {
func (h *fsHandler) createDirIndex(base *URI, dirPath string, mustCompress bool, fileEncoding string) (*fsFile, error) {
w := &bytebufferpool.ByteBuffer{}
basePathEscaped := html.EscapeString(string(base.Path()))
@@ -953,11 +1001,14 @@ func (h *fsHandler) createDirIndex(base *URI, dirPath string, mustCompress bool)
fm := make(map[string]os.FileInfo, len(fileinfos))
filenames := make([]string, 0, len(fileinfos))
nestedContinue:
for _, fi := range fileinfos {
name := fi.Name()
if strings.HasSuffix(name, h.compressedFileSuffix) {
// Do not show compressed files on index page.
continue
for _, cfs := range h.compressedFileSuffixes {
if strings.HasSuffix(name, cfs) {
// Do not show compressed files on index page.
continue nestedContinue
}
}
fm[name] = fi
filenames = append(filenames, name)
@@ -986,7 +1037,11 @@ func (h *fsHandler) createDirIndex(base *URI, dirPath string, mustCompress bool)
if mustCompress {
var zbuf bytebufferpool.ByteBuffer
zbuf.B = AppendGzipBytesLevel(zbuf.B, w.B, CompressDefaultCompression)
if fileEncoding == "br" {
zbuf.B = AppendBrotliBytesLevel(zbuf.B, w.B, CompressDefaultCompression)
} else if fileEncoding == "gzip" {
zbuf.B = AppendGzipBytesLevel(zbuf.B, w.B, CompressDefaultCompression)
}
w = &zbuf
}
@@ -1011,7 +1066,7 @@ const (
fsMaxCompressibleFileSize = 8 * 1024 * 1024
)
func (h *fsHandler) compressAndOpenFSFile(filePath string) (*fsFile, error) {
func (h *fsHandler) compressAndOpenFSFile(filePath string, fileEncoding string) (*fsFile, error) {
f, err := os.Open(filePath)
if err != nil {
return nil, err
@@ -1028,13 +1083,13 @@ func (h *fsHandler) compressAndOpenFSFile(filePath string) (*fsFile, error) {
return nil, errDirIndexRequired
}
if strings.HasSuffix(filePath, h.compressedFileSuffix) ||
if strings.HasSuffix(filePath, h.compressedFileSuffixes[fileEncoding]) ||
fileInfo.Size() > fsMaxCompressibleFileSize ||
!isFileCompressible(f, fsMinCompressRatio) {
return h.newFSFile(f, fileInfo, false)
return h.newFSFile(f, fileInfo, false, "")
}
compressedFilePath := filePath + h.compressedFileSuffix
compressedFilePath := filePath + h.compressedFileSuffixes[fileEncoding]
absPath, err := filepath.Abs(compressedFilePath)
if err != nil {
f.Close()
@@ -1043,20 +1098,20 @@ func (h *fsHandler) compressAndOpenFSFile(filePath string) (*fsFile, error) {
flock := getFileLock(absPath)
flock.Lock()
ff, err := h.compressFileNolock(f, fileInfo, filePath, compressedFilePath)
ff, err := h.compressFileNolock(f, fileInfo, filePath, compressedFilePath, fileEncoding)
flock.Unlock()
return ff, err
}
func (h *fsHandler) compressFileNolock(f *os.File, fileInfo os.FileInfo, filePath, compressedFilePath string) (*fsFile, error) {
func (h *fsHandler) compressFileNolock(f *os.File, fileInfo os.FileInfo, filePath, compressedFilePath string, fileEncoding string) (*fsFile, error) {
// Attempt to open compressed file created by another concurrent
// goroutine.
// It is safe opening such a file, since the file creation
// is guarded by file mutex - see getFileLock call.
if _, err := os.Stat(compressedFilePath); err == nil {
f.Close()
return h.newCompressedFSFile(compressedFilePath)
return h.newCompressedFSFile(compressedFilePath, fileEncoding)
}
// Create temporary file, so concurrent goroutines don't use
@@ -1070,13 +1125,21 @@ func (h *fsHandler) compressFileNolock(f *os.File, fileInfo os.FileInfo, filePat
}
return nil, errNoCreatePermission
}
zw := acquireStacklessGzipWriter(zf, CompressDefaultCompression)
_, err = copyZeroAlloc(zw, f)
if err1 := zw.Flush(); err == nil {
err = err1
if fileEncoding == "br" {
zw := acquireStacklessBrotliWriter(zf, CompressDefaultCompression)
_, err = copyZeroAlloc(zw, f)
if err1 := zw.Flush(); err == nil {
err = err1
}
releaseStacklessBrotliWriter(zw, CompressDefaultCompression)
} else if fileEncoding == "gzip" {
zw := acquireStacklessGzipWriter(zf, CompressDefaultCompression)
_, err = copyZeroAlloc(zw, f)
if err1 := zw.Flush(); err == nil {
err = err1
}
releaseStacklessGzipWriter(zw, CompressDefaultCompression)
}
releaseStacklessGzipWriter(zw, CompressDefaultCompression)
zf.Close()
f.Close()
if err != nil {
@@ -1089,10 +1152,10 @@ func (h *fsHandler) compressFileNolock(f *os.File, fileInfo os.FileInfo, filePat
if err = os.Rename(tmpFilePath, compressedFilePath); err != nil {
return nil, fmt.Errorf("cannot move compressed file from %q to %q: %s", tmpFilePath, compressedFilePath, err)
}
return h.newCompressedFSFile(compressedFilePath)
return h.newCompressedFSFile(compressedFilePath, fileEncoding)
}
func (h *fsHandler) newCompressedFSFile(filePath string) (*fsFile, error) {
func (h *fsHandler) newCompressedFSFile(filePath string, fileEncoding string) (*fsFile, error) {
f, err := os.Open(filePath)
if err != nil {
return nil, fmt.Errorf("cannot open compressed file %q: %s", filePath, err)
@@ -1102,19 +1165,19 @@ func (h *fsHandler) newCompressedFSFile(filePath string) (*fsFile, error) {
f.Close()
return nil, fmt.Errorf("cannot obtain info for compressed file %q: %s", filePath, err)
}
return h.newFSFile(f, fileInfo, true)
return h.newFSFile(f, fileInfo, true, fileEncoding)
}
func (h *fsHandler) openFSFile(filePath string, mustCompress bool) (*fsFile, error) {
func (h *fsHandler) openFSFile(filePath string, mustCompress bool, fileEncoding string) (*fsFile, error) {
filePathOriginal := filePath
if mustCompress {
filePath += h.compressedFileSuffix
filePath += h.compressedFileSuffixes[fileEncoding]
}
f, err := os.Open(filePath)
if err != nil {
if mustCompress && os.IsNotExist(err) {
return h.compressAndOpenFSFile(filePathOriginal)
return h.compressAndOpenFSFile(filePathOriginal, fileEncoding)
}
return nil, err
}
@@ -1129,7 +1192,7 @@ func (h *fsHandler) openFSFile(filePath string, mustCompress bool) (*fsFile, err
f.Close()
if mustCompress {
return nil, fmt.Errorf("directory with unexpected suffix found: %q. Suffix: %q",
filePath, h.compressedFileSuffix)
filePath, h.compressedFileSuffixes[fileEncoding])
}
return nil, errDirIndexRequired
}
@@ -1148,14 +1211,14 @@ func (h *fsHandler) openFSFile(filePath string, mustCompress bool) (*fsFile, err
// The compressed file became stale. Re-create it.
f.Close()
os.Remove(filePath)
return h.compressAndOpenFSFile(filePathOriginal)
return h.compressAndOpenFSFile(filePathOriginal, fileEncoding)
}
}
return h.newFSFile(f, fileInfo, mustCompress)
return h.newFSFile(f, fileInfo, mustCompress, fileEncoding)
}
func (h *fsHandler) newFSFile(f *os.File, fileInfo os.FileInfo, compressed bool) (*fsFile, error) {
func (h *fsHandler) newFSFile(f *os.File, fileInfo os.FileInfo, compressed bool, fileEncoding string) (*fsFile, error) {
n := fileInfo.Size()
contentLength := int(n)
if n != int64(contentLength) {
@@ -1164,10 +1227,10 @@ func (h *fsHandler) newFSFile(f *os.File, fileInfo os.FileInfo, compressed bool)
}
// detect content-type
ext := fileExtension(fileInfo.Name(), compressed, h.compressedFileSuffix)
ext := fileExtension(fileInfo.Name(), compressed, h.compressedFileSuffixes[fileEncoding])
contentType := mime.TypeByExtension(ext)
if len(contentType) == 0 {
data, err := readFileHeader(f, compressed)
data, err := readFileHeader(f, compressed, fileEncoding)
if err != nil {
return nil, fmt.Errorf("cannot read header of the file %q: %s", f.Name(), err)
}
@@ -1189,15 +1252,25 @@ func (h *fsHandler) newFSFile(f *os.File, fileInfo os.FileInfo, compressed bool)
return ff, nil
}
func readFileHeader(f *os.File, compressed bool) ([]byte, error) {
func readFileHeader(f *os.File, compressed bool, fileEncoding string) ([]byte, error) {
r := io.Reader(f)
var zr *gzip.Reader
var (
br *brotli.Reader
zr *gzip.Reader
)
if compressed {
var err error
if zr, err = acquireGzipReader(f); err != nil {
return nil, err
if fileEncoding == "br" {
if br, err = acquireBrotliReader(f); err != nil {
return nil, err
}
r = br
} else if fileEncoding == "gzip" {
if zr, err = acquireGzipReader(f); err != nil {
return nil, err
}
r = zr
}
r = zr
}
lr := &io.LimitedReader{
@@ -1209,6 +1282,10 @@ func readFileHeader(f *os.File, compressed bool) ([]byte, error) {
return nil, err
}
if br != nil {
releaseBrotliReader(br)
}
if zr != nil {
releaseGzipReader(zr)
}
fs_test.go
+67 -10
View File
@@ -201,14 +201,15 @@ func TestServeFileCompressed(t *testing.T) {
t.Parallel()
var ctx RequestCtx
var req Request
req.SetRequestURI("http://foobar.com/baz")
req.Header.Set(HeaderAcceptEncoding, "gzip")
ctx.Init(&req, nil, nil)
ServeFile(&ctx, "fs.go")
ctx.Init(&Request{}, nil, nil)
var resp Response
// request compressed gzip file
ctx.Request.SetRequestURI("http://foobar.com/baz")
ctx.Request.Header.Set(HeaderAcceptEncoding, "gzip")
ServeFile(&ctx, "fs.go")
s := ctx.Response.String()
br := bufio.NewReader(bytes.NewBufferString(s))
if err := resp.Read(br); err != nil {
@@ -231,6 +232,35 @@ func TestServeFileCompressed(t *testing.T) {
if !bytes.Equal(body, expectedBody) {
t.Fatalf("unexpected body %q. expecting %q", body, expectedBody)
}
// request compressed brotli file
ctx.Request.Reset()
ctx.Request.SetRequestURI("http://foobar.com/baz")
ctx.Request.Header.Set(HeaderAcceptEncoding, "br")
ServeFile(&ctx, "fs.go")
s = ctx.Response.String()
br = bufio.NewReader(bytes.NewBufferString(s))
if err = resp.Read(br); err != nil {
t.Fatalf("unexpected error: %s", err)
}
ce = resp.Header.Peek(HeaderContentEncoding)
if string(ce) != "br" {
t.Fatalf("Unexpected 'Content-Encoding' %q. Expecting %q", ce, "br")
}
body, err = resp.BodyUnbrotli()
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
expectedBody, err = getFileContents("/fs.go")
if err != nil {
t.Fatalf("unexpected error: %s", err)
}
if !bytes.Equal(body, expectedBody) {
t.Fatalf("unexpected body %q. expecting %q", body, expectedBody)
}
}
func TestServeFileUncompressed(t *testing.T) {
@@ -442,6 +472,7 @@ func TestFSCompressConcurrent(t *testing.T) {
Root: ".",
GenerateIndexPages: true,
Compress: true,
CompressBrotli: true,
}
h := fs.NewRequestHandler()
@@ -461,7 +492,7 @@ func TestFSCompressConcurrent(t *testing.T) {
for i := 0; i < concurrency; i++ {
select {
case <-ch:
case <-time.After(time.Second):
case <-time.After(time.Second * 2):
t.Fatalf("timeout")
}
}
@@ -474,6 +505,7 @@ func TestFSCompressSingleThread(t *testing.T) {
Root: ".",
GenerateIndexPages: true,
Compress: true,
CompressBrotli: true,
}
h := fs.NewRequestHandler()
@@ -486,12 +518,12 @@ func testFSCompress(t *testing.T, h RequestHandler, filePath string) {
var ctx RequestCtx
ctx.Init(&Request{}, nil, nil)
var resp Response
// request uncompressed file
ctx.Request.Reset()
ctx.Request.SetRequestURI(filePath)
h(&ctx)
var resp Response
s := ctx.Response.String()
br := bufio.NewReader(bytes.NewBufferString(s))
if err := resp.Read(br); err != nil {
@@ -506,7 +538,7 @@ func testFSCompress(t *testing.T, h RequestHandler, filePath string) {
}
body := string(resp.Body())
// request compressed file
// request compressed gzip file
ctx.Request.Reset()
ctx.Request.SetRequestURI(filePath)
ctx.Request.Header.Set(HeaderAcceptEncoding, "gzip")
@@ -530,6 +562,31 @@ func testFSCompress(t *testing.T, h RequestHandler, filePath string) {
if string(zbody) != body {
t.Fatalf("unexpected body len=%d. Expected len=%d. FilePath=%q", len(zbody), len(body), filePath)
}
// request compressed brotli file
ctx.Request.Reset()
ctx.Request.SetRequestURI(filePath)
ctx.Request.Header.Set(HeaderAcceptEncoding, "br")
h(&ctx)
s = ctx.Response.String()
br = bufio.NewReader(bytes.NewBufferString(s))
if err = resp.Read(br); err != nil {
t.Fatalf("unexpected error: %s. filePath=%q", err, filePath)
}
if resp.StatusCode() != StatusOK {
t.Fatalf("unexpected status code: %d. Expecting %d. filePath=%q", resp.StatusCode(), StatusOK, filePath)
}
ce = resp.Header.Peek(HeaderContentEncoding)
if string(ce) != "br" {
t.Fatalf("unexpected content-encoding %q. Expecting %q. filePath=%q", ce, "br", filePath)
}
zbody, err = resp.BodyUnbrotli()
if err != nil {
t.Fatalf("unexpected error when unbrotling response body: %s. filePath=%q", err, filePath)
}
if string(zbody) != body {
t.Fatalf("unexpected body len=%d. Expected len=%d. FilePath=%q", len(zbody), len(body), filePath)
}
}
func TestFileLock(t *testing.T) {