From 805af0ee73bd914c4ee3ee8eccf400edaa0ccfb7 Mon Sep 17 00:00:00 2001 From: hex0x00 <68422816+hex0x00@users.noreply.github.com> Date: Mon, 28 Sep 2020 10:14:28 -0500 Subject: [PATCH] Brotli support in FS handler. (#880) * Add files via upload * Update fs.go * Add files via upload * Update fs_test.go --- fs.go | 221 ++++++++++++++++++++++++++++++++++++----------------- fs_test.go | 77 ++++++++++++++++--- 2 files changed, 216 insertions(+), 82 deletions(-) diff --git a/fs.go b/fs.go index 9db399d..77a2a24 100644 --- a/fs.go +++ b/fs.go @@ -16,6 +16,7 @@ import ( "sync" "time" + "github.com/andybalholm/brotli" "github.com/klauspost/compress/gzip" "github.com/valyala/bytebufferpool" ) @@ -105,6 +106,7 @@ var ( Root: "/", GenerateIndexPages: true, Compress: true, + CompressBrotli: true, AcceptByteRange: true, } rootFSHandler RequestHandler @@ -236,6 +238,13 @@ type FS struct { // Transparent compression is disabled by default. Compress bool + // Uses brotli encoding and fallbacks to gzip in responses if set to true, uses gzip if set to false. + // + // This value has sense only if Compress is set. + // + // Brotli encoding is disabled by default. + CompressBrotli bool + // Enables byte range requests if set to true. // // Byte range requests are disabled by default. @@ -266,6 +275,13 @@ type FS struct { // FSCompressedFileSuffix is used by default. CompressedFileSuffix string + // Suffixes list to add to compressedFileSuffix depending on encoding + // + // This value has sense only if Compress is set. + // + // FSCompressedFileSuffixes is used by default. + CompressedFileSuffixes map[string]string + once sync.Once h RequestHandler } @@ -275,6 +291,14 @@ type FS struct { // See FS.Compress for details. const FSCompressedFileSuffix = ".fasthttp.gz" +// FSCompressedFileSuffixes is the suffixes FS adds to the original file names depending on encoding +// when trying to store compressed file under the new file name. +// See FS.Compress for details. +var FSCompressedFileSuffixes = map[string]string{ + "gzip": ".fasthttp.gz", + "br": ".fasthttp.br", +} + // FSHandlerCacheDuration is the default expiration duration for inactive // file handlers opened by FS. const FSHandlerCacheDuration = 10 * time.Second @@ -345,23 +369,32 @@ func (fs *FS) initRequestHandler() { if cacheDuration <= 0 { cacheDuration = FSHandlerCacheDuration } - compressedFileSuffix := fs.CompressedFileSuffix - if len(compressedFileSuffix) == 0 { - compressedFileSuffix = FSCompressedFileSuffix + + compressedFileSuffixes := fs.CompressedFileSuffixes + if len(compressedFileSuffixes["br"]) == 0 || len(compressedFileSuffixes["gzip"]) == 0 || + compressedFileSuffixes["br"] == compressedFileSuffixes["gzip"] { + compressedFileSuffixes = FSCompressedFileSuffixes + } + + if len(fs.CompressedFileSuffix) > 0 { + compressedFileSuffixes["gzip"] = fs.CompressedFileSuffix + compressedFileSuffixes["br"] = FSCompressedFileSuffixes["br"] } h := &fsHandler{ - root: root, - indexNames: fs.IndexNames, - pathRewrite: fs.PathRewrite, - generateIndexPages: fs.GenerateIndexPages, - compress: fs.Compress, - pathNotFound: fs.PathNotFound, - acceptByteRange: fs.AcceptByteRange, - cacheDuration: cacheDuration, - compressedFileSuffix: compressedFileSuffix, - cache: make(map[string]*fsFile), - compressedCache: make(map[string]*fsFile), + root: root, + indexNames: fs.IndexNames, + pathRewrite: fs.PathRewrite, + generateIndexPages: fs.GenerateIndexPages, + compress: fs.Compress, + compressBrotli: fs.CompressBrotli, + pathNotFound: fs.PathNotFound, + acceptByteRange: fs.AcceptByteRange, + cacheDuration: cacheDuration, + compressedFileSuffixes: compressedFileSuffixes, + cache: make(map[string]*fsFile), + cacheBrotli: make(map[string]*fsFile), + cacheGzip: make(map[string]*fsFile), } go func() { @@ -376,19 +409,21 @@ func (fs *FS) initRequestHandler() { } type fsHandler struct { - root string - indexNames []string - pathRewrite PathRewriteFunc - pathNotFound RequestHandler - generateIndexPages bool - compress bool - acceptByteRange bool - cacheDuration time.Duration - compressedFileSuffix string + root string + indexNames []string + pathRewrite PathRewriteFunc + pathNotFound RequestHandler + generateIndexPages bool + compress bool + compressBrotli bool + acceptByteRange bool + cacheDuration time.Duration + compressedFileSuffixes map[string]string - cache map[string]*fsFile - compressedCache map[string]*fsFile - cacheLock sync.Mutex + cache map[string]*fsFile + cacheBrotli map[string]*fsFile + cacheGzip map[string]*fsFile + cacheLock sync.Mutex smallFileReaderPool sync.Pool } @@ -652,7 +687,8 @@ func (h *fsHandler) cleanCache(pendingFiles []*fsFile) []*fsFile { pendingFiles = remainingFiles pendingFiles, filesToRelease = cleanCacheNolock(h.cache, pendingFiles, filesToRelease, h.cacheDuration) - pendingFiles, filesToRelease = cleanCacheNolock(h.compressedCache, pendingFiles, filesToRelease, h.cacheDuration) + pendingFiles, filesToRelease = cleanCacheNolock(h.cacheBrotli, pendingFiles, filesToRelease, h.cacheDuration) + pendingFiles, filesToRelease = cleanCacheNolock(h.cacheGzip, pendingFiles, filesToRelease, h.cacheDuration) h.cacheLock.Unlock() @@ -709,10 +745,18 @@ func (h *fsHandler) handleRequest(ctx *RequestCtx) { mustCompress := false fileCache := h.cache + fileEncoding := "" byteRange := ctx.Request.Header.peek(strRange) - if len(byteRange) == 0 && h.compress && ctx.Request.Header.HasAcceptEncodingBytes(strGzip) { - mustCompress = true - fileCache = h.compressedCache + if len(byteRange) == 0 && h.compress { + if h.compressBrotli && ctx.Request.Header.HasAcceptEncodingBytes(strBr) { + mustCompress = true + fileCache = h.cacheBrotli + fileEncoding = "br" + } else if ctx.Request.Header.HasAcceptEncodingBytes(strGzip) { + mustCompress = true + fileCache = h.cacheGzip + fileEncoding = "gzip" + } } h.cacheLock.Lock() @@ -726,19 +770,19 @@ func (h *fsHandler) handleRequest(ctx *RequestCtx) { pathStr := string(path) filePath := h.root + pathStr var err error - ff, err = h.openFSFile(filePath, mustCompress) + ff, err = h.openFSFile(filePath, mustCompress, fileEncoding) if mustCompress && err == errNoCreatePermission { ctx.Logger().Printf("insufficient permissions for saving compressed file for %q. Serving uncompressed file. "+ "Allow write access to the directory with this file in order to improve fasthttp performance", filePath) mustCompress = false - ff, err = h.openFSFile(filePath, mustCompress) + ff, err = h.openFSFile(filePath, mustCompress, fileEncoding) } if err == errDirIndexRequired { if !hasTrailingSlash { ctx.RedirectBytes(append(path, '/'), StatusFound) return } - ff, err = h.openIndexFile(ctx, filePath, mustCompress) + ff, err = h.openIndexFile(ctx, filePath, mustCompress, fileEncoding) if err != nil { ctx.Logger().Printf("cannot open dir index %q: %s", filePath, err) ctx.Error("Directory index is forbidden", StatusForbidden) @@ -789,7 +833,11 @@ func (h *fsHandler) handleRequest(ctx *RequestCtx) { hdr := &ctx.Response.Header if ff.compressed { - hdr.SetCanonical(strContentEncoding, strGzip) + if fileEncoding == "br" { + hdr.SetCanonical(strContentEncoding, strBr) + } else if fileEncoding == "gzip" { + hdr.SetCanonical(strContentEncoding, strGzip) + } } statusCode := StatusOK @@ -900,10 +948,10 @@ func ParseByteRange(byteRange []byte, contentLength int) (startPos, endPos int, return startPos, endPos, nil } -func (h *fsHandler) openIndexFile(ctx *RequestCtx, dirPath string, mustCompress bool) (*fsFile, error) { +func (h *fsHandler) openIndexFile(ctx *RequestCtx, dirPath string, mustCompress bool, fileEncoding string) (*fsFile, error) { for _, indexName := range h.indexNames { indexFilePath := dirPath + "/" + indexName - ff, err := h.openFSFile(indexFilePath, mustCompress) + ff, err := h.openFSFile(indexFilePath, mustCompress, fileEncoding) if err == nil { return ff, nil } @@ -916,7 +964,7 @@ func (h *fsHandler) openIndexFile(ctx *RequestCtx, dirPath string, mustCompress return nil, fmt.Errorf("cannot access directory without index page. Directory %q", dirPath) } - return h.createDirIndex(ctx.URI(), dirPath, mustCompress) + return h.createDirIndex(ctx.URI(), dirPath, mustCompress, fileEncoding) } var ( @@ -924,7 +972,7 @@ var ( errNoCreatePermission = errors.New("no 'create file' permissions") ) -func (h *fsHandler) createDirIndex(base *URI, dirPath string, mustCompress bool) (*fsFile, error) { +func (h *fsHandler) createDirIndex(base *URI, dirPath string, mustCompress bool, fileEncoding string) (*fsFile, error) { w := &bytebufferpool.ByteBuffer{} basePathEscaped := html.EscapeString(string(base.Path())) @@ -953,11 +1001,14 @@ func (h *fsHandler) createDirIndex(base *URI, dirPath string, mustCompress bool) fm := make(map[string]os.FileInfo, len(fileinfos)) filenames := make([]string, 0, len(fileinfos)) +nestedContinue: for _, fi := range fileinfos { name := fi.Name() - if strings.HasSuffix(name, h.compressedFileSuffix) { - // Do not show compressed files on index page. - continue + for _, cfs := range h.compressedFileSuffixes { + if strings.HasSuffix(name, cfs) { + // Do not show compressed files on index page. + continue nestedContinue + } } fm[name] = fi filenames = append(filenames, name) @@ -986,7 +1037,11 @@ func (h *fsHandler) createDirIndex(base *URI, dirPath string, mustCompress bool) if mustCompress { var zbuf bytebufferpool.ByteBuffer - zbuf.B = AppendGzipBytesLevel(zbuf.B, w.B, CompressDefaultCompression) + if fileEncoding == "br" { + zbuf.B = AppendBrotliBytesLevel(zbuf.B, w.B, CompressDefaultCompression) + } else if fileEncoding == "gzip" { + zbuf.B = AppendGzipBytesLevel(zbuf.B, w.B, CompressDefaultCompression) + } w = &zbuf } @@ -1011,7 +1066,7 @@ const ( fsMaxCompressibleFileSize = 8 * 1024 * 1024 ) -func (h *fsHandler) compressAndOpenFSFile(filePath string) (*fsFile, error) { +func (h *fsHandler) compressAndOpenFSFile(filePath string, fileEncoding string) (*fsFile, error) { f, err := os.Open(filePath) if err != nil { return nil, err @@ -1028,13 +1083,13 @@ func (h *fsHandler) compressAndOpenFSFile(filePath string) (*fsFile, error) { return nil, errDirIndexRequired } - if strings.HasSuffix(filePath, h.compressedFileSuffix) || + if strings.HasSuffix(filePath, h.compressedFileSuffixes[fileEncoding]) || fileInfo.Size() > fsMaxCompressibleFileSize || !isFileCompressible(f, fsMinCompressRatio) { - return h.newFSFile(f, fileInfo, false) + return h.newFSFile(f, fileInfo, false, "") } - compressedFilePath := filePath + h.compressedFileSuffix + compressedFilePath := filePath + h.compressedFileSuffixes[fileEncoding] absPath, err := filepath.Abs(compressedFilePath) if err != nil { f.Close() @@ -1043,20 +1098,20 @@ func (h *fsHandler) compressAndOpenFSFile(filePath string) (*fsFile, error) { flock := getFileLock(absPath) flock.Lock() - ff, err := h.compressFileNolock(f, fileInfo, filePath, compressedFilePath) + ff, err := h.compressFileNolock(f, fileInfo, filePath, compressedFilePath, fileEncoding) flock.Unlock() return ff, err } -func (h *fsHandler) compressFileNolock(f *os.File, fileInfo os.FileInfo, filePath, compressedFilePath string) (*fsFile, error) { +func (h *fsHandler) compressFileNolock(f *os.File, fileInfo os.FileInfo, filePath, compressedFilePath string, fileEncoding string) (*fsFile, error) { // Attempt to open compressed file created by another concurrent // goroutine. // It is safe opening such a file, since the file creation // is guarded by file mutex - see getFileLock call. if _, err := os.Stat(compressedFilePath); err == nil { f.Close() - return h.newCompressedFSFile(compressedFilePath) + return h.newCompressedFSFile(compressedFilePath, fileEncoding) } // Create temporary file, so concurrent goroutines don't use @@ -1070,13 +1125,21 @@ func (h *fsHandler) compressFileNolock(f *os.File, fileInfo os.FileInfo, filePat } return nil, errNoCreatePermission } - - zw := acquireStacklessGzipWriter(zf, CompressDefaultCompression) - _, err = copyZeroAlloc(zw, f) - if err1 := zw.Flush(); err == nil { - err = err1 + if fileEncoding == "br" { + zw := acquireStacklessBrotliWriter(zf, CompressDefaultCompression) + _, err = copyZeroAlloc(zw, f) + if err1 := zw.Flush(); err == nil { + err = err1 + } + releaseStacklessBrotliWriter(zw, CompressDefaultCompression) + } else if fileEncoding == "gzip" { + zw := acquireStacklessGzipWriter(zf, CompressDefaultCompression) + _, err = copyZeroAlloc(zw, f) + if err1 := zw.Flush(); err == nil { + err = err1 + } + releaseStacklessGzipWriter(zw, CompressDefaultCompression) } - releaseStacklessGzipWriter(zw, CompressDefaultCompression) zf.Close() f.Close() if err != nil { @@ -1089,10 +1152,10 @@ func (h *fsHandler) compressFileNolock(f *os.File, fileInfo os.FileInfo, filePat if err = os.Rename(tmpFilePath, compressedFilePath); err != nil { return nil, fmt.Errorf("cannot move compressed file from %q to %q: %s", tmpFilePath, compressedFilePath, err) } - return h.newCompressedFSFile(compressedFilePath) + return h.newCompressedFSFile(compressedFilePath, fileEncoding) } -func (h *fsHandler) newCompressedFSFile(filePath string) (*fsFile, error) { +func (h *fsHandler) newCompressedFSFile(filePath string, fileEncoding string) (*fsFile, error) { f, err := os.Open(filePath) if err != nil { return nil, fmt.Errorf("cannot open compressed file %q: %s", filePath, err) @@ -1102,19 +1165,19 @@ func (h *fsHandler) newCompressedFSFile(filePath string) (*fsFile, error) { f.Close() return nil, fmt.Errorf("cannot obtain info for compressed file %q: %s", filePath, err) } - return h.newFSFile(f, fileInfo, true) + return h.newFSFile(f, fileInfo, true, fileEncoding) } -func (h *fsHandler) openFSFile(filePath string, mustCompress bool) (*fsFile, error) { +func (h *fsHandler) openFSFile(filePath string, mustCompress bool, fileEncoding string) (*fsFile, error) { filePathOriginal := filePath if mustCompress { - filePath += h.compressedFileSuffix + filePath += h.compressedFileSuffixes[fileEncoding] } f, err := os.Open(filePath) if err != nil { if mustCompress && os.IsNotExist(err) { - return h.compressAndOpenFSFile(filePathOriginal) + return h.compressAndOpenFSFile(filePathOriginal, fileEncoding) } return nil, err } @@ -1129,7 +1192,7 @@ func (h *fsHandler) openFSFile(filePath string, mustCompress bool) (*fsFile, err f.Close() if mustCompress { return nil, fmt.Errorf("directory with unexpected suffix found: %q. Suffix: %q", - filePath, h.compressedFileSuffix) + filePath, h.compressedFileSuffixes[fileEncoding]) } return nil, errDirIndexRequired } @@ -1148,14 +1211,14 @@ func (h *fsHandler) openFSFile(filePath string, mustCompress bool) (*fsFile, err // The compressed file became stale. Re-create it. f.Close() os.Remove(filePath) - return h.compressAndOpenFSFile(filePathOriginal) + return h.compressAndOpenFSFile(filePathOriginal, fileEncoding) } } - return h.newFSFile(f, fileInfo, mustCompress) + return h.newFSFile(f, fileInfo, mustCompress, fileEncoding) } -func (h *fsHandler) newFSFile(f *os.File, fileInfo os.FileInfo, compressed bool) (*fsFile, error) { +func (h *fsHandler) newFSFile(f *os.File, fileInfo os.FileInfo, compressed bool, fileEncoding string) (*fsFile, error) { n := fileInfo.Size() contentLength := int(n) if n != int64(contentLength) { @@ -1164,10 +1227,10 @@ func (h *fsHandler) newFSFile(f *os.File, fileInfo os.FileInfo, compressed bool) } // detect content-type - ext := fileExtension(fileInfo.Name(), compressed, h.compressedFileSuffix) + ext := fileExtension(fileInfo.Name(), compressed, h.compressedFileSuffixes[fileEncoding]) contentType := mime.TypeByExtension(ext) if len(contentType) == 0 { - data, err := readFileHeader(f, compressed) + data, err := readFileHeader(f, compressed, fileEncoding) if err != nil { return nil, fmt.Errorf("cannot read header of the file %q: %s", f.Name(), err) } @@ -1189,15 +1252,25 @@ func (h *fsHandler) newFSFile(f *os.File, fileInfo os.FileInfo, compressed bool) return ff, nil } -func readFileHeader(f *os.File, compressed bool) ([]byte, error) { +func readFileHeader(f *os.File, compressed bool, fileEncoding string) ([]byte, error) { r := io.Reader(f) - var zr *gzip.Reader + var ( + br *brotli.Reader + zr *gzip.Reader + ) if compressed { var err error - if zr, err = acquireGzipReader(f); err != nil { - return nil, err + if fileEncoding == "br" { + if br, err = acquireBrotliReader(f); err != nil { + return nil, err + } + r = br + } else if fileEncoding == "gzip" { + if zr, err = acquireGzipReader(f); err != nil { + return nil, err + } + r = zr } - r = zr } lr := &io.LimitedReader{ @@ -1209,6 +1282,10 @@ func readFileHeader(f *os.File, compressed bool) ([]byte, error) { return nil, err } + if br != nil { + releaseBrotliReader(br) + } + if zr != nil { releaseGzipReader(zr) } diff --git a/fs_test.go b/fs_test.go index 7f85649..8465ff8 100644 --- a/fs_test.go +++ b/fs_test.go @@ -201,14 +201,15 @@ func TestServeFileCompressed(t *testing.T) { t.Parallel() var ctx RequestCtx - var req Request - req.SetRequestURI("http://foobar.com/baz") - req.Header.Set(HeaderAcceptEncoding, "gzip") - ctx.Init(&req, nil, nil) - - ServeFile(&ctx, "fs.go") + ctx.Init(&Request{}, nil, nil) var resp Response + + // request compressed gzip file + ctx.Request.SetRequestURI("http://foobar.com/baz") + ctx.Request.Header.Set(HeaderAcceptEncoding, "gzip") + ServeFile(&ctx, "fs.go") + s := ctx.Response.String() br := bufio.NewReader(bytes.NewBufferString(s)) if err := resp.Read(br); err != nil { @@ -231,6 +232,35 @@ func TestServeFileCompressed(t *testing.T) { if !bytes.Equal(body, expectedBody) { t.Fatalf("unexpected body %q. expecting %q", body, expectedBody) } + + // request compressed brotli file + ctx.Request.Reset() + ctx.Request.SetRequestURI("http://foobar.com/baz") + ctx.Request.Header.Set(HeaderAcceptEncoding, "br") + ServeFile(&ctx, "fs.go") + + s = ctx.Response.String() + br = bufio.NewReader(bytes.NewBufferString(s)) + if err = resp.Read(br); err != nil { + t.Fatalf("unexpected error: %s", err) + } + + ce = resp.Header.Peek(HeaderContentEncoding) + if string(ce) != "br" { + t.Fatalf("Unexpected 'Content-Encoding' %q. Expecting %q", ce, "br") + } + + body, err = resp.BodyUnbrotli() + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + expectedBody, err = getFileContents("/fs.go") + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + if !bytes.Equal(body, expectedBody) { + t.Fatalf("unexpected body %q. expecting %q", body, expectedBody) + } } func TestServeFileUncompressed(t *testing.T) { @@ -442,6 +472,7 @@ func TestFSCompressConcurrent(t *testing.T) { Root: ".", GenerateIndexPages: true, Compress: true, + CompressBrotli: true, } h := fs.NewRequestHandler() @@ -461,7 +492,7 @@ func TestFSCompressConcurrent(t *testing.T) { for i := 0; i < concurrency; i++ { select { case <-ch: - case <-time.After(time.Second): + case <-time.After(time.Second * 2): t.Fatalf("timeout") } } @@ -474,6 +505,7 @@ func TestFSCompressSingleThread(t *testing.T) { Root: ".", GenerateIndexPages: true, Compress: true, + CompressBrotli: true, } h := fs.NewRequestHandler() @@ -486,12 +518,12 @@ func testFSCompress(t *testing.T, h RequestHandler, filePath string) { var ctx RequestCtx ctx.Init(&Request{}, nil, nil) + var resp Response + // request uncompressed file ctx.Request.Reset() ctx.Request.SetRequestURI(filePath) h(&ctx) - - var resp Response s := ctx.Response.String() br := bufio.NewReader(bytes.NewBufferString(s)) if err := resp.Read(br); err != nil { @@ -506,7 +538,7 @@ func testFSCompress(t *testing.T, h RequestHandler, filePath string) { } body := string(resp.Body()) - // request compressed file + // request compressed gzip file ctx.Request.Reset() ctx.Request.SetRequestURI(filePath) ctx.Request.Header.Set(HeaderAcceptEncoding, "gzip") @@ -530,6 +562,31 @@ func testFSCompress(t *testing.T, h RequestHandler, filePath string) { if string(zbody) != body { t.Fatalf("unexpected body len=%d. Expected len=%d. FilePath=%q", len(zbody), len(body), filePath) } + + // request compressed brotli file + ctx.Request.Reset() + ctx.Request.SetRequestURI(filePath) + ctx.Request.Header.Set(HeaderAcceptEncoding, "br") + h(&ctx) + s = ctx.Response.String() + br = bufio.NewReader(bytes.NewBufferString(s)) + if err = resp.Read(br); err != nil { + t.Fatalf("unexpected error: %s. filePath=%q", err, filePath) + } + if resp.StatusCode() != StatusOK { + t.Fatalf("unexpected status code: %d. Expecting %d. filePath=%q", resp.StatusCode(), StatusOK, filePath) + } + ce = resp.Header.Peek(HeaderContentEncoding) + if string(ce) != "br" { + t.Fatalf("unexpected content-encoding %q. Expecting %q. filePath=%q", ce, "br", filePath) + } + zbody, err = resp.BodyUnbrotli() + if err != nil { + t.Fatalf("unexpected error when unbrotling response body: %s. filePath=%q", err, filePath) + } + if string(zbody) != body { + t.Fatalf("unexpected body len=%d. Expected len=%d. FilePath=%q", len(zbody), len(body), filePath) + } } func TestFileLock(t *testing.T) {