Add zstd support (#28)

Author: vearutop

README.md

@@ -89,7 +89,7 @@ Recommended way of embedding assets is to compress assets before the build, so t
 files. This can be inconvenient in some cases, there is `EncodeOnInit` option to compress assets in runtime when
 creating file server. Once compressed, assets will be served directly without additional dynamic compression.
 
-Files with extensions ".gz", ".br", ".gif", ".jpg", ".png", ".webp" are excluded from runtime encoding by default.
+Files with extensions ".gz", ".br", ".zst", ".gif", ".jpg", ".png", ".webp" are excluded from runtime encoding by default.
 
 > **_NOTE:_** Compressing assets in runtime can degrade startup performance and increase memory usage to prepare and store compressed data.
 
@@ -106,6 +106,7 @@ import (
 	"net/http"
 
 	"github.com/vearutop/statigz"
+	"github.com/vearutop/statigz/zstd"
 	"github.com/vearutop/statigz/brotli"
 )
 
@@ -116,7 +117,7 @@ var st embed.FS
 
 func main() {
 	// Plug static assets handler to your server or router.
-	err := http.ListenAndServe(":80", statigz.FileServer(st, brotli.AddEncoding, statigz.FSPrefix("static")))
+	err := http.ListenAndServe(":80", statigz.FileServer(st, brotli.AddEncoding, zstd.AddEncoding, statigz.FSPrefix("static")))
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -125,17 +126,17 @@ func main() {
 
 ### Custom error handling
 
-Error states can be handled with the `staticgz.OnError` and `staticgz.OnNotFound` options. These allow you to customize
+Error states can be handled with the `statigz.OnError` and `statigz.OnNotFound` options. These allow you to customize
 the response sent to the client when an error occurs or when no resource is found.
 
 ```go
 fileServer := statigz.FileServer(
 	st,
-	staticgz.OnError(func(w http.ResponseWriter, r *http.Request, err error) {
+	statigz.OnError(func(w http.ResponseWriter, r *http.Request, err error) {
 		// Handle error.
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 	}),
-	staticgz.OnNotFound(func(w http.ResponseWriter, r *http.Request) {
+	statigz.OnNotFound(func(w http.ResponseWriter, r *http.Request) {
 		// Handle not found.
 		http.Error(w, "Not found", http.StatusNotFound)
 

go.mod

@@ -1,10 +1,11 @@
 module github.com/vearutop/statigz
 
-go 1.17
+go 1.22
 
 require (
 	github.com/andybalholm/brotli v1.1.1
 	github.com/bool64/dev v0.2.39
+	github.com/klauspost/compress v1.18.0
 	github.com/stretchr/testify v1.4.0
 )
 

go.sum

@@ -4,6 +4,8 @@ github.com/bool64/dev v0.2.39 h1:kP8DnMGlWXhGYJEZE/J0l/gVBdbuhoPGL+MJG4QbofE=
 github.com/bool64/dev v0.2.39/go.mod h1:iJbh1y/HkunEPhgebWRNcs8wfGq7sjvJ6W5iabL8ACg=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

server.go

@@ -69,7 +69,7 @@ const (
 )
 
 // SkipCompressionExt lists file extensions of data that is already compressed.
-var SkipCompressionExt = []string{".gz", ".br", ".gif", ".jpg", ".png", ".webp"}
+var SkipCompressionExt = []string{".gz", ".br", ".zst", ".gif", ".jpg", ".png", ".webp"}
 
 // FileServer creates an instance of Server from file system.
 //
@@ -123,50 +123,54 @@ func (s *Server) encodeFiles() error {
 		}
 
 		for fn, i := range s.info {
-			isEncoded := false
-
-			for _, ext := range SkipCompressionExt {
-				if strings.HasSuffix(fn, ext) {
-					isEncoded = true
-
-					break
-				}
-			}
-
-			if isEncoded {
+			if i.isDir {
 				continue
 			}
 
-			if _, found := s.info[fn+enc.FileExt]; found {
-				continue
+			if err := s.encodeFile(fn, i, enc); err != nil {
+				return err
 			}
+		}
+	}
 
-			// Skip encoding of small data.
-			if i.size < minSizeToEncode {
-				continue
-			}
+	return nil
+}
 
-			f, err := s.fs.Open(fn)
-			if err != nil {
-				return err
-			}
+func (s *Server) encodeFile(fn string, i fileInfo, enc Encoding) error {
+	for _, ext := range SkipCompressionExt {
+		if strings.HasSuffix(fn, ext) {
+			return nil
+		}
+	}
 
-			b, err := enc.Encoder(f)
-			if err != nil {
-				return err
-			}
+	if _, found := s.info[fn+enc.FileExt]; found {
+		return nil
+	}
 
-			// Skip encoding for non-compressible data.
-			if float64(len(b))/float64(i.size) > minCompressionRatio {
-				continue
-			}
+	// Skip encoding of small data.
+	if i.size < minSizeToEncode {
+		return nil
+	}
 
-			s.info[fn+enc.FileExt] = fileInfo{
-				hash:    i.hash + enc.FileExt,
-				size:    len(b),
-				content: b[0:len(b):len(b)],
-			}
-		}
+	f, err := s.fs.Open(fn)
+	if err != nil {
+		return err
+	}
+
+	b, err := enc.Encoder(f)
+	if err != nil {
+		return err
+	}
+
+	// Skip encoding for non-compressible data.
+	if float64(len(b))/float64(i.size) > minCompressionRatio {
+		return nil
+	}
+
+	s.info[fn+enc.FileExt] = fileInfo{
+		hash:    i.hash + enc.FileExt,
+		size:    len(b),
+		content: b[0:len(b):len(b)],
 	}
 
 	return nil

server_test.go

@@ -12,10 +12,12 @@ import (
 	"testing"
 
 	brotli2 "github.com/andybalholm/brotli"
+	zstd2 "github.com/klauspost/compress/zstd"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/vearutop/statigz"
 	"github.com/vearutop/statigz/brotli"
+	"github.com/vearutop/statigz/zstd"
 )
 
 //go:embed testdata/*
@@ -285,6 +287,62 @@ func TestServer_ServeHTTP_get_br(t *testing.T) {
 	assert.Equal(t, raw, decoded)
 }
 
+func TestServer_ServeHTTP_get_zst_init(t *testing.T) {
+	s := statigz.FileServer(v, statigz.EncodeOnInit, brotli.AddEncoding, zstd.AddEncoding)
+
+	req, err := http.NewRequest(http.MethodGet, "/testdata/swagger.json", nil)
+	require.NoError(t, err)
+
+	req.Header.Set("Accept-Encoding", "zstd")
+
+	rw := httptest.NewRecorder()
+	s.ServeHTTP(rw, req)
+
+	assert.Equal(t, http.StatusOK, rw.Code)
+	assert.Equal(t, "zstd", rw.Header().Get("Content-Encoding"))
+	assert.Equal(t, "1bp69hxb9nd93.zst", rw.Header().Get("Etag"))
+	assert.NotEmpty(t, rw.Body.String())
+
+	r, err := zstd2.NewReader(rw.Body)
+	require.NoError(t, err)
+
+	decoded, err := io.ReadAll(r)
+	assert.NoError(t, err)
+
+	raw, err := os.ReadFile("testdata/swagger.json")
+	assert.NoError(t, err)
+
+	assert.Equal(t, raw, decoded)
+}
+
+func TestServer_ServeHTTP_get_zst(t *testing.T) {
+	s := statigz.FileServer(v, statigz.EncodeOnInit, brotli.AddEncoding, zstd.AddEncoding)
+
+	req, err := http.NewRequest(http.MethodGet, "/testdata/deeper/swagger.json", nil)
+	require.NoError(t, err)
+
+	req.Header.Set("Accept-Encoding", "zstd")
+
+	rw := httptest.NewRecorder()
+	s.ServeHTTP(rw, req)
+
+	assert.Equal(t, http.StatusOK, rw.Code)
+	assert.Equal(t, "zstd", rw.Header().Get("Content-Encoding"))
+	assert.Equal(t, "1061t8bc8jx4s", rw.Header().Get("Etag"))
+	assert.NotEmpty(t, rw.Body.String())
+
+	r, err := zstd2.NewReader(rw.Body)
+	require.NoError(t, err)
+
+	decoded, err := io.ReadAll(r)
+	assert.NoError(t, err)
+
+	raw, err := os.ReadFile("testdata/swagger.json")
+	assert.NoError(t, err)
+
+	assert.Equal(t, raw, decoded)
+}
+
 func TestServer_ServeHTTP_indexCompressed(t *testing.T) {
 	s := statigz.FileServer(v)
 

testdata/deeper/swagger.json.zst

@@ -0,0 +1,43 @@
+// Package zstd provides encoding for statigz.Server.
+package zstd
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/klauspost/compress/zstd"
+	"github.com/vearutop/statigz"
+)
+
+// AddEncoding is an option that prepends zstd to encodings of statigz.Server.
+//
+// It is located in a separate package to allow better control of imports graph.
+func AddEncoding(server *statigz.Server) {
+	enc := statigz.Encoding{
+		FileExt:         ".zst",
+		ContentEncoding: "zstd",
+		Decoder: func(r io.Reader) (io.Reader, error) {
+			return zstd.NewReader(r)
+		},
+		Encoder: func(r io.Reader) ([]byte, error) {
+			res := bytes.NewBuffer(nil)
+
+			w, err := zstd.NewWriter(res, zstd.WithEncoderLevel(zstd.SpeedBestCompression))
+			if err != nil {
+				return nil, err
+			}
+
+			if _, err := io.Copy(w, r); err != nil {
+				return nil, err
+			}
+
+			if err := w.Close(); err != nil {
+				return nil, err
+			}
+
+			return res.Bytes(), nil
+		},
+	}
+
+	server.Encodings = append([]statigz.Encoding{enc}, server.Encodings...)
+}

zstd/encoding.go

@@ -0,0 +1,27 @@
+package zstd_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/vearutop/statigz"
+	"github.com/vearutop/statigz/zstd"
+)
+
+func TestAddEncoding(t *testing.T) {
+	s := &statigz.Server{}
+	s.Encodings = append(s.Encodings, statigz.GzipEncoding())
+	zstd.AddEncoding(s)
+
+	assert.Equal(t, ".zst", s.Encodings[0].FileExt)
+	assert.Equal(t, ".gz", s.Encodings[1].FileExt)
+	d, err := s.Encodings[0].Decoder(nil)
+	assert.NoError(t, err)
+	assert.NotNil(t, d)
+
+	e, err := s.Encodings[0].Encoder(strings.NewReader(strings.Repeat("A", 10000)))
+	assert.NoError(t, err)
+	assert.NotEmpty(t, e)
+	assert.Less(t, len(e), 100)
+}