compress internal/lib/*.{so,dylib} files when embedded and uncompress when loading to disk (#103)

Those embedded files represent a real weight that is impacting the
serverless agent (where every MiB impacts the latency of the
serverless). This PR compresses the files (so that the embedded version
is stored compressed in the binary), and uncompresses them in
`DumpEmbeddedWAF`

Author: paulcacheux

.gitignore

@@ -18,6 +18,6 @@
 *.swp
 
 ## Ensure WAF static libraries are not excluded from vendor folders
-!internal/lib/*.so
-!internal/lib/*.dylib
+!internal/lib/*.so.gz
+!internal/lib/*.dylib.gz
 

_tools/libddwaf-updater/update.go

@@ -101,7 +101,7 @@ func main() {
 		written += wrote
 	}
 
-	fmt.Println("All done! Don't forget to check in changes to internal/lib/ and internal/log/ddhaf.h, check the libddwaf upgrade guide to update bindings!")
+	fmt.Println("All done! Don't forget to check in changes to internal/lib/ and internal/log/ddwaf.h, check the libddwaf upgrade guide to update bindings!")
 }
 
 // createEmbedSource creates the embed source file for the given target.
@@ -197,10 +197,12 @@ func handleTarget(wg *sync.WaitGroup, version string, tgt target, assets map[str
 		}
 
 		var destPath string
+		var compress bool
 		switch name := header.FileInfo().Name(); name {
 		case "libddwaf.so", "libddwaf.dylib", "ddwaf.dll":
 			destPath = path.Join(libDir, tgt.binaryLibName())
 			foundLib = true
+			compress = true
 		case "ddwaf.h":
 			if !tgt.primary {
 				continue
@@ -211,8 +213,13 @@ func handleTarget(wg *sync.WaitGroup, version string, tgt target, assets map[str
 			continue
 		}
 
+		pipe := script.NewPipe().WithReader(arch)
+		if compress {
+			pipe = pipe.Filter(compressFilter)
+		}
+
 		fmt.Printf("... downloaded %s\n", destPath)
-		if _, err := script.NewPipe().WithReader(arch).WriteFile(destPath); err != nil {
+		if _, err := pipe.WriteFile(destPath); err != nil {
 			panic(err)
 		}
 		if path.Ext(destPath) != ".h" {
@@ -240,6 +247,13 @@ func handleTarget(wg *sync.WaitGroup, version string, tgt target, assets map[str
 	}
 }
 
+func compressFilter(r io.Reader, w io.Writer) error {
+	gz := gzip.NewWriter(w)
+	defer gz.Close()
+	_, err := io.Copy(gz, r)
+	return err
+}
+
 type target struct {
 	os         string
 	arch       string
@@ -290,7 +304,7 @@ var targets = []target{
 }
 
 func (t target) binaryLibName() string {
-	return fmt.Sprintf("%s-%s-%s.%s", t.base, t.os, t.arch, t.ext)
+	return fmt.Sprintf("%s-%s-%s.%s.gz", t.base, t.os, t.arch, t.ext)
 }
 
 func (t target) embedSourceFilename() string {

internal/lib/lib.go

@@ -8,7 +8,10 @@
 package lib
 
 import (
+	"bytes"
+	"compress/gzip"
 	"fmt"
+	"io"
 	"os"
 
 	_ "embed"
@@ -41,7 +44,21 @@ func DumpEmbeddedWAF() (path string, err error) {
 		}
 	}()
 
-	if err := os.WriteFile(file.Name(), libddwaf, 0400); err != nil {
+	gr, err := gzip.NewReader(bytes.NewReader(libddwaf))
+	if err != nil {
+		return path, fmt.Errorf("error creating gzip reader: %w", err)
+	}
+
+	uncompressedLibddwaf, err := io.ReadAll(gr)
+	if err != nil {
+		return path, fmt.Errorf("error reading gzip content: %w", err)
+	}
+
+	if err := gr.Close(); err != nil {
+		return path, fmt.Errorf("error closing gzip reader: %w", err)
+	}
+
+	if err := os.WriteFile(file.Name(), uncompressedLibddwaf, 0400); err != nil {
 		return path, fmt.Errorf("error writing file: %w", err)
 	}
 

internal/lib/lib_darwin_amd64.go

@@ -11,7 +11,7 @@ package lib
 
 import _ "embed" // Needed for go:embed
 
-//go:embed libddwaf-darwin-amd64.dylib
+//go:embed libddwaf-darwin-amd64.dylib.gz
 var libddwaf []byte
 
 const embedNamePattern = "libddwaf-*.dylib"

internal/lib/lib_darwin_arm64.go

@@ -11,7 +11,7 @@ package lib
 
 import _ "embed" // Needed for go:embed
 
-//go:embed libddwaf-darwin-arm64.dylib
+//go:embed libddwaf-darwin-arm64.dylib.gz
 var libddwaf []byte
 
 const embedNamePattern = "libddwaf-*.dylib"

internal/lib/lib_linux_amd64.go

@@ -11,7 +11,7 @@ package lib
 
 import _ "embed" // Needed for go:embed
 
-//go:embed libddwaf-linux-amd64.so
+//go:embed libddwaf-linux-amd64.so.gz
 var libddwaf []byte
 
 const embedNamePattern = "libddwaf-*.so"

internal/lib/lib_linux_arm64.go

@@ -11,7 +11,7 @@ package lib
 
 import _ "embed" // Needed for go:embed
 
-//go:embed libddwaf-linux-arm64.so
+//go:embed libddwaf-linux-arm64.so.gz
 var libddwaf []byte
 
 const embedNamePattern = "libddwaf-*.so"