refactor embed storage and packaging for bazel (#50)

Author: eliottness

_tools/libddwaf-updater/update.go

@@ -30,6 +30,10 @@ var (
 	currentVersion string
 )
 
+const (
+	goVersionUnsupported = "go1.22"
+)
+
 func main() {
 	force := os.Args[1] == "--force"
 
@@ -63,21 +67,7 @@ func main() {
 	wg := sync.WaitGroup{}
 	wg.Add(len(targets))
 	for _, tgt := range targets {
-		embedDir := path.Join(libDir, fmt.Sprintf("%s-%s", tgt.os, tgt.arch))
-		created := false
-		if _, err = os.Stat(embedDir); errors.Is(err, os.ErrNotExist) {
-			if err = os.MkdirAll(embedDir, 0755); err != nil {
-				panic(err)
-			}
-			created = true
-		}
-		if created || force {
-			if err := os.WriteFile(path.Join(embedDir, "vendor.go"), []byte(vendorMarkerFile), 0644); err != nil {
-				panic(err)
-			}
-			createEmbedSource(tgt)
-		}
-		go handleTarget(&wg, version, tgt, embedDir, assets)
+		go handleTarget(&wg, version, tgt, assets, force)
 	}
 
 	wg.Wait()
@@ -98,6 +88,9 @@ func main() {
 	fmt.Println("All done! Don't forget to check in changes to include/ and internal/lib/, check the libddwaf upgrade guide to update bindings!")
 }
 
+// createEmbedSource creates the embed source file for the given target.
+// The go:embed directive MUST be in the same package as the target of the directive.
+// See bazelbuild/bazel-gazelle#1316 for more details.
 func createEmbedSource(tgt target) {
 	gosource := strings.Join(
 		[]string{
@@ -106,25 +99,28 @@ func createEmbedSource(tgt target) {
 			"// This product includes software developed at Datadog (https://www.datadoghq.com/).",
 			"// Copyright 2016-present Datadog, Inc.",
 			"",
-			fmt.Sprintf("//go:build %s && %s && !go1.22", tgt.os, tgt.arch),
+			tgt.buildConstraintDirective(),
+			"",
 			"package lib",
 			"",
-			`import _ "embed" // Needed for go:embed`,
+			"// THIS FILE IS AUTOGENERATED. DO NOT EDIT.",
 			"",
-			fmt.Sprintf("//go:embed %s-%s/%s.%s", tgt.os, tgt.arch, tgt.base, tgt.ext),
+			"import _ \"embed\" // Needed for go:embed",
+			"",
+			tgt.embedSourceDirective(),
 			"var libddwaf []byte",
 			"",
-			fmt.Sprintf(`const embedNamePattern = "%s-*.%s"`, tgt.base, tgt.ext),
+			tgt.tempFilePatternStatement(),
 			"", // Trailing new line...
 		},
 		"\n",
 	)
-	if err := os.WriteFile(path.Join(libDir, fmt.Sprintf("lib_%s_%s.go", tgt.os, tgt.arch)), []byte(gosource), 0644); err != nil {
+	if err := os.WriteFile(path.Join(libDir, tgt.embedSourceFilename()), []byte(gosource), 0644); err != nil {
 		panic(err)
 	}
 }
 
-func handleTarget(wg *sync.WaitGroup, version string, tgt target, embedDir string, assets map[string]*github.ReleaseAsset) {
+func handleTarget(wg *sync.WaitGroup, version string, tgt target, assets map[string]*github.ReleaseAsset, force bool) {
 	defer wg.Done()
 
 	tarName := fmt.Sprintf("libddwaf-%s-%s.tar.gz", version, tgt.assetLabel)
@@ -187,15 +183,14 @@ func handleTarget(wg *sync.WaitGroup, version string, tgt target, embedDir strin
 		var destPath string
 		switch name := header.FileInfo().Name(); name {
 		case "libddwaf.so", "libddwaf.dylib", "ddwaf.dll":
-			destPath = path.Join(embedDir, name)
+			destPath = path.Join(libDir, tgt.binaryLibName())
 			foundLib = true
 		case "ddwaf.h":
-			if tgt.primary {
-				destPath = path.Join(rootDir, "include", name)
-				foundHdr = true
-			} else {
+			if !tgt.primary {
 				continue
 			}
+			destPath = path.Join(rootDir, "include", name)
+			foundHdr = true
 		default:
 			continue
 		}
@@ -217,11 +212,16 @@ func handleTarget(wg *sync.WaitGroup, version string, tgt target, embedDir strin
 	}
 
 	if !foundLib {
-		panic(fmt.Errorf("could not find libddwaf.so/libddwaf.dylib in %s", tarUrl))
+		panic(fmt.Errorf("could not find libddwaf.%s in %s", tgt.ext, tarUrl))
 	}
 	if tgt.primary && !foundHdr {
 		panic(fmt.Errorf("could not find ddwaf.h in %s", tarUrl))
 	}
+
+	// If the embed source file doesn't exist, or if --force is set, create it.
+	if _, err = os.Stat(path.Join(libDir, tgt.embedSourceFilename())); errors.Is(err, os.ErrNotExist) || force {
+		createEmbedSource(tgt)
+	}
 }
 
 type target struct {
@@ -273,16 +273,25 @@ var targets = []target{
 	// {os: "linux", arch: "i386", ext: "so", assetLabel: "i386-linux-musl"},
 }
 
-const vendorMarkerFile = `// Unless explicitly stated otherwise all files in this repository are licensed
-// under the Apache License Version 2.0.
-// This product includes software developed at Datadog (https://www.datadoghq.com/).
-// Copyright 2016-present Datadog, Inc.
+func (t target) binaryLibName() string {
+	return fmt.Sprintf("%s-%s-%s.%s", t.base, t.os, t.arch, t.ext)
+}
 
-package vendor
+func (t target) embedSourceFilename() string {
+	return fmt.Sprintf("lib_%s_%s.go", t.os, t.arch)
+}
 
-// This file is there to make sure "go mod vendor" does not drop those directories, as it otherwise removes
-// any directiory that contains no ".go" files.
-`
+func (t target) buildConstraintDirective() string {
+	return fmt.Sprintf("//go:build %s && %s && !%s && !datadog.no_waf", t.os, t.arch, goVersionUnsupported)
+}
+
+func (t target) tempFilePatternStatement() string {
+	return fmt.Sprintf("const embedNamePattern = \"%s-*.%s\"", t.base, t.ext)
+}
+
+func (t target) embedSourceDirective() string {
+	return fmt.Sprintf("//go:embed %s", t.binaryLibName())
+}
 
 func init() {
 	_, filename, _, _ := runtime.Caller(0)

internal/lib/darwin-amd64/vendor.go

@@ -7,9 +7,11 @@
 
 package lib
 
+// THIS FILE IS AUTOGENERATED. DO NOT EDIT.
+
 import _ "embed" // Needed for go:embed
 
-//go:embed darwin-amd64/libddwaf.dylib
+//go:embed libddwaf-darwin-amd64.dylib
 var libddwaf []byte
 
 const embedNamePattern = "libddwaf-*.dylib"

internal/lib/darwin-arm64/vendor.go

@@ -7,9 +7,11 @@
 
 package lib
 
+// THIS FILE IS AUTOGENERATED. DO NOT EDIT.
+
 import _ "embed" // Needed for go:embed
 
-//go:embed darwin-arm64/libddwaf.dylib
+//go:embed libddwaf-darwin-arm64.dylib
 var libddwaf []byte
 
 const embedNamePattern = "libddwaf-*.dylib"

internal/lib/lib_darwin_amd64.go

@@ -7,9 +7,11 @@
 
 package lib
 
+// THIS FILE IS AUTOGENERATED. DO NOT EDIT.
+
 import _ "embed" // Needed for go:embed
 
-//go:embed linux-amd64/libddwaf.so
+//go:embed libddwaf-linux-amd64.so
 var libddwaf []byte
 
 const embedNamePattern = "libddwaf-*.so"

internal/lib/lib_darwin_arm64.go

@@ -7,9 +7,11 @@
 
 package lib
 
+// THIS FILE IS AUTOGENERATED. DO NOT EDIT.
+
 import _ "embed" // Needed for go:embed
 
-//go:embed linux-arm64/libddwaf.so
+//go:embed libddwaf-linux-arm64.so
 var libddwaf []byte
 
 const embedNamePattern = "libddwaf-*.so"