Refactor syz-declextract in preparation for codesearch tool

Makefile

@@ -272,7 +272,8 @@ format_cpp:
 	clang-format --style=file -i executor/*.cc executor/*.h \
 		executor/android/android_seccomp.h \
 		tools/kcovtrace/*.c tools/kcovfuzzer/*.c tools/fops_probe/*.cc \
-		tools/syz-declextract/clangtool/*.cpp tools/syz-declextract/clangtool/*.h
+		tools/clang/*.h \
+		tools/clang/declextract/*.h tools/clang/declextract/*.cpp
 
 format_sys: bin/syz-fmt
 	bin/syz-fmt all

pkg/clangtool/clangtool.go

@@ -5,6 +5,7 @@ package clangtool
 
 import (
 	"bytes"
+	"crypto/sha256"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -18,7 +19,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/google/syzkaller/pkg/declextract"
 	"github.com/google/syzkaller/pkg/osutil"
 )
 
@@ -30,17 +30,21 @@ type Config struct {
 	DebugTrace io.Writer
 }
 
+type OutputDataPtr[T any] interface {
+	*T
+	Merge(*T)
+	SetSourceFile(string, func(filename string) string)
+	SortAndDedup()
+}
+
 // Run runs the clang tool on all files in the compilation database
 // in the kernel build dir and returns combined output for all files.
 // It always caches results, and optionally reuses previously cached results.
-func Run(cfg *Config) (*declextract.Output, error) {
+func Run[Output any, OutputPtr OutputDataPtr[Output]](cfg *Config) (OutputPtr, error) {
 	if cfg.CacheFile != "" {
-		data, err := os.ReadFile(cfg.CacheFile)
+		out, err := osutil.ReadJSON[OutputPtr](cfg.CacheFile)
 		if err == nil {
-			out, err := unmarshal(data)
-			if err == nil {
-				return out, nil
-			}
+			return out, nil
 		}
 	}
 
@@ -51,15 +55,15 @@ func Run(cfg *Config) (*declextract.Output, error) {
 	}
 
 	type result struct {
-		out *declextract.Output
+		out OutputPtr
 		err error
 	}
 	results := make(chan *result, 10)
 	files := make(chan string, len(cmds))
 	for w := 0; w < runtime.NumCPU(); w++ {
 		go func() {
 			for file := range files {
-				out, err := runTool(cfg, dbFile, file)
+				out, err := runTool[Output, OutputPtr](cfg, dbFile, file)
 				results <- &result{out, err}
 			}
 		}()
@@ -69,7 +73,7 @@ func Run(cfg *Config) (*declextract.Output, error) {
 	}
 	close(files)
 
-	out := new(declextract.Output)
+	out := OutputPtr(new(Output))
 	for range cmds {
 		res := <-results
 		if res.err != nil {
@@ -91,7 +95,7 @@ func Run(cfg *Config) (*declextract.Output, error) {
 	return out, nil
 }
 
-func runTool(cfg *Config, dbFile, file string) (*declextract.Output, error) {
+func runTool[Output any, OutputPtr OutputDataPtr[Output]](cfg *Config, dbFile, file string) (OutputPtr, error) {
 	relFile := strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(filepath.Clean(file),
 		cfg.KernelSrc), cfg.KernelObj), "/")
 	// Suppress warning since we may build the tool on a different clang
@@ -104,33 +108,20 @@ func runTool(cfg *Config, dbFile, file string) (*declextract.Output, error) {
 		}
 		return nil, err
 	}
-	out, err := unmarshal(data)
+	out, err := osutil.ParseJSON[OutputPtr](data)
 	if err != nil {
 		return nil, err
 	}
-	fixupFileNames(cfg, out, relFile)
-	return out, nil
-}
-
-func unmarshal(data []byte) (*declextract.Output, error) {
-	dec := json.NewDecoder(bytes.NewReader(data))
-	dec.DisallowUnknownFields()
-	out := new(declextract.Output)
-	if err := dec.Decode(out); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal clang tool output: %w\n%s", err, data)
-	}
-	return out, nil
-}
-
-func fixupFileNames(cfg *Config, out *declextract.Output, file string) {
 	// All includes in the tool output are relative to the build dir.
 	// Make them relative to the source dir.
-	out.SetSourceFile(file, func(filename string) string {
-		if res, err := filepath.Rel(cfg.KernelSrc, filepath.Join(cfg.KernelObj, filename)); err == nil {
-			return res
+	out.SetSourceFile(relFile, func(filename string) string {
+		rel, err := filepath.Rel(cfg.KernelSrc, filepath.Join(cfg.KernelObj, filename))
+		if err == nil && filename != "" {
+			return rel
 		}
 		return filename
 	})
+	return out, nil
 }
 
 type compileCommand struct {
@@ -170,3 +161,21 @@ func loadCompileCommands(dbFile string) ([]compileCommand, error) {
 	}
 	return cmds, nil
 }
+
+func SortAndDedupSlice[Slice ~[]E, E comparable](s Slice) Slice {
+	dedup := make(map[[sha256.Size]byte]E)
+	text := make(map[E][]byte)
+	for _, e := range s {
+		t, _ := json.Marshal(e)
+		dedup[sha256.Sum256(t)] = e
+		text[e] = t
+	}
+	s = make([]E, 0, len(dedup))
+	for _, e := range dedup {
+		s = append(s, e)
+	}
+	slices.SortFunc(s, func(a, b E) int {
+		return bytes.Compare(text[a], text[b])
+	})
+	return s
+}

pkg/clangtool/tooltest/tooltest.go

@@ -0,0 +1,116 @@
+// Copyright 2025 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package tooltest
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/syzkaller/pkg/clangtool"
+	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/pkg/testutil"
+)
+
+var (
+	FlagBin    = flag.String("bin", "", "path to the clang tool binary to use")
+	FlagUpdate = flag.Bool("update", false, "update golden files")
+)
+
+func TestClangTool[Output any, OutputPtr clangtool.OutputDataPtr[Output]](t *testing.T) {
+	if *FlagBin == "" {
+		t.Skipf("clang tool path is not specified, run with -bin=clangtool flag")
+	}
+	ForEachTestFile(t, func(t *testing.T, cfg *clangtool.Config, file string) {
+		out, err := clangtool.Run[Output, OutputPtr](cfg)
+		if err != nil {
+			t.Fatal(err)
+		}
+		got, err := json.MarshalIndent(out, "", "\t")
+		if err != nil {
+			t.Fatal(err)
+		}
+		CompareGoldenData(t, file+".json", got)
+	})
+}
+
+func LoadOutput[Output any, OutputPtr clangtool.OutputDataPtr[Output]](t *testing.T) OutputPtr {
+	out := OutputPtr(new(Output))
+	forEachTestFile(t, func(t *testing.T, file string) {
+		tmp, err := osutil.ReadJSON[OutputPtr](file + ".json")
+		if err != nil {
+			t.Fatal(err)
+		}
+		out.Merge(tmp)
+	})
+	out.SortAndDedup()
+	return out
+}
+
+func ForEachTestFile(t *testing.T, fn func(t *testing.T, cfg *clangtool.Config, file string)) {
+	forEachTestFile(t, func(t *testing.T, file string) {
+		t.Run(filepath.Base(file), func(t *testing.T) {
+			t.Parallel()
+			buildDir := t.TempDir()
+			commands := fmt.Sprintf(`[{
+					"file": "%s",
+					"directory": "%s",
+					"command": "clang -c %s -DKBUILD_BASENAME=foo"
+				}]`,
+				file, buildDir, file)
+			dbFile := filepath.Join(buildDir, "compile_commands.json")
+			if err := os.WriteFile(dbFile, []byte(commands), 0600); err != nil {
+				t.Fatal(err)
+			}
+			cfg := &clangtool.Config{
+				ToolBin:    *FlagBin,
+				KernelSrc:  osutil.Abs("testdata"),
+				KernelObj:  buildDir,
+				CacheFile:  filepath.Join(buildDir, filepath.Base(file)+".json"),
+				DebugTrace: &testutil.Writer{TB: t},
+			}
+			fn(t, cfg, file)
+		})
+	})
+}
+
+func forEachTestFile(t *testing.T, fn func(t *testing.T, file string)) {
+	files, err := filepath.Glob(filepath.Join(osutil.Abs("testdata"), "*.c"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(files) == 0 {
+		t.Fatal("found no source files")
+	}
+	for _, file := range files {
+		fn(t, file)
+	}
+}
+
+func CompareGoldenFile(t *testing.T, goldenFile, gotFile string) {
+	got, err := os.ReadFile(gotFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	CompareGoldenData(t, goldenFile, got)
+}
+
+func CompareGoldenData(t *testing.T, goldenFile string, got []byte) {
+	if *FlagUpdate {
+		if err := os.WriteFile(goldenFile, got, 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	want, err := os.ReadFile(goldenFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Fatal(diff)
+	}
+}

pkg/declextract/declextract.go

@@ -12,6 +12,7 @@ import (
 	"slices"
 	"strings"
 
+	"github.com/google/syzkaller/pkg/clangtool"
 	"github.com/google/syzkaller/pkg/cover"
 	"github.com/google/syzkaller/pkg/ifaceprobe"
 )
@@ -136,8 +137,8 @@ func (ctx *context) processConsts() map[string]string {
 			Value: fmt.Sprint(ci.Value),
 		})
 	}
-	ctx.includes = sortAndDedupSlice(ctx.includes)
-	ctx.defines = sortAndDedupSlice(ctx.defines)
+	ctx.includes = clangtool.SortAndDedupSlice(ctx.includes)
+	ctx.defines = clangtool.SortAndDedupSlice(ctx.defines)
 	// These additional includes must be at the top, because other kernel headers
 	// are broken and won't compile without these additional ones included first.
 	ctx.includes = append([]string{
@@ -194,7 +195,7 @@ func (ctx *context) processSyscalls() {
 		}
 		ctx.emitSyscall(&syscalls, call, "", "", -1, "")
 	}
-	ctx.Syscalls = sortAndDedupSlice(syscalls)
+	ctx.Syscalls = clangtool.SortAndDedupSlice(syscalls)
 }
 
 func (ctx *context) emitSyscall(syscalls *[]*Syscall, call *Syscall,

pkg/declextract/entity.go

@@ -4,11 +4,9 @@
 package declextract
 
 import (
-	"bytes"
-	"crypto/sha256"
-	"encoding/json"
-	"slices"
 	"strings"
+
+	"github.com/google/syzkaller/pkg/clangtool"
 )
 
 type Output struct {
@@ -244,16 +242,16 @@ func (out *Output) Merge(other *Output) {
 }
 
 func (out *Output) SortAndDedup() {
-	out.Functions = sortAndDedupSlice(out.Functions)
-	out.Consts = sortAndDedupSlice(out.Consts)
-	out.Enums = sortAndDedupSlice(out.Enums)
-	out.Structs = sortAndDedupSlice(out.Structs)
-	out.Syscalls = sortAndDedupSlice(out.Syscalls)
-	out.FileOps = sortAndDedupSlice(out.FileOps)
-	out.Ioctls = sortAndDedupSlice(out.Ioctls)
-	out.IouringOps = sortAndDedupSlice(out.IouringOps)
-	out.NetlinkFamilies = sortAndDedupSlice(out.NetlinkFamilies)
-	out.NetlinkPolicies = sortAndDedupSlice(out.NetlinkPolicies)
+	out.Functions = clangtool.SortAndDedupSlice(out.Functions)
+	out.Consts = clangtool.SortAndDedupSlice(out.Consts)
+	out.Enums = clangtool.SortAndDedupSlice(out.Enums)
+	out.Structs = clangtool.SortAndDedupSlice(out.Structs)
+	out.Syscalls = clangtool.SortAndDedupSlice(out.Syscalls)
+	out.FileOps = clangtool.SortAndDedupSlice(out.FileOps)
+	out.Ioctls = clangtool.SortAndDedupSlice(out.Ioctls)
+	out.IouringOps = clangtool.SortAndDedupSlice(out.IouringOps)
+	out.NetlinkFamilies = clangtool.SortAndDedupSlice(out.NetlinkFamilies)
+	out.NetlinkPolicies = clangtool.SortAndDedupSlice(out.NetlinkPolicies)
 }
 
 // SetSoureFile attaches the source file to the entities that need it.
@@ -285,21 +283,3 @@ func (out *Output) SetSourceFile(file string, updatePath func(string) string) {
 		op.SourceFile = file
 	}
 }
-
-func sortAndDedupSlice[Slice ~[]E, E comparable](s Slice) Slice {
-	dedup := make(map[[sha256.Size]byte]E)
-	text := make(map[E][]byte)
-	for _, e := range s {
-		t, _ := json.Marshal(e)
-		dedup[sha256.Sum256(t)] = e
-		text[e] = t
-	}
-	s = make([]E, 0, len(dedup))
-	for _, e := range dedup {
-		s = append(s, e)
-	}
-	slices.SortFunc(s, func(a, b E) int {
-		return bytes.Compare(text[a], text[b])
-	})
-	return s
-}

pkg/declextract/fileops.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/google/syzkaller/pkg/ast"
+	"github.com/google/syzkaller/pkg/clangtool"
 )
 
 const (
@@ -243,7 +244,7 @@ func (ctx *context) mapFileToFops(funcs map[*Function]bool, funcToFops map[*Func
 			best = append(best, fops)
 		}
 	}
-	best = sortAndDedupSlice(best)
+	best = clangtool.SortAndDedupSlice(best)
 	// Now, filter out some excessive file_operations.
 	// An example of an excessive case is if we have 2 file_operations with just read+write,
 	// currently we emit generic read/write operations, so we would emit completly equal