feat(caching): content based caching to discovery, loader, output

Author: cmilesio

.github/workflows/tests.yml

@@ -20,7 +20,10 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
         go-version: [1.25.x]
+        cache-mode: [mtime, content]
     runs-on: ${{ matrix.os }}
+    env:
+      WIRE_CACHE_MODE: ${{ matrix.cache-mode }}
     steps:
       - name: Install Go
         uses: actions/setup-go@v2
@@ -34,9 +37,6 @@ jobs:
         shell: bash
         run: 'internal/runtests.sh'
 
-      - name: Run tests
-        run: go test ./... -v
-
   codecov:
     runs-on: ubuntu-latest
 

internal/cachepolicy/cachepolicy.go

@@ -0,0 +1,40 @@
+package cachepolicy
+
+import "os"
+
+const (
+	ModeEnv     = "WIRE_CACHE_MODE"
+	ModeMTime   = "mtime"
+	ModeContent = "content"
+)
+
+var mode = loadMode()
+
+func UseFileContent() bool {
+	return mode == ModeContent
+}
+
+func ModeLabel() string {
+	return mode
+}
+
+func SetForTest(next string) func() {
+	prev := mode
+	mode = normalizeMode(next)
+	return func() {
+		mode = prev
+	}
+}
+
+func loadMode() string {
+	return normalizeMode(os.Getenv(ModeEnv))
+}
+
+func normalizeMode(value string) string {
+	switch value {
+	case ModeContent:
+		return ModeContent
+	default:
+		return ModeMTime
+	}
+}

internal/loader/artifact_cache.go

@@ -24,10 +24,12 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 
 	"golang.org/x/tools/go/gcexportdata"
 
 	"github.com/goforj/wire/internal/cachepaths"
+	"github.com/goforj/wire/internal/cachepolicy"
 )
 
 const (
@@ -57,25 +59,25 @@ func loaderArtifactPath(env []string, meta *packageMeta, isLocal bool) (string,
 
 func loaderArtifactKey(meta *packageMeta, isLocal bool) (string, error) {
 	sum := sha256.New()
-	sum.Write([]byte("wire-loader-artifact-v4\n"))
+	sum.Write([]byte("wire-loader-artifact-v5\n"))
 	sum.Write([]byte(runtime.Version()))
 	sum.Write([]byte{'\n'})
+	sum.Write([]byte(cachepolicy.ModeLabel()))
+	sum.Write([]byte{'\n'})
 	sum.Write([]byte(meta.ImportPath))
 	sum.Write([]byte{'\n'})
 	sum.Write([]byte(meta.Name))
 	sum.Write([]byte{'\n'})
+	useFileContent := cachepolicy.UseFileContent()
 	if !isLocal {
 		sum.Write([]byte(meta.Export))
 		sum.Write([]byte{'\n'})
 		if meta.Export != "" {
-			h, err := hashFileContent(meta.Export)
-			if err != nil {
+			if err := hashFileInput(sum, meta.Export, useFileContent); err != nil {
 				return "", err
 			}
-			sum.Write([]byte(h))
-			sum.Write([]byte{'\n'})
 		} else {
-			if err := hashMetaFiles(sum, metaFiles(meta)); err != nil {
+			if err := hashMetaFiles(sum, metaFiles(meta), useFileContent); err != nil {
 				return "", err
 			}
 		}
@@ -85,7 +87,7 @@ func loaderArtifactKey(meta *packageMeta, isLocal bool) (string, error) {
 		}
 		return hex.EncodeToString(sum.Sum(nil)), nil
 	}
-	if err := hashMetaFiles(sum, metaFiles(meta)); err != nil {
+	if err := hashMetaFiles(sum, metaFiles(meta), useFileContent); err != nil {
 		return "", err
 	}
 	return hex.EncodeToString(sum.Sum(nil)), nil
@@ -101,18 +103,36 @@ func hashFileContent(path string) (string, error) {
 	return hex.EncodeToString(h[:]), nil
 }
 
-// hashMetaFiles writes content-based hashes for each file into sum.
-func hashMetaFiles(sum io.Writer, names []string) error {
+// hashMetaFiles writes file identity inputs for each file into sum.
+func hashMetaFiles(sum io.Writer, names []string, useFileContent bool) error {
 	for _, name := range names {
 		sum.Write([]byte(name))
 		sum.Write([]byte{'\n'})
-		h, err := hashFileContent(name)
+		if err := hashFileInput(sum, name, useFileContent); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func hashFileInput(sum io.Writer, path string, useFileContent bool) error {
+	if useFileContent {
+		h, err := hashFileContent(path)
 		if err != nil {
 			return err
 		}
 		sum.Write([]byte(h))
 		sum.Write([]byte{'\n'})
+		return nil
+	}
+	info, err := os.Stat(path)
+	if err != nil {
+		return err
 	}
+	sum.Write([]byte(strconv.FormatInt(info.Size(), 10)))
+	sum.Write([]byte{'\n'})
+	sum.Write([]byte(strconv.FormatInt(info.ModTime().UnixNano(), 10)))
+	sum.Write([]byte{'\n'})
 	return nil
 }
 

internal/loader/discovery_cache.go

@@ -13,6 +13,7 @@ import (
 	"sort"
 
 	"github.com/goforj/wire/internal/cachepaths"
+	"github.com/goforj/wire/internal/cachepolicy"
 )
 
 type discoveryCacheEntry struct {
@@ -32,7 +33,7 @@ type discoveryLocalPackage struct {
 type discoveryFileMeta struct {
 	Path        string
 	Size        int64
-	ModTime     int64  // deprecated: kept for gob compat, not used for matching
+	ModTime     int64
 	ContentHash string // sha256 of file content
 	IsDir       bool
 }
@@ -47,7 +48,7 @@ type discoveryFileFingerprint struct {
 	Hash string
 }
 
-const discoveryCacheVersion = 4
+const discoveryCacheVersion = 5
 
 func readDiscoveryCache(req goListRequest) (map[string]*packageMeta, bool) {
 	entry, err := loadDiscoveryCacheEntry(req)
@@ -62,6 +63,7 @@ func readDiscoveryCache(req goListRequest) (map[string]*packageMeta, bool) {
 
 func buildDiscoveryCacheEntry(req goListRequest, meta map[string]*packageMeta) (*discoveryCacheEntry, error) {
 	workspace := detectModuleRoot(req.WD)
+	useFileContent := cachepolicy.UseFileContent()
 	entry := &discoveryCacheEntry{
 		Version: discoveryCacheVersion,
 		Meta:    meta,
@@ -73,7 +75,7 @@ func buildDiscoveryCacheEntry(req goListRequest, meta map[string]*packageMeta) (
 		filepath.Join(workspace, "go.work.sum"),
 	}
 	for _, name := range global {
-		if fm, ok := statDiscoveryFile(name); ok {
+		if fm, ok := statDiscoveryFile(name, useFileContent); ok {
 			entry.Global = append(entry.Global, fm)
 		}
 	}
@@ -106,8 +108,9 @@ func validateDiscoveryCacheEntry(entry *discoveryCacheEntry) bool {
 	if entry == nil || entry.Version != discoveryCacheVersion {
 		return false
 	}
+	useFileContent := cachepolicy.UseFileContent()
 	for _, fm := range entry.Global {
-		if !matchesDiscoveryFile(fm) {
+		if !matchesDiscoveryFile(fm, useFileContent) {
 			return false
 		}
 	}
@@ -133,6 +136,7 @@ func discoveryCachePath(req goListRequest) (string, error) {
 	}
 	sumReq := struct {
 		Version      int
+		CacheMode    string
 		WD           string
 		Tags         string
 		Patterns     []string
@@ -141,6 +145,7 @@ func discoveryCachePath(req goListRequest) (string, error) {
 		Go           string
 	}{
 		Version:      discoveryCacheVersion,
+		CacheMode:    cachepolicy.ModeLabel(),
 		WD:           canonicalLoaderPath(req.WD),
 		Tags:         req.Tags,
 		Patterns:     append([]string(nil), req.Patterns...),
@@ -188,13 +193,13 @@ func saveDiscoveryCacheEntry(req goListRequest, entry *discoveryCacheEntry) erro
 	return gob.NewEncoder(f).Encode(entry)
 }
 
-func statDiscoveryFile(path string) (discoveryFileMeta, bool) {
+func statDiscoveryFile(path string, useFileContent bool) (discoveryFileMeta, bool) {
 	info, err := os.Stat(path)
 	if err != nil {
 		return discoveryFileMeta{}, false
 	}
 	h := ""
-	if !info.IsDir() {
+	if !info.IsDir() && useFileContent {
 		var err error
 		h, err = hashFileContent(path)
 		if err != nil {
@@ -204,17 +209,24 @@ func statDiscoveryFile(path string) (discoveryFileMeta, bool) {
 	return discoveryFileMeta{
 		Path:        canonicalLoaderPath(path),
 		Size:        info.Size(),
+		ModTime:     info.ModTime().UnixNano(),
 		ContentHash: h,
 		IsDir:       info.IsDir(),
 	}, true
 }
 
-func matchesDiscoveryFile(fm discoveryFileMeta) bool {
-	cur, ok := statDiscoveryFile(fm.Path)
+func matchesDiscoveryFile(fm discoveryFileMeta, useFileContent bool) bool {
+	cur, ok := statDiscoveryFile(fm.Path, useFileContent)
 	if !ok {
 		return false
 	}
-	return cur.ContentHash == fm.ContentHash && cur.IsDir == fm.IsDir
+	if cur.IsDir != fm.IsDir {
+		return false
+	}
+	if useFileContent {
+		return cur.ContentHash == fm.ContentHash
+	}
+	return cur.Size == fm.Size && cur.ModTime == fm.ModTime
 }
 
 func statDiscoveryDir(path string) (discoveryDirMeta, bool) {

internal/loader/discovery_cache_test.go

@@ -4,6 +4,8 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/goforj/wire/internal/cachepolicy"
 )
 
 func TestDiscoveryFingerprintIgnoresBodyOnlyEdits(t *testing.T) {
@@ -124,3 +126,67 @@ func TestDiscoveryDirDetectsFileSetChange(t *testing.T) {
 		t.Fatalf("directory metadata did not detect added file")
 	}
 }
+
+func TestMatchesDiscoveryFileDefaultModTimeIgnoresContentOnlyChangeWithSameMetadata(t *testing.T) {
+	restore := cachepolicy.SetForTest(cachepolicy.ModeMTime)
+	defer restore()
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "go.mod")
+	write := func(content string) {
+		if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	write("module example.com/one\n")
+	info, err := os.Stat(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	meta, ok := statDiscoveryFile(path, false)
+	if !ok {
+		t.Fatalf("statDiscoveryFile(%q) failed", path)
+	}
+
+	write("module example.com/two\n")
+	if err := os.Chtimes(path, info.ModTime(), info.ModTime()); err != nil {
+		t.Fatal(err)
+	}
+
+	if !matchesDiscoveryFile(meta, false) {
+		t.Fatalf("default mod-time matching rejected unchanged size/modtime metadata")
+	}
+}
+
+func TestMatchesDiscoveryFileContentModeDetectsContentOnlyChangeWithSameMetadata(t *testing.T) {
+	restore := cachepolicy.SetForTest(cachepolicy.ModeContent)
+	defer restore()
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "go.mod")
+	write := func(content string) {
+		if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	write("module example.com/one\n")
+	info, err := os.Stat(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	meta, ok := statDiscoveryFile(path, true)
+	if !ok {
+		t.Fatalf("statDiscoveryFile(%q) failed", path)
+	}
+
+	write("module example.com/two\n")
+	if err := os.Chtimes(path, info.ModTime(), info.ModTime()); err != nil {
+		t.Fatal(err)
+	}
+
+	if matchesDiscoveryFile(meta, true) {
+		t.Fatalf("content-based matching accepted changed file content")
+	}
+}