fix zstash for Windows CI

Author: zhming0

.bk.yaml

@@ -0,0 +1 @@
+selected_org: buildkite

internal/zstash/archive/.gitattributes

@@ -0,0 +1 @@
+testdata/** -text

internal/zstash/archive/create_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"runtime"
 	"slices"
 	"strings"
 	"testing"
@@ -12,6 +13,12 @@ import (
 )
 
 func TestBuildArchive(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		// The expected sha256/size encode unix file modes and LF line
+		// endings, which differ on Windows.
+		t.Skip("archive byte layout is platform-specific")
+	}
+
 	_, err := trace.NewProvider(context.Background(), "noop", "test", "0.0.1")
 	if err != nil {
 		t.Fatalf("trace.NewProvider: %v", err)
@@ -22,7 +29,7 @@ func TestBuildArchive(t *testing.T) {
 		t.Fatalf("os.Getwd: %v", err)
 	}
 
-	t.Setenv("HOME", home)
+	setHomeDir(t, home)
 
 	archiveInfo, err := BuildArchive(context.Background(), []string{"testdata"}, "test")
 	if err != nil {
@@ -51,7 +58,7 @@ func TestBuildAndExtractArchive_MultipleHomeDirPaths(t *testing.T) {
 	}
 
 	home := t.TempDir()
-	t.Setenv("HOME", home)
+	setHomeDir(t, home)
 
 	goBuildDir := filepath.Join(home, ".go-build")
 	err = os.MkdirAll(goBuildDir, 0o755)
@@ -168,7 +175,7 @@ func TestBuildArchive_MissingPathOnFilesystem(t *testing.T) {
 	}
 
 	home := t.TempDir()
-	t.Setenv("HOME", home)
+	setHomeDir(t, home)
 
 	goBuildDir := filepath.Join(home, ".go-build")
 	err = os.MkdirAll(goBuildDir, 0o755)
@@ -220,7 +227,7 @@ func TestExtractArchive_MissingPathInArchive(t *testing.T) {
 	}
 
 	home := t.TempDir()
-	t.Setenv("HOME", home)
+	setHomeDir(t, home)
 
 	goBuildDir := filepath.Join(home, ".go-build")
 	err = os.MkdirAll(goBuildDir, 0o755)

internal/zstash/archive/extract.go

@@ -55,8 +55,12 @@ func ExtractFiles(ctx context.Context, zipFile *os.File, zipFileLen int64, paths
 	foundPaths := make(map[string]bool)
 
 	err = extract.ExtractWithPathMapper(ctx, func(file *zip.File) (string, error) {
+		// Zip entry names always use forward slashes (per the zip spec),
+		// but mapping.RelativePath comes from filepath.Rel and may use the
+		// OS native separator (backslash on Windows). Normalise the mapping
+		// to forward slashes for the comparison.
 		for _, mapping := range mappings {
-			if strings.HasPrefix(file.Name, mapping.RelativePath) {
+			if strings.HasPrefix(file.Name, filepath.ToSlash(mapping.RelativePath)) {
 				foundPaths[mapping.Path] = true
 				return filepath.Join(mapping.Chroot, file.Name), nil
 			}

internal/zstash/archive/helpers_test.go

@@ -0,0 +1,13 @@
+package archive
+
+import "testing"
+
+// setHomeDir overrides the home directory used by os.UserHomeDir() for the
+// duration of the test. On unix-like systems os.UserHomeDir() reads $HOME,
+// while on Windows it reads %USERPROFILE%; set both so callers don't have
+// to care about the host platform.
+func setHomeDir(t *testing.T, home string) {
+	t.Helper()
+	t.Setenv("HOME", home)
+	t.Setenv("USERPROFILE", home)
+}

internal/zstash/archive/mappings_test.go

@@ -8,7 +8,7 @@ import (
 
 func TestPathsToMappings_AbsolutePathUnderHome(t *testing.T) {
 	home := t.TempDir()
-	t.Setenv("HOME", home)
+	setHomeDir(t, home)
 
 	paths := []string{
 		filepath.Join(home, ".go-build"),

internal/zstash/cache_integration_test.go

@@ -270,18 +270,25 @@ func setupTestCache(t *testing.T, storageType string) (cacheClient *Cache, cache
 	// Create mock API client
 	mockClient := newMockAPIClient(storageType)
 
-	// Build storage URL based on type (need absolute path for file:// URLs)
+	// Build storage URL based on type (need absolute path for file:// URLs).
+	// On Windows, paths like "C:\foo" must be encoded as "/C:/foo" so the
+	// resulting URL is the well-formed "file:///C:/foo".
 	absStorageDir, err := filepath.Abs(storageDir)
 	if err != nil {
 		t.Fatalf("filepath.Abs: %v", err)
 	}
 
+	urlPath := filepath.ToSlash(absStorageDir)
+	if !strings.HasPrefix(urlPath, "/") {
+		urlPath = "/" + urlPath
+	}
+
 	var bucketURL string
 	switch storageType {
 	case "local_file":
-		bucketURL = fmt.Sprintf("file://%s", absStorageDir)
+		bucketURL = fmt.Sprintf("file://%s", urlPath)
 	case "local_s3":
-		bucketURL = fmt.Sprintf("file://%s", absStorageDir) // Use file:// for testing
+		bucketURL = fmt.Sprintf("file://%s", urlPath) // Use file:// for testing
 	default:
 		t.Fatalf("unsupported storage type: %s", storageType)
 	}

internal/zstash/restore.go

@@ -331,7 +331,28 @@ func cleanPath(ctx context.Context, dir string) error {
 	}
 
 	// Module cache has 0555 directories; make them writable in order to remove content.
-	// Use os.Root to confine chmod operations within `clean` and prevent symlink TOCTOU traversal.
+	if err := makeTreeWritable(ctx, clean); err != nil {
+		return err
+	}
+
+	// Check context again before potentially long RemoveAll
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+
+	if err := os.RemoveAll(clean); err != nil {
+		return fmt.Errorf("cleanPath: failed to remove %q: %w", clean, err)
+	}
+
+	return nil
+}
+
+// makeTreeWritable walks `clean` and chmods every directory to 0755 so that
+// the subsequent os.RemoveAll can delete read-only entries (e.g. Go module
+// cache). The os.Root handle is closed before returning so that the caller
+// can remove `clean` on platforms (Windows) that disallow removing a
+// directory with an open handle.
+func makeTreeWritable(ctx context.Context, clean string) error {
 	root, err := os.OpenRoot(clean)
 	if err != nil {
 		if errors.Is(err, fs.ErrNotExist) {
@@ -342,7 +363,6 @@ func cleanPath(ctx context.Context, dir string) error {
 	defer func() { _ = root.Close() }()
 
 	err = fs.WalkDir(root.FS(), ".", func(relPath string, info fs.DirEntry, walkErr error) error {
-		// Respect context cancellation for long directory trees
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
@@ -367,15 +387,5 @@ func cleanPath(ctx context.Context, dir string) error {
 		}
 		return fmt.Errorf("cleanPath: error preparing %q for removal: %w", clean, err)
 	}
-
-	// Check context again before potentially long RemoveAll
-	if ctx.Err() != nil {
-		return ctx.Err()
-	}
-
-	if err := os.RemoveAll(clean); err != nil {
-		return fmt.Errorf("cleanPath: failed to remove %q: %w", clean, err)
-	}
-
 	return nil
 }

internal/zstash/store/file.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"strings"
 	"time"
 
@@ -92,6 +93,12 @@ func NewLocalFileBlob(ctx context.Context, fileURL string) (*LocalFileBlob, erro
 		path = filepath.Join(homeDir, path)
 	}
 
+	// A Windows file URL like "file:///C:/foo/bar" parses to u.Path = "/C:/foo/bar".
+	// Strip the spurious leading "/" so it becomes a valid OS path "C:/foo/bar".
+	if runtime.GOOS == "windows" && len(path) >= 3 && path[0] == '/' && path[2] == ':' {
+		path = path[1:]
+	}
+
 	root := filepath.Clean(filepath.FromSlash(path))
 	if root == "" || root == "/" || root == "." {
 		return nil, fmt.Errorf("invalid root directory: %s", root)

internal/zstash/store/file_test.go

@@ -155,7 +155,7 @@ func TestLocalFileBlobUpload(t *testing.T) {
 		t.Fatalf("MkdirAll: %v", err)
 	}
 
-	blob, err := NewLocalFileBlob(ctx, "file://"+rootDir)
+	blob, err := NewLocalFileBlob(ctx, fileURL(rootDir))
 	if err != nil {
 		t.Fatalf("NewLocalFileBlob: %v", err)
 	}
@@ -216,7 +216,7 @@ func TestLocalFileBlobDownload(t *testing.T) {
 		t.Fatalf("MkdirAll destDir: %v", err)
 	}
 
-	blob, err := NewLocalFileBlob(ctx, "file://"+rootDir)
+	blob, err := NewLocalFileBlob(ctx, fileURL(rootDir))
 	if err != nil {
 		t.Fatalf("NewLocalFileBlob: %v", err)
 	}
@@ -271,7 +271,7 @@ func TestLocalFileBlobUploadOverwrite(t *testing.T) {
 		t.Fatalf("MkdirAll: %v", err)
 	}
 
-	blob, err := NewLocalFileBlob(ctx, "file://"+rootDir)
+	blob, err := NewLocalFileBlob(ctx, fileURL(rootDir))
 	if err != nil {
 		t.Fatalf("NewLocalFileBlob: %v", err)
 	}
@@ -326,7 +326,7 @@ func TestLocalFileBlobDownloadNonExistent(t *testing.T) {
 		t.Fatalf("MkdirAll: %v", err)
 	}
 
-	blob, err := NewLocalFileBlob(ctx, "file://"+rootDir)
+	blob, err := NewLocalFileBlob(ctx, fileURL(rootDir))
 	if err != nil {
 		t.Fatalf("NewLocalFileBlob: %v", err)
 	}
@@ -352,7 +352,7 @@ func TestLocalFileBlobUploadInvalidKey(t *testing.T) {
 		t.Fatalf("MkdirAll: %v", err)
 	}
 
-	blob, err := NewLocalFileBlob(ctx, "file://"+rootDir)
+	blob, err := NewLocalFileBlob(ctx, fileURL(rootDir))
 	if err != nil {
 		t.Fatalf("NewLocalFileBlob: %v", err)
 	}
@@ -382,7 +382,7 @@ func TestLocalFileBlobDownloadInvalidKey(t *testing.T) {
 		t.Fatalf("MkdirAll: %v", err)
 	}
 
-	blob, err := NewLocalFileBlob(ctx, "file://"+rootDir)
+	blob, err := NewLocalFileBlob(ctx, fileURL(rootDir))
 	if err != nil {
 		t.Fatalf("NewLocalFileBlob: %v", err)
 	}
@@ -401,7 +401,7 @@ func TestKeyToPaths(t *testing.T) {
 	tmpDir := t.TempDir()
 	ctx := context.Background()
 
-	blob, err := NewLocalFileBlob(ctx, "file://"+tmpDir)
+	blob, err := NewLocalFileBlob(ctx, fileURL(tmpDir))
 	if err != nil {
 		t.Fatalf("NewLocalFileBlob: %v", err)
 	}
@@ -485,7 +485,7 @@ func TestLocalFileBlobConcurrentUpload(t *testing.T) {
 		t.Fatalf("MkdirAll: %v", err)
 	}
 
-	blob, err := NewLocalFileBlob(ctx, "file://"+rootDir)
+	blob, err := NewLocalFileBlob(ctx, fileURL(rootDir))
 	if err != nil {
 		t.Fatalf("NewLocalFileBlob: %v", err)
 	}
@@ -551,7 +551,7 @@ func TestNewBlobStoreLocalFile(t *testing.T) {
 	ctx := context.Background()
 	tmpDir := t.TempDir()
 
-	blob, err := NewBlobStore(ctx, LocalFileStore, "file://"+tmpDir)
+	blob, err := NewBlobStore(ctx, LocalFileStore, fileURL(tmpDir))
 	if err != nil {
 		t.Fatalf("NewBlobStore: %v", err)
 	}