chore(cli): implement Paket 2.1 DX hardening (skip-checks, ringbuffer, rename-to-trash, matrix async)

Author: mannomannX

cli/toob-cli/cmd/build.go

@@ -4,15 +4,47 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/spf13/cobra"
 	"github.com/toob-boot/toob/internal/paths"
 )
 
+var flagToolchains bool
+
+func init() {
+	cleanCmd.Flags().BoolVar(&flagToolchains, "toolchains", false, "Remove all globally cached cross-compiler toolchains to free up disk space")
+}
+
 var cleanCmd = &cobra.Command{
 	Use:   "clean",
 	Short: "Remove all build artifacts (builds/ directory)",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		if flagToolchains {
+			home, err := os.UserHomeDir()
+			if err != nil {
+				return err
+			}
+			tcDir := filepath.Join(home, ".toob", "toolchains")
+			if _, err := os.Stat(tcDir); os.IsNotExist(err) {
+				fmt.Println("[toob] No toolchains found. Nothing to clean.")
+				return nil
+			}
+			fmt.Printf("[toob] Removing globally cached toolchains at %s ...\n", tcDir)
+			// Safe rename-to-trash pattern for Windows locking safety
+			trashDir := filepath.Join(home, ".toob", ".trash", "toolchains-"+time.Now().Format("20060102150405"))
+			os.MkdirAll(filepath.Dir(trashDir), 0o755)
+			if err := os.Rename(tcDir, trashDir); err != nil {
+				return fmt.Errorf("failed to unlock toolchains directory (is a file currently open in an IDE or terminal?): %w", err)
+			}
+
+			if err := os.RemoveAll(trashDir); err != nil {
+				fmt.Printf("\033[33m[toob] Warning: Could not fully delete all files (some are locked), but toolchains are deactivated.\033[0m\n")
+			}
+			fmt.Println("\033[32m[toob] Successfully freed disk space.\033[0m")
+			return nil
+		}
+
 		root, err := paths.FindProjectRoot("")
 		if err != nil || root == "" {
 			return fmt.Errorf("not in a Toob-Loader project (device.toml not found)")

cli/toob-cli/cmd/clean.go

@@ -19,6 +19,7 @@ require (
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.9 // indirect
 	golang.org/x/crypto v0.45.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.38.0 // indirect
 	golang.org/x/term v0.37.0 // indirect
 )

cli/toob-cli/go.mod

@@ -43,6 +43,8 @@ golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM=
 golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

cli/toob-cli/go.sum

@@ -48,6 +48,12 @@ func parseChipArg(arg string) (string, string) {
 
 // Add installs a chip from the registry (not tracked by git).
 func (inst *Installer) Add(arg string) error {
+	// Async Matrix Fetch (Gap 4.1)
+	matrixChan := make(chan *registry.Matrix, 1)
+	go func() {
+		m, _ := inst.cache.FetchLiveMatrix()
+		matrixChan <- m
+	}()
 	name, version := parseChipArg(arg)
 	if inst.lock.HasChip(name) {
 		e := inst.lock.GetChip(name)
@@ -115,16 +121,26 @@ func (inst *Installer) Add(arg string) error {
 	}
 	fmt.Printf("Added chip '%s' (v%s) to lockfile [arch=%s, vendor=%s].\n", name, ci.Version, ci.Arch, ci.Vendor)
 
-	if !ci.Verified {
-		fmt.Println("\n\033[33mWarning: This hardware configuration is marked as UNVERIFIED by the CI Compatibility Matrix. Build stability is not guaranteed.\033[0m")
+	// Wait up to 1 second for the matrix to avoid blocking
+	var matrix *registry.Matrix
+	select {
+	case matrix = <-matrixChan:
+	case <-time.After(1 * time.Second):
 	}
+	printMatrixCompatibility(matrix, ci.Name, ci.Version, ci.Verified)
 
 	fmt.Println("Registry link established. Run `toob build` to compile.")
 	return nil
 }
 
 // Spawn installs a chip as locally editable (tracked by git).
 func (inst *Installer) Spawn(arg string) error {
+	// Async Matrix Fetch (Gap 4.1)
+	matrixChan := make(chan *registry.Matrix, 1)
+	go func() {
+		m, _ := inst.cache.FetchLiveMatrix()
+		matrixChan <- m
+	}()
 	name, version := parseChipArg(arg)
 	if inst.lock.HasChip(name) {
 		e := inst.lock.GetChip(name)
@@ -229,9 +245,13 @@ func (inst *Installer) Spawn(arg string) error {
 	}
 	fmt.Printf("Spawned chip '%s' (v%s)  [locally editable]\n", name, ci.Version)
 
-	if !ci.Verified {
-		fmt.Println("\n\033[33mWarning: This hardware configuration is marked as UNVERIFIED by the CI Compatibility Matrix. Build stability is not guaranteed.\033[0m")
+	// Wait up to 1 second for the matrix to avoid blocking
+	var matrix *registry.Matrix
+	select {
+	case matrix = <-matrixChan:
+	case <-time.After(1 * time.Second):
 	}
+	printMatrixCompatibility(matrix, ci.Name, ci.Version, ci.Verified)
 
 	return nil
 }
@@ -246,6 +266,36 @@ func moveToTrash(dir string) {
 	os.Rename(dir, trashDir)
 }
 
+func printMatrixCompatibility(matrix *registry.Matrix, chipName, chipVersion string, verified bool) {
+	if !verified {
+		fmt.Println("\n\033[33mWarning: This hardware configuration is marked as UNVERIFIED by the CI Compatibility Matrix. Build stability is not guaranteed.\033[0m")
+		return
+	}
+
+	if matrix == nil {
+		return
+	}
+
+	if chipEntry, has := (*matrix)[chipName]; has {
+		// Normalize version string to handle mismatching prefixes (Gap 4.2)
+		searchVer := strings.TrimPrefix(chipVersion, "v")
+		for vKey, verEntry := range chipEntry.Versions {
+			if strings.TrimPrefix(vKey, "v") == searchVer {
+				var verifiedClis []string
+				for cliVer, info := range verEntry.VerifiedCliVersions {
+					if info.Status == "SUCCESS" {
+						verifiedClis = append(verifiedClis, cliVer)
+					}
+				}
+				if len(verifiedClis) > 0 {
+					fmt.Printf("\n\033[32m[toob] Chip %s v%s — Verified with CLI: %s\033[0m\n", chipName, chipVersion, strings.Join(verifiedClis, ", "))
+				}
+				break
+			}
+		}
+	}
+}
+
 // Remove uninstalls a chip and cleans up unshared dependencies.
 func (inst *Installer) Remove(name string) error {
 	entry := inst.lock.GetChip(name)
@@ -349,6 +399,7 @@ func (inst *Installer) installDeps(ci *registry.ChipInfo) ([]string, error) {
 }
 
 // copyTree recursively copies src to dst.
+// Uses hard-links where possible for instant, zero-disk-cost copies.
 func copyTree(src, dst string) error {
 	info, err := os.Stat(src)
 	if err != nil {
@@ -373,12 +424,25 @@ func copyTree(src, dst string) error {
 		if d.IsDir() {
 			return os.MkdirAll(target, 0o755)
 		}
-		data, err := os.ReadFile(path)
-		if err != nil {
-			return err
-		}
-		return os.WriteFile(target, data, 0o644)
+		return linkOrCopy(path, target)
 	})
 }
 
+// linkOrCopy attempts a hard-link first, falling back to a full byte-copy.
+// Hard-links share the inode and are instant with zero additional disk cost.
+// Fallback handles cross-device mounts, FAT32, and Windows restrictions.
+func linkOrCopy(src, dst string) error {
+	if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
+		return err
+	}
+	if err := os.Link(src, dst); err == nil {
+		return nil
+	}
+	data, err := os.ReadFile(src)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(dst, data, 0o644)
+}
+
 

cli/toob-cli/internal/installer/installer.go

@@ -298,7 +298,7 @@ func VerifyMacroUsage(headerPath, bootloaderDir string) error {
 		errb.WriteString("  2. The manifest is generating dead code that should be pruned.\n")
 		errb.WriteString("Fix the C code to use these macros, or remove them from the generator.\n")
 		errb.WriteString("======================================================================\n")
-		return fmt.Errorf(errb.String())
+		return fmt.Errorf("%s", errb.String())
 	}
 
 	fmt.Println("[Manifest Verifier] SUCCESS: All generated macros are perfectly synchronized with the C code!")

cli/toob-cli/internal/manifest/generator.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/toob-boot/toob/internal/paths"
@@ -119,13 +120,51 @@ func (c *Cache) lock() (func(), error) {
 	lockDir := filepath.Join(filepath.Dir(c.dir), "registry.lock")
 	for i := 0; i < 100; i++ { // wait up to 10 seconds
 		if err := os.Mkdir(lockDir, 0o755); err == nil {
-			return func() { os.Remove(lockDir) }, nil
+			// Write PID file for stale-lock detection
+			pidFile := filepath.Join(lockDir, "pid")
+			_ = os.WriteFile(pidFile, []byte(fmt.Sprintf("%d", os.Getpid())), 0o644)
+			return func() { os.RemoveAll(lockDir) }, nil
+		}
+
+		// Check if the locking process is still alive
+		if i%10 == 9 { // every ~1 second
+			if c.tryCleanStaleLock(lockDir) {
+				continue // Retry immediately after cleaning stale lock
+			}
 		}
 		time.Sleep(100 * time.Millisecond)
 	}
 	return nil, fmt.Errorf("timeout waiting for registry lock. Is another toob process running? (If not, delete %s)", lockDir)
 }
 
+// tryCleanStaleLock reads the PID from the lock directory and checks if the process is alive.
+// Returns true if a stale lock was cleaned up.
+func (c *Cache) tryCleanStaleLock(lockDir string) bool {
+	pidBytes, err := os.ReadFile(filepath.Join(lockDir, "pid"))
+	if err != nil {
+		return false
+	}
+	pid := 0
+	if _, err := fmt.Sscanf(string(pidBytes), "%d", &pid); err != nil || pid == 0 {
+		return false
+	}
+	proc, err := os.FindProcess(pid)
+	if err != nil {
+		// Process doesn't exist — stale lock
+		fmt.Printf("[toob] Cleaning stale registry lock (PID %d no longer running)\n", pid)
+		os.RemoveAll(lockDir)
+		return true
+	}
+	// On Unix, FindProcess always succeeds. Send signal 0 to probe liveness.
+	// On Windows, FindProcess fails for dead processes, so reaching here means alive.
+	if err := proc.Signal(syscall.Signal(0)); err != nil {
+		fmt.Printf("[toob] Cleaning stale registry lock (PID %d no longer running)\n", pid)
+		os.RemoveAll(lockDir)
+		return true
+	}
+	return false
+}
+
 // getHubURL returns the URL of the Toob Hub API
 func getHubURL() string {
 	if url := os.Getenv("TOOB_HUB_URL"); url != "" {

cli/toob-cli/internal/registry/cache.go

@@ -2,7 +2,6 @@ package registry
 
 import (
 	"archive/zip"
-	"bytes"
 	"fmt"
 	"io"
 	"net/http"
@@ -11,8 +10,9 @@ import (
 	"strings"
 )
 
-// downloadAndExtractZip downloads a ZIP archive and extracts it to targetDir.
-// It automatically strips the root folder (e.g. 'Toob-Registry-main/') from the zip entries.
+// downloadAndExtractZip downloads a ZIP archive to a temp file and extracts it to targetDir.
+// Streams directly to disk to avoid holding the full archive in RAM.
+// Automatically strips the root folder (e.g. 'Toob-Registry-main/') from zip entries.
 func downloadAndExtractZip(url string, targetDir string) error {
 	resp, err := http.Get(url)
 	if err != nil {
@@ -24,15 +24,24 @@ func downloadAndExtractZip(url string, targetDir string) error {
 		return fmt.Errorf("bad status %d from %s", resp.StatusCode, url)
 	}
 
-	body, err := io.ReadAll(resp.Body)
+	// Stream to temp file instead of buffering entire archive in RAM
+	tmpFile, err := os.CreateTemp("", "toob-registry-*.zip")
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to create temp file: %w", err)
+	}
+	defer os.Remove(tmpFile.Name())
+	defer tmpFile.Close()
+
+	if _, err := io.Copy(tmpFile, resp.Body); err != nil {
+		return fmt.Errorf("failed to download archive: %w", err)
 	}
+	tmpFile.Close()
 
-	zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
+	zipReader, err := zip.OpenReader(tmpFile.Name())
 	if err != nil {
 		return err
 	}
+	defer zipReader.Close()
 
 	if err := os.MkdirAll(targetDir, 0o755); err != nil {
 		return err
@@ -42,9 +51,9 @@ func downloadAndExtractZip(url string, targetDir string) error {
 		// Strip the top-level directory
 		parts := strings.SplitN(filepath.ToSlash(f.Name), "/", 2)
 		if len(parts) < 2 || parts[1] == "" {
-			continue // Skip the root directory itself
+			continue
 		}
-		
+
 		relPath := filepath.FromSlash(parts[1])
 		destPath := filepath.Join(targetDir, relPath)