Refactor: break up the cli god-package + code-smell cleanups

CLAUDE.md

@@ -45,28 +45,36 @@ This split is a direct consequence of the soul: thin on the wire, rich on the ma
 ### Core CLI (`cmd/rekal/cli/`)
 
 - `root.go`: Root command (recall is the default) + command registration
-- `recall.go`: Hybrid search — BM25 + LSA + Nomic ranking, signal weighting
-  (steering-turn boost, subagent down-weight), and grouping subagent/workflow
-  hits under their trunk conversation (see `docs/agent-metadata.md`)
+- `recall.go`: Recall command orchestration — open/migrate/auto-rebuild the
+  index DB, call the `search` package, marshal JSON. The ranking engine itself
+  lives in `search/`.
 - `checkpoint.go`: Capture session after commit
-- `push.go`: Push data to remote branch
-- `sync.go`: Sync team context
-- `sync_remote.go`: Remote sync implementation
-- `export.go`: Encode checkpoints to wire format for push
-- `import.go`: Decode wire format during sync
+- `push.go`: Push data to remote branch (wire encode/commit lives in `transport/`)
+- `sync.go`: Sync team context (wire decode/import lives in `transport/`)
 - `init.go`: Bootstrap Rekal in a git repo
 - `clean.go`: Remove Rekal setup — completely, no residue
 - `index_cmd.go`: Rebuild index DB from data DB
 - `log.go`: Show recent checkpoints
 - `query.go`: Raw SQL access
 - `version.go`: Version constant (set via ldflags)
 - `errors.go`: SilentError pattern for clean error output
-- `preconditions.go`: Shared checks (git repo, init done, index exists)
+- `preconditions.go`: Shared checks — `RequireInitializedRepo` (git repo +
+  init done) plus the individual `EnsureGitRoot`/`EnsureInitDone` helpers
 
 ### Packages (`cmd/rekal/cli/`)
 
 - `codec/`: Binary wire format — frame encoding/decoding, body, dictionary, preset zstd dictionary
+- `transport/`: Git-side sync — encode checkpoints to the orphan-branch wire
+  format and decode them back (`export`/`import`/remote-sync glue, orphan-branch
+  commit). Sits above `codec`, `db`, and `gitx`; called by `push`/`sync`/`init`
+- `gitx/`: Thin git-plumbing helpers (rev-parse, show, hash-object, config,
+  `rekal/<email>` branch name) shared by the command and transport layers
+- `search/`: Recall ranking engine — hybrid BM25 + LSA + Nomic scoring, signal
+  weighting (steering-turn boost, subagent down-weight), conversation grouping
+  (see `docs/agent-metadata.md`), snippet extraction, and the LSA
+  query-projection cache (`projection.go`)
 - `session/`: Claude Code `.jsonl` parsing — extract turns, tool calls, deduplicate
+- `scrub/`: Redact secrets and anonymize file paths in a session payload before any DB insert (runs in `checkpoint` after parse)
 - `db/`: DuckDB backend — open, close, schema, insert helpers, index population
 - `lsa/`: Latent Semantic Analysis embeddings
 - `nomic/`: Nomic-embed-text deep semantic embeddings (platform build tags)

cmd/rekal/cli/checkpoint.go

@@ -5,15 +5,14 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
-	"math/rand"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
 
-	"github.com/oklog/ulid/v2"
 	"github.com/rekal-dev/rekal-cli/cmd/rekal/cli/db"
+	"github.com/rekal-dev/rekal-cli/cmd/rekal/cli/gitx"
+	"github.com/rekal-dev/rekal-cli/cmd/rekal/cli/ids"
 	"github.com/rekal-dev/rekal-cli/cmd/rekal/cli/scrub"
 	"github.com/rekal-dev/rekal-cli/cmd/rekal/cli/session"
 	"github.com/spf13/cobra"
@@ -33,16 +32,9 @@ records which files were changed.
 Normally runs automatically via the post-commit hook installed by 'rekal init'.
 Run manually to capture a session without committing.`,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			cmd.SilenceUsage = true
-
-			gitRoot, err := EnsureGitRoot()
+			gitRoot, err := RequireInitializedRepo(cmd)
 			if err != nil {
-				fmt.Fprintln(cmd.ErrOrStderr(), err)
-				return NewSilentError(err)
-			}
-			if err := EnsureInitDone(gitRoot); err != nil {
-				fmt.Fprintln(cmd.ErrOrStderr(), err)
-				return NewSilentError(err)
+				return err
 			}
 
 			return runCheckpoint(cmd, gitRoot)
@@ -74,11 +66,8 @@ func doCheckpoint(gitRoot string, w io.Writer) error {
 		return fmt.Errorf("data DB is corrupt or unreadable: %w", err)
 	}
 
-	email := gitConfigValue("user.email")
-	entropy := rand.New(rand.NewSource(time.Now().UnixNano())) //nolint:gosec
-	newID := func() string {
-		return ulid.MustNew(ulid.Timestamp(time.Now()), entropy).String()
-	}
+	email := gitx.ConfigValue("user.email")
+	newID := ids.NewULIDFunc()
 
 	var sessionIDs []string
 	// trunkOnlySessionIDs is the subset of sessionIDs with no parent — used to
@@ -265,9 +254,9 @@ func doCheckpoint(gitRoot string, w io.Writer) error {
 	}
 
 	// Get git state for checkpoint.
-	gitSHA := gitHeadSHA(gitRoot)
-	gitBranch := gitCurrentBranch(gitRoot)
-	filesTouched := gitFilesChanged(gitRoot)
+	gitSHA := gitx.HeadSHA(gitRoot)
+	gitBranch := gitx.CurrentBranch(gitRoot)
+	filesTouched := gitx.FilesChanged(gitRoot)
 
 	// Generate checkpoint ULID.
 	checkpointID := newID()
@@ -318,46 +307,6 @@ func doCheckpoint(gitRoot string, w io.Writer) error {
 	return nil
 }
 
-func gitHeadSHA(gitRoot string) string {
-	out, err := exec.Command("git", "-C", gitRoot, "rev-parse", "HEAD").Output()
-	if err != nil {
-		return strings.Repeat("0", 40)
-	}
-	return strings.TrimSpace(string(out))
-}
-
-func gitCurrentBranch(gitRoot string) string {
-	out, err := exec.Command("git", "-C", gitRoot, "rev-parse", "--abbrev-ref", "HEAD").Output()
-	if err != nil {
-		return "unknown"
-	}
-	return strings.TrimSpace(string(out))
-}
-
-func gitFilesChanged(gitRoot string) []string {
-	out, err := exec.Command("git", "-C", gitRoot, "diff", "--name-status", "HEAD~1", "HEAD").Output()
-	if err != nil {
-		return nil
-	}
-	lines := strings.Split(strings.TrimSpace(string(out)), "\n")
-	var result []string
-	for _, line := range lines {
-		if strings.TrimSpace(line) != "" {
-			result = append(result, line)
-		}
-	}
-	return result
-}
-
-// gitShowFile reads a file from a git ref. Returns nil if not found.
-func gitShowFile(gitRoot, ref, path string) []byte {
-	out, err := exec.Command("git", "-C", gitRoot, "show", ref+":"+path).Output()
-	if err != nil {
-		return nil
-	}
-	return out
-}
-
 func sha256Hex(data []byte) string {
 	h := sha256.Sum256(data)
 	return hex.EncodeToString(h[:])

cmd/rekal/cli/codec/body.go

@@ -6,6 +6,13 @@ import (
 	"fmt"
 )
 
+// Wire-format filenames on the rekal orphan branch: the append-only frame
+// body and the string dictionary it references.
+const (
+	BodyFilename = "rekal.body"
+	DictFilename = "dict.bin"
+)
+
 const (
 	bodyMagic    = "RKLBODY"
 	bodyVersion  = 0x01

cmd/rekal/cli/db/db.go

@@ -115,28 +115,29 @@ func InsertSessionMeta(d *sql.DB, id, parentSessionID, hash, actorType, agentID,
 	_, err := d.Exec(
 		`INSERT INTO sessions (id, parent_session_id, session_hash, captured_at, actor_type, agent_id, user_email, branch, source, team_name, workflow_name, agent_type, description, spawn_depth)
 		 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`,
-		id, nullIfEmpty(parentSessionID), hash, capturedAt, actorType, agentID, userEmail, branch, source,
-		nullIfEmpty(meta.TeamName), nullIfEmpty(meta.WorkflowName),
-		nullIfEmpty(meta.AgentType), nullIfEmpty(meta.Description), nullIfZero(meta.SpawnDepth),
+		id, NullIfEmpty(parentSessionID), hash, capturedAt, actorType, agentID, userEmail, branch, source,
+		NullIfEmpty(meta.TeamName), NullIfEmpty(meta.WorkflowName),
+		NullIfEmpty(meta.AgentType), NullIfEmpty(meta.Description), NullIfZero(meta.SpawnDepth),
 	)
 	if err != nil {
 		return fmt.Errorf("insert session: %w", err)
 	}
 	return nil
 }
 
-// nullIfEmpty returns nil if s is empty, otherwise s.
-// Used to store NULL in VARCHAR columns instead of empty strings.
-func nullIfEmpty(s string) interface{} {
+// NullIfEmpty returns nil if s is empty, otherwise s. Used to store NULL in
+// VARCHAR columns instead of empty strings, from both this package's insert
+// helpers and callers that build raw INSERTs against these tables.
+func NullIfEmpty(s string) interface{} {
 	if s == "" {
 		return nil
 	}
 	return s
 }
 
-// nullIfZero returns nil if n is zero, otherwise n.
-// Used to store NULL in INTEGER columns instead of a misleading 0.
-func nullIfZero(n int) interface{} {
+// NullIfZero returns nil if n is zero, otherwise n. Used to store NULL in
+// INTEGER columns instead of a misleading 0.
+func NullIfZero(n int) interface{} {
 	if n == 0 {
 		return nil
 	}
@@ -148,7 +149,7 @@ func InsertTurn(d *sql.DB, id, sessionID string, turnIndex int, role, content, t
 	_, err := d.Exec(
 		`INSERT INTO turns (id, session_id, turn_index, role, content, ts)
 		 VALUES ($1, $2, $3, $4, $5, $6)`,
-		id, sessionID, turnIndex, role, content, nullIfEmpty(ts),
+		id, sessionID, turnIndex, role, content, NullIfEmpty(ts),
 	)
 	if err != nil {
 		return fmt.Errorf("insert turn: %w", err)