fix(catalog): keep alias bookkeeping out of the xorq catalog repo (#48)

tallyman wrote aliases.json/alias_history.json under catalog_dir — the same
directory that is the xorq catalog git repo — and checkpoint_catalog committed
them. xorq's Catalog.assert_consistency() requires the repo's tracked files to be
exactly its managed set (catalog.yaml + per-entry .zip/.metadata.yaml + alias
symlinks), so the two extra tracked files made every Catalog.from_kwargs(...,
init=False) raise a bare AssertionError. add_entry swallowed it, so every
`xorq catalog add` after the first silently no-op'd and 17/18 recipes never
reached git.

Move the bookkeeping to artifacts/alias_state/ (outside the repo) and mirror its
content into catalog.yaml as tallyman-owned keys (alias_map, alias_history), so
reset_to still rolls alias state back now that git no longer tracks the files.
checkpoint_catalog's _TRACKED drops to just catalog.yaml.

Surface the previously-swallowed failure: build_and_persist exposes
BuildResult.catalog_registered, and the failed-add path logs at error level
instead of warning.

- failing tests for this landed in 80d2e9b; this is the fix half of the split
- test_failed_registration_is_observable: its premise (add fails on its own) was
  bug #48 itself; rewritten to inject the same inconsistency so it tests the
  catalog_registered signal rather than the bug
- test_reset_rolls_back_alias_state: the deferred regression guard, shipping with
  the fix per the plan (it passes today, so it can't ride the failing-tests PR)
- test_pack: alias file path updated to artifacts/alias_state/

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: paddymul

src/tallyman_core/aliases.py

@@ -16,7 +16,7 @@
 from pathlib import Path
 
 from tallyman_core import xorq_catalog as _xcat
-from tallyman_core.paths import catalog_dir, ensure_project
+from tallyman_core.paths import alias_state_dir, ensure_project
 
 
 class AliasExists(ValueError):
@@ -28,11 +28,11 @@ class AliasNotFound(KeyError):
 
 
 def _aliases_path(project: str) -> Path:
-    return catalog_dir(project) / "aliases.json"
+    return alias_state_dir(project) / "aliases.json"
 
 
 def _history_path(project: str) -> Path:
-    return catalog_dir(project) / "alias_history.json"
+    return alias_state_dir(project) / "alias_history.json"
 
 
 def _read_json(p: Path, default):
@@ -57,6 +57,17 @@ def load_history(project: str) -> dict[str, list[str]]:
     return _read_json(_history_path(project), {})
 
 
+def write_state(project: str, aliases: dict[str, str], history: dict[str, list[str]]) -> None:
+    """Overwrite the alias bookkeeping files wholesale.
+
+    Used by ``catalog_state.materialize`` to reconstruct alias state from the
+    catalog.yaml mirror on a ``reset_to``: the files live outside the git repo
+    now (#48), so git can't roll them back — this does.
+    """
+    _write_json(_aliases_path(project), aliases)
+    _write_json(_history_path(project), history)
+
+
 def get_alias(project: str, name: str) -> str | None:
     return load_aliases(project).get(name)
 

src/tallyman_core/catalog_state.py

@@ -11,8 +11,15 @@
     entry_hashes:    [content_hash, ...]      # pointer to untracked entries/<hash>/
     result_cache:    [relpath, ...]           # pointer to untracked result cache
     compute_cache:   [relpath, ...]           # pointer to untracked compute cache
-
-The catalog git repo tracks only ``catalog.yaml`` + the two alias files. The
+    alias_map:       {alias: latest_hash}     # mirror of artifacts/alias_state/aliases.json
+    alias_history:   {alias: [hash, ...]}     # mirror of artifacts/alias_state/alias_history.json
+
+The catalog git repo tracks only ``catalog.yaml``. tallyman's alias bookkeeping
+(``aliases.json``/``alias_history.json``) lives outside the repo, under
+``artifacts/alias_state/`` — committing it inside made xorq's
+``assert_consistency`` reject the repo and every ``xorq catalog add`` after the
+first silently no-op (#48). Its content is mirrored into ``catalog.yaml`` above
+so ``reset_to`` still rolls alias state back (git no longer can). The
 heavy artifacts (entries/, the caches) are content-addressed, additive, and
 untracked — ``git reset`` can't roll them back, so ``reset_to`` reconciles them
 to the recorded pointer lists: evictions retire to the bullpen (not deleted),
@@ -40,6 +47,7 @@
 
 import yaml
 
+from tallyman_core import aliases as al
 from tallyman_core import charts, notebook
 from tallyman_core import display_configs as dc
 from tallyman_core import post_processing as pp
@@ -60,7 +68,10 @@
 
 # git-level identity flags, so commits work without a global git config.
 _GIT_ID = ["-c", "user.email=tallyman@local", "-c", "user.name=tallyman"]
-_TRACKED = ("catalog.yaml", "aliases.json", "alias_history.json")
+# Only catalog.yaml is git-tracked in the catalog repo. The alias bookkeeping
+# files moved out to artifacts/alias_state/ (#48) — tracking them here made
+# xorq's assert_consistency reject the repo.
+_TRACKED = ("catalog.yaml",)
 _STEP_RE = re.compile(r"^step-(\d+)$")
 # Refs and labels are operator input that ends up as git arguments. A plain
 # tag-shaped name only: no leading dash (option injection), no revision
@@ -96,6 +107,8 @@ def read_tallyman_state(project: str) -> dict:
         "entry_hashes": raw.get("entry_hashes", []),
         "result_cache": raw.get("result_cache", []),
         "compute_cache": raw.get("compute_cache", []),
+        "alias_map": raw.get("alias_map", {}),
+        "alias_history": raw.get("alias_history", {}),
     }
 
 
@@ -154,6 +167,8 @@ def capture_tallyman_state(project: str) -> dict:
         "entry_hashes": sorted(c.name for c in ed.iterdir() if c.is_dir()) if ed.exists() else [],
         "result_cache": _list_cache_files(result_cache_dir(project)),
         "compute_cache": _list_cache_files(compute_cache_dir(project)),
+        "alias_map": al.load_aliases(project),
+        "alias_history": al.load_history(project),
     }
     write_tallyman_state(project, **state)
     return state
@@ -223,6 +238,8 @@ def materialize(project: str) -> None:
         _materialize_display_configs(project, raw["display_configs"])
     if "notebook" in raw:
         _materialize_notebook(project, raw["notebook"] or {"cells": []})
+    if "alias_map" in raw:
+        al.write_state(project, raw["alias_map"] or {}, raw.get("alias_history") or {})
 
 
 # ---------------------------------------------------------------------------

src/tallyman_core/paths.py

@@ -7,8 +7,8 @@
     ├── buckaroo_sessions.json         # global Buckaroo session map
     └── projects/<name>/
         ├── artifacts/                 # everything the system produces
-        │   ├── catalog/entries/<hash>/
-        │   ├── catalog/aliases.json, alias_history.json, chart_specs/
+        │   ├── catalog/entries/<hash>/    # the xorq catalog git repo
+        │   ├── alias_state/aliases.json, alias_history.json
         │   ├── post_processing/<name>.py
         │   ├── stats/<name>.py
         │   ├── exports/...            # marimo .py, screenshots, CSVs
@@ -119,6 +119,20 @@ def catalog_dir(project: str) -> Path:
     return artifacts_dir(project) / "catalog"
 
 
+def alias_state_dir(project: str) -> Path:
+    """Tallyman's alias bookkeeping — deliberately OUTSIDE the xorq catalog repo.
+
+    ``aliases.json`` / ``alias_history.json`` are tallyman-owned, not
+    xorq-managed. They used to live under ``catalog_dir``, so ``checkpoint_catalog``
+    committed them into the xorq catalog git repo, and xorq's
+    ``assert_consistency`` then refused to open it — every ``xorq catalog add``
+    after the first silently no-op'd (#48). They live here instead; their content
+    is mirrored into ``catalog.yaml`` (tallyman-owned keys), so ``reset_to`` still
+    rolls alias state back even though git no longer tracks the files.
+    """
+    return artifacts_dir(project) / "alias_state"
+
+
 def entries_dir(project: str) -> Path:
     return catalog_dir(project) / "entries"
 

src/tallyman_core/xorq_catalog.py

@@ -63,7 +63,9 @@ def add_entry(
     def _run(d: Path) -> bool:
         r = _xorq(project, "add", "--no-sync", str(d), *alias_flags)
         if r.returncode != 0:
-            log.warning("xorq catalog add failed for %s: %s", d, r.stderr.strip())
+            # error, not warning: a failed add means the recipe never reached git
+            # and is silently lost (#48). The caller surfaces this on BuildResult.
+            log.error("xorq catalog add failed for %s: %s", d, r.stderr.strip())
             return False
         return True
 

src/tallyman_xorq/build.py

@@ -51,6 +51,10 @@ class BuildResult:
     row_count: int
     execute_seconds: float
     schema: dict
+    # Whether the entry's recipe landed in the xorq catalog git repo. None on the
+    # re-run path (entry already on disk, registration not re-attempted); True/False
+    # on a fresh build. False means the recipe is NOT durable — see #48.
+    catalog_registered: bool | None = None
 
 
 class BuildError(RuntimeError):
@@ -369,22 +373,32 @@ def build_and_persist(
     except OSError:
         pass
 
-    # Best-effort: register with the xorq catalog git repo.
+    # Best-effort by design (a missing/misconfigured catalog repo must not break
+    # the build), but the outcome is surfaced on the result instead of swallowed:
+    # a failed add means the recipe never reached git and is not durable (#48).
+    import logging
+
+    catalog_registered: bool | None = None
     try:
         from tallyman_core.xorq_catalog import add_entry as _xcat_add
 
-        _xcat_add(project, xorq_build_dir, entry_name=content_hash)
+        catalog_registered = _xcat_add(project, xorq_build_dir, entry_name=content_hash)
     except Exception as exc:
-        import logging
-
-        logging.getLogger(__name__).warning("xorq catalog add skipped: %s", exc)
+        catalog_registered = False
+        logging.getLogger(__name__).error("xorq catalog add raised for %s: %s", content_hash, exc)
+    if catalog_registered is False:
+        logging.getLogger(__name__).error(
+            "xorq catalog add failed for %s — entry recipe is not durable in the catalog repo (#48)",
+            content_hash,
+        )
 
     return BuildResult(
         content_hash=content_hash,
         entry_path=target,
         row_count=row_count,
         execute_seconds=execute_seconds,
         schema=schema_doc,
+        catalog_registered=catalog_registered,
     )
 
 

tests/test_catalog_xorq_integration.py

@@ -194,16 +194,51 @@ def test_back_then_forward_restores_xorq_recipe(project, orders_parquet):
 def test_failed_registration_is_observable(project, orders_parquet):
     """A failed xorq catalog add must be observable, not reported as success.
 
-    Entry 1 registers and (via the committed bookkeeping files) leaves the
-    catalog inconsistent; the next add then fails ``assert_consistency``. Today
-    ``add_entry``'s failure is swallowed in a best-effort wrapper and its bool
-    return is discarded, so ``build_and_persist`` exposes no signal. The loose,
-    fix-shaped property: a ``catalog_registered``-style flag exists and is
-    ``False`` on failure."""
+    Pre-fix this failure happened on its own (entry 1's checkpoint committed the
+    bookkeeping files, leaving the catalog inconsistent so the next add fails) —
+    that was bug #48 itself, and ``add_entry`` swallowed the failure with no
+    signal. The fix removes that spontaneous failure, so to test the *signal* we
+    inject the same kind of inconsistency a stray tracked file gives, then assert
+    ``build_and_persist`` reports ``catalog_registered is False`` rather than
+    claiming success."""
     _make_entries(project, orders_parquet, 1)
+
+    # A tracked file outside xorq's managed set is exactly the shape #48 created:
+    # assert_consistency then rejects the repo and the next add fails.
+    cd = paths.catalog_dir(project)
+    (cd / "stray.txt").write_text("not xorq-managed\n")
+    _git(project, "add", "stray.txt")
+    subprocess.run(
+        ["git", "-C", str(cd), "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-q", "-m", "stray"],
+        capture_output=True,
+        text=True,
+    )
+
     result = build_and_persist(project, _agg_code(orders_parquet, _AGGS[1]))
-    registered = getattr(result, "catalog_registered", None)
-    assert registered is False, (
+    assert result.catalog_registered is False, (
         "failed xorq catalog add must be observable: build_and_persist should expose "
-        f"catalog_registered=False, got {registered!r}"
+        f"catalog_registered=False, got {result.catalog_registered!r}"
+    )
+
+
+# ---------------------------------------------------------------------------
+# Group E — reset rolls back alias state (regression guard shipping WITH the fix)
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.integration
+def test_reset_rolls_back_alias_state(project, orders_parquet):
+    """A reset must roll alias bookkeeping back, even though the files left the repo.
+
+    Pre-fix, ``aliases.json`` was git-tracked, so ``git reset --hard`` rolled it
+    back for free. The #48 fix moves the bookkeeping out of the catalog repo, so
+    ``reset_to`` has to reconcile alias state from the catalog.yaml mirror
+    instead — this guards that it does (it would silently regress otherwise)."""
+    from tallyman_core import aliases as al
+
+    _make_entries(project, orders_parquet, 2)  # creates aliases agg0, agg1
+    assert set(al.load_aliases(project)) == {"agg0", "agg1"}
+    cs.reset_to(project, 1)  # back to the one-entry step
+    assert set(al.load_aliases(project)) == {"agg0"}, (
+        f"reset_to(step-1) did not roll alias state back: {al.load_aliases(project)}"
     )

tests/test_pack.py

@@ -36,7 +36,7 @@ def test_pack_creates_tgz(project: str, orders_parquet: Path, isolated_home: Pat
 
     with tarfile.open(out_path) as tar:
         names = tar.getnames()
-    assert f"{project}/artifacts/catalog/aliases.json" in names
+    assert f"{project}/artifacts/alias_state/aliases.json" in names
     assert f"{project}/data/orders.parquet" in names
     assert any(f"{project}/artifacts/catalog/entries/{h}/" in n for n in names)