Remove dense/sparse mode flags and update related documentation for clarity

Author: justinlietz93

cli/args.py

@@ -35,13 +35,6 @@ def make_parser() -> argparse.ArgumentParser:
     p.set_defaults(use_time_dynamics=True)
     p.add_argument('--seed', type=int, default=0)
 
-    # Ultra-scale/sparse flags
-    p.add_argument('--sparse-mode', dest='sparse_mode', action='store_true')
-    p.add_argument('--dense-mode', dest='sparse_mode', action='store_false')
-    # Aliases
-    p.add_argument('--sparse', dest='sparse_mode', action='store_true')
-    p.add_argument('--dense', dest='sparse_mode', action='store_false')
-    p.set_defaults(sparse_mode=None)
     p.add_argument('--threshold', type=float, default=0.15)
     p.add_argument('--lambda-omega', dest='lambda_omega', type=float, default=0.1)
     p.add_argument('--candidates', type=int, default=64)

control/process_manager.py

@@ -81,9 +81,7 @@ def add(flag: str, val: Any, cast=str):
             cmd.append("--use-time-dynamics")
         else:
             cmd.append("--no-time-dynamics")
-        # sparse / structure
-        if profile.get("sparse_mode", False):
-            cmd.append("--sparse-mode")
+        # structure
         add("--threshold", profile.get("threshold"), str)
         add("--lambda-omega", profile.get("lambda_omega"), str)
         add("--candidates", profile.get("candidates"), str)

core/global_system.py

@@ -135,7 +135,7 @@ def run(self, W: np.ndarray, rng: np.random.Generator) -> Tuple[np.ndarray, int,
 # -----------------------------------
 
 @dataclass
-class SIE: # TODO: This is not the canonical SIE; compare it with vdm_rt/core/sie.py.
+class SIE: # TODO: classify this legacy SIE variant as keep, port, archive, or delete after parity review with core/sie.py.
     """
     Self‑Improvement Engine (Rule 3)
     total_reward = w_td * TD_error_norm + w_nov * novelty_norm - w_hab * habituation_norm + w_hsi * hsi_norm

core/metrics.py

@@ -12,10 +12,11 @@ def compute_metrics(connectome):
    """
    Rule Ref: Blueprint Rule 4.1 (Pathology Detection Mechanisms)
    - Adds connectome_entropy to support Active Domain Cartography (Rule 7) scheduling.
-   - Prefers connectome.connectome_entropy() when available (sparse-mode), falling back to local function.
+   - Prefers connectome.connectome_entropy() when available, falling back to local function.
    """
-   # TODO GET THESE FOR FREE FROM THE VOID WALKERS
-   # Prefer a connectome-native entropy calculator for sparse-mode
+   # TODO: replace scan-derived metric calls with SparseConnectome.metrics_snapshot()
+   # and event-spine reducer outputs after parity tests pass.
+   # Prefer a connectome-native entropy calculator when available.
    try:
        h = float(connectome.connectome_entropy())
    except Exception:

docs/governance/tools/check_docs_front_matter.py

@@ -20,7 +20,7 @@ The current runtime is halfway through a good migration: from scan-heavy global
 | 2 | `docs/roadmap/runtime-event-spine/TODO.md` | Unify `Observation`, `BaseEvent`, ADC, maps, memory, and future motor events behind one typed event spine. |
 | 3 | `docs/roadmap/adc-territory-richness/TODO.md` | Feed ADC more than `region_stat` and `cycle_hit`; produce `boundary_probe` and `novel_frontier`. |
 | 4 | `docs/roadmap/scout-map-ownership/TODO.md` | Keep void walkers/maps as core systems, remove duplicate scout ownership, and route scout products globally. |
-| 5 | `docs/roadmap/memory-gdsp-territory/TODO.md` | Fix memory ownership, stale observation reuse, GDSP one-tick territory lag, and dead `bias_hint` shape. |
+| 5 | `docs/roadmap/memory-gdsp-territory/TODO.md` | Fix memory ownership, stale observation reuse, GDSP one-tick territory lag, dead `bias_hint` shape, and engram resume integrity. |
 | 6 | `docs/roadmap/loop-scan-reduction/TODO.md` | Replace repeated full graph scans with sparse cached snapshots and reducer-fed metrics. |
 | 7 | `docs/roadmap/sparse-neurogenesis/TODO.md` | Recover dynamic node growth/culling as sparse-native runtime code without torch or dense substrate ownership. |
 | 8 | `docs/roadmap/repo-scan-upgrades/TODO.md` | Add scanners for bus schema, hot loops, state ownership, tick order, and contamination policies. |
@@ -32,7 +32,8 @@ Motor-learning work depends on three runtime tracks:
 
 1. The event spine must exist before actuator trace, release, and reafference events can be globally routed.
 2. ADC territory richness must improve before motor skill can use territory/correction feedback with useful locality.
-3. Dense-scan reduction must begin before new motor systems add more per-tick load.
+3. Engram resume integrity must be proven before long-run motor learning claims can rely on stopped/reloaded state.
+4. Dense-scan reduction must begin before new motor systems add more per-tick load.
 
 ## Runtime stance
 
@@ -65,6 +66,7 @@ This roadmap is complete when:
 - ADC receives rich territory-producing events,
 - void walkers/maps are globally visible and not duplicate-owned,
 - GDSP reads current territory data or explicitly declares next-tick scheduling,
+- engram resume restores real state history or fails loudly instead of silently loading config-only state,
 - repeated scan-heavy metrics are replaced by cached sparse snapshots where safe,
 - torch/dense substrate paths are gone from live runtime,
 - sparse neurogenesis has an implementation plan and first bounded policy tests,

docs/pages/roadmap/runtime-reboot-roadmap.md

@@ -23,7 +23,7 @@ reports/20260622/vdm_rt_void_bus_adc_loop_analysis.md
 3. `adc-territory-richness/TODO.md` — make ADC territory and boundary production meaningful.
 4. `scout-map-ownership/TODO.md` — make void walkers/maps first-class producers and remove duplicate ownership.
 5. `loop-scan-reduction/TODO.md` — replace repeated graph scans with cached sparse snapshots and reducer-fed metrics.
-6. `memory-gdsp-territory/TODO.md` — fix memory ownership, GDSP timing, stale observations, and territory availability.
+6. `memory-gdsp-territory/TODO.md` — fix memory ownership, GDSP timing, stale observations, territory availability, and engram resume integrity.
 7. `sparse-neurogenesis/TODO.md` — preserve legacy growth/pruning intent while removing dense/GPU substrate code.
 8. `repo-scan-upgrades/TODO.md` — improve Arachnid/repo-scan tooling for runtime dataflow analysis.
 9. `motor-learning-system/TODO.md` — motor-learning / decoder-removal upgrade path.

docs/roadmap/README.md

@@ -69,3 +69,46 @@ This checklist fixes ownership and timing issues around memory fields, territory
 - [ ] Step 3.2.2 — Emit `adc_folded_this_tick` per tick.
 - [ ] Step 3.2.3 — Emit `territory_folded_this_tick` per tick.
 - [ ] Step 3.2.4 — Emit `gdsp_territory_source` per GDSP invocation.
+
+## Phase 4 — Engram Resume Integrity
+
+### Task 4.1 — Reproduce the Aura reload failure mode
+
+- [ ] Step 4.1.1 — Create a short run with checkpointing enabled and nontrivial connectome, ADC, memory, map, SIE, and lexicon state.
+- [ ] Step 4.1.2 — Stop the run, reload from the checkpoint with `--load-engram`, and continue for a small number of ticks.
+- [ ] Step 4.1.3 — Compare pre-stop state against post-load state before the first continued tick.
+- [ ] Step 4.1.4 — Record whether only config-like values are restored while state history is lost.
+- [ ] Step 4.1.5 — Preserve an Aura-style regression fixture once the exact failure is reproduced.
+
+### Task 4.2 — Audit checkpoint save coverage
+
+- [ ] Step 4.2.1 — List every field saved by `core/memory/engram_io.py` for sparse checkpoints.
+- [ ] Step 4.2.2 — List every live runtime state field needed to resume a run without semantic reset.
+- [ ] Step 4.2.3 — Compare saved fields against `SparseConnectome`, ADC, memory field, trail/map reducers, SIE, phase, lexicon, and runtime counters.
+- [ ] Step 4.2.4 — Mark each field as `required`, `optional`, `derived`, or `do_not_resume`.
+- [ ] Step 4.2.5 — Add a manifest section to every checkpoint describing what was saved and what was intentionally omitted.
+
+### Task 4.3 — Audit checkpoint load behavior
+
+- [ ] Step 4.3.1 — Trace `runtime/helpers/engram.py::maybe_load_engram`.
+- [ ] Step 4.3.2 — Trace `core/memory/engram_io.py::load_engram`.
+- [ ] Step 4.3.3 — Trace phase-file hot loading through `runtime/phase.py`.
+- [ ] Step 4.3.4 — Verify load restores arrays, adjacency, active-state trackers, ADC territories, memory fields, and reducer state instead of only applying config.
+- [ ] Step 4.3.5 — Make incomplete restore fail loudly unless explicitly requested as config-only import.
+
+### Task 4.4 — Separate config load from state resume
+
+- [ ] Step 4.4.1 — Define `resume_state` semantics for full state restoration.
+- [ ] Step 4.4.2 — Define `load_config` semantics for importing run parameters without state history.
+- [ ] Step 4.4.3 — Prevent `--load-engram` from silently behaving like config-only load.
+- [ ] Step 4.4.4 — Add telemetry field `resume_mode` with values `none`, `state`, `config_only`, or `failed`.
+- [ ] Step 4.4.5 — Add telemetry field `resume_state_fields_loaded` listing restored sections.
+
+### Task 4.5 — Add resume parity tests
+
+- [ ] Step 4.5.1 — Save a checkpoint from a deterministic synthetic sparse runtime.
+- [ ] Step 4.5.2 — Load the checkpoint into a fresh runtime object.
+- [ ] Step 4.5.3 — Assert structural parity for node count, adjacency, active weights, stimulation state, and traversal state.
+- [ ] Step 4.5.4 — Assert ADC parity for territories, boundaries, cycle hits, TTL, and split state.
+- [ ] Step 4.5.5 — Assert memory/map parity for memory field, trail state, and event-map heads where those systems are enabled.
+- [ ] Step 4.5.6 — Assert continued tick numbering and checkpoint retention do not overwrite or prune the resumed history incorrectly.

docs/roadmap/memory-gdsp-territory/TODO.md

@@ -78,3 +78,21 @@ This checklist protects the fresh independent runtime identity. It is allowed to
 - [ ] Step 3.2.2 — Prove `import vdm_rt.core.sparse_connectome` works without torch.
 - [ ] Step 3.2.3 — Prove `import vdm_rt.run_nexus` works from the fresh repo root.
 - [ ] Step 3.2.4 — Add these import checks to CI.
+
+## Phase 4 — TODO Comment Hygiene
+
+### Task 4.1 — Classify live-code TODOs
+
+- [x] Step 4.1.1 — Remove stale `TODO REMOVE DENSE SCANS` comments from `nexus.py`.
+- [x] Step 4.1.2 — Rewrite the scan-reduction TODO in `core/metrics.py` to point at `SparseConnectome.metrics_snapshot()` and event-spine reducers.
+- [x] Step 4.1.3 — Rewrite the legacy SIE TODO in `core/global_system.py` as a keep/port/archive/delete classification task.
+- [ ] Step 4.1.4 — Add a TODO scanner that reports every live-code TODO with file, line, category, and owner.
+- [ ] Step 4.1.5 — Require each live-code TODO to be classified as `bug`, `cleanup`, `port`, `archive`, `performance`, or `roadmap-linked`.
+
+### Task 4.2 — Remove stale dense backend compatibility
+
+- [x] Step 4.2.1 — Remove `FORCE_DENSE` runtime backend switching from `nexus.py`.
+- [x] Step 4.2.2 — Remove CLI dense/sparse selection flags.
+- [x] Step 4.2.3 — Remove `sparse_mode` forwarding from the runtime launcher and process manager.
+- [x] Step 4.2.4 — Update the dense-connectome guard so dense backend selection cannot reappear silently.
+- [ ] Step 4.2.5 — Add a scan report that distinguishes runtime dense scans from accurate comments saying a module avoids dense scans.

docs/roadmap/runtime-cleanup-guardrails/TODO.md

@@ -51,7 +51,7 @@ class Nexus:
     def __init__(self, run_dir: str, N:int=1000, k:int=12, hz:int=10,
                  domain:str='biology_consciousness', use_time_dynamics:bool=True,
                  log_every:int=1, checkpoint_every:int=0, seed:int=0,
-                 sparse_mode:bool=False, threshold:float=0.15, lambda_omega:float=0.1,
+                 threshold:float=0.15, lambda_omega:float=0.1,
                  candidates:int=64, walkers:int=256, hops:int=3, status_interval:int=1,
                  bundle_size:int=3, prune_factor:float=0.10,
                  speak_auto:bool=True, speak_z:float=1.0, speak_hysteresis:float=1.0,
@@ -121,49 +121,14 @@ def __init__(self, run_dir: str, N:int=1000, k:int=12, hz:int=10,
         self._ng2 = {}  # bigram: w1 -> {w2: count}
         self._ng3 = {}  # trigram: (w1,w2) -> {w3: count}
 
-        # Sparse-first backend policy (void-faithful, no scans):
-        # Runtime uses SparseConnectome by default. Dense is validation-only via FORCE_DENSE=1.
-        use_dense = str(os.getenv("FORCE_DENSE", "0")).strip().lower() in ("1", "true", "yes", "on", "y", "t")
+        from vdm_rt.core.sparse_connectome import SparseConnectome
 
-        # Keep 'sparse_mode' arg for compatibility but ignore it; log once.
-        if sparse_mode:
-            try:
-                self.logger.info("deprecated_arg_sparse_mode_ignored", extra={"extra": {"arg": bool(sparse_mode)}})
-            except Exception:
-                pass
-
-        # Select implementation with dynamic import to avoid accidental dense usage
-        try:
-            if use_dense: # TODO REMOVE DENSE SCANS
-                from vdm_rt.core.connectome import Connectome as _Conn  # validation-only
-            else:
-                from vdm_rt.core.sparse_connectome import SparseConnectome as _Conn
-        except Exception:
-            # Fallback to sparse on any import failure
-            from vdm_rt.core.sparse_connectome import SparseConnectome as _Conn
-
-        # Instantiate connectome (both backends accept the same constructor args here)
-        try: # TODO REMOVE DENSE SCANS
-            self.connectome = _Conn(
-                N=self.N, k=self.k, seed=self.seed,
-                threshold=threshold, lambda_omega=lambda_omega,
-                candidates=candidates, traversal_walkers=walkers, traversal_hops=hops,
-                bundle_size=bundle_size, prune_factor=prune_factor
-            )
-            if use_dense:
-                try:
-                    self.logger.info("backend_dense_forced", extra={"extra": {"reason": "FORCE_DENSE"}})
-                except Exception:
-                    pass
-        except Exception:
-            # Ensure we have a working sparse connectome if constructor failed
-            from vdm_rt.core.sparse_connectome import SparseConnectome as _SConn
-            self.connectome = _SConn(
-                N=self.N, k=self.k, seed=self.seed,
-                threshold=threshold, lambda_omega=lambda_omega,
-                candidates=candidates, traversal_walkers=walkers, traversal_hops=hops,
-                bundle_size=bundle_size, prune_factor=prune_factor
-            )
+        self.connectome = SparseConnectome(
+            N=self.N, k=self.k, seed=self.seed,
+            threshold=threshold, lambda_omega=lambda_omega,
+            candidates=candidates, traversal_walkers=walkers, traversal_hops=hops,
+            bundle_size=bundle_size, prune_factor=prune_factor
+        )
         # Load engram if provided (after backend selection)
         # Defer engram loading until after ADC is initialized to avoid spurious errors/logs.
         # The actual load (with logging) happens below after ADC is constructed.
@@ -179,7 +144,7 @@ def __init__(self, run_dir: str, N:int=1000, k:int=12, hz:int=10,
         self.stim_group_size = int(max(1, stim_group_size))
         self.stim_amp = float(stim_amp)
         self.stim_max_symbols = int(max(1, stim_max_symbols))
-        try:  # TODO REMOVE DENSE SCANS
+        try:
             if hasattr(self.connectome, "_stim_decay"):
                 self.connectome._stim_decay = float(stim_decay)
         except Exception:
@@ -236,7 +201,7 @@ def __init__(self, run_dir: str, N:int=1000, k:int=12, hz:int=10,
         self.adc = ADC(r_attach=float(r_attach), ttl_init=int(ttl_init), split_patience=int(split_patience))
         # Attach bus to connectome so walkers can publish Observation events
         try:
-            self.connectome.bus = self.bus # TODO REMOVE DENSE SCANS
+            self.connectome.bus = self.bus
         except Exception:
             pass