fix(harness): serve the system prompt reliably — SYSTEM_MD env-forward + materialization + observability

.motoko/config/ollama/compaction_ai.json

@@ -0,0 +1,5 @@
+{
+  "model": "ollama/qwen3.6:35b-a3b-mxfp8",
+  "threshold_pct": 50,
+  "keep_recent": 6
+}

src/core/agent_loop_v2.ail

@@ -55,7 +55,7 @@ import src/core/types (Msg, ToolBackend)
 import src/core/config (CostRates)
 import std/option (Option, Some, None)
 import pkg/sunholo/motoko_ext_abi/types (BudgetPlan, ExtCtx, ExtRuntime, PolicyDefault, ResponseInterceptDecision, FinalizeDecision, ToolPolicyDecision, ToolHandleDecision, ExtensionHooks)
-import src/core/compaction (compact_step)
+import src/core/compaction (compact_step, compact_step_actual)
 import src/core/context_usage (context_limit_for)
 import src/core/ext/runtime (dispatch_response_intercept, dispatch_solver_candidate, dispatch_tool_policy, dispatch_tool_handle, dispatch_pre_step)
 import pkg/sunholo/motoko_ext_abi/types (PreStepDecision, PassThrough, Compacted)
@@ -166,11 +166,12 @@ type LoopTotals = {
   cache_read: int,
   cache_creation: int,
   cost_millicents: int,
-  cost_warned_pct: int
+  cost_warned_pct: int,
+  last_input_tokens: int
 }
 
 pure func zero_totals() -> LoopTotals {
-  { input_tokens: 0, output_tokens: 0, cache_read: 0, cache_creation: 0, cost_millicents: 0, cost_warned_pct: 0 }
+  { input_tokens: 0, output_tokens: 0, cache_read: 0, cache_creation: 0, cost_millicents: 0, cost_warned_pct: 0, last_input_tokens: 0 }
 }
 
 -- Emit a schema-v1 event: prepends schema_version + session_id + type to
@@ -1031,6 +1032,22 @@ func any_writefile_attempt(msgs: [Message]) -> bool {
   }
 }
 
+-- Partition out ALL system messages (the AILANG reference / instructions) from the
+-- conversation. AI 101: the system prompt is a CONSTANT, not mutable history — it must
+-- never be subject to compaction. We split it off before BOTH compaction layers and
+-- re-prepend it for the provider call, so no compaction strategy can drop or reorder it.
+type MsgSysSplit = { sys: [Message], conv: [Message] }
+func partition_system_msgs(msgs: [Message]) -> MsgSysSplit {
+  match msgs {
+    [] => { sys: [], conv: [] },
+    m :: tail => {
+      let r = partition_system_msgs(tail);
+      if m.role == "system" then { sys: m :: r.sys, conv: r.conv }
+      else { sys: r.sys, conv: m :: r.conv }
+    }
+  }
+}
+
 func loop_v2(
   rt: ExtRuntime,
   task: string,
@@ -1076,25 +1093,35 @@ func loop_v2(
   -- Build extension context for this step (needed for dispatch_pre_step)
   let ctx = mk_v2_ext_ctx(task, step_idx, model, workdir, env_url, hybrid_tools, budget, msgs);
 
-  -- DP0a: let extensions compact first (e.g., AI summarization)
-  let after_ext = match dispatch_pre_step(rt, ctx, messages_to_msgs(msgs)) {
+  -- AI 101: the system message (AILANG reference) is a CONSTANT, never mutable history.
+  -- Split it off so NEITHER compaction layer can drop/reorder it; re-prepended below for
+  -- the provider call. Without this the model loses its syntax reference after compaction
+  -- and drifts into invalid syntax.
+  let sys_split = partition_system_msgs(msgs);
+  let sys_prefix = sys_split.sys;
+  let conv_msgs = sys_split.conv;
+
+  -- DP0a: let extensions compact first (e.g., AI summarization) — on the conversation only
+  let after_ext = match dispatch_pre_step(rt, ctx, messages_to_msgs(conv_msgs)) {
     Compacted(compacted_msgs, note) => {
       let _ = emit_event(session_id, "compaction_extension", [
         kv("step", jnum(_int_to_float(step_idx))),
         kv("note", js(note))
       ]);
       Ok(msgs_to_messages(compacted_msgs))
     },
-    PassThrough => Ok(msgs)
+    PassThrough => Ok(conv_msgs)
   };
 
   match after_ext {
     Err(e) => Err(e),
     Ok(msgs_after_ext) => {
   -- DP0b: structural compaction as fallback
   -- compact_step is a no-op below 70% usage; elides old tool_result content
-  -- at 70-85% and 85-95%; at ≥95% tries emergency compaction.
-  match compact_step(msgs_after_ext, model) {
+  -- at 70-85% and 85-95%; at ≥95% tries emergency compaction. The elision LEVEL is
+  -- driven off the provider's ACTUAL last input_tokens (ground truth) rather than a
+  -- char-count estimate (the old chars/4 over-counted ~1.9x -> premature compaction).
+  match compact_step_actual(msgs_after_ext, model, totals.last_input_tokens) {
     Err(compaction_reason) => {
       let _ = emit_event(session_id, "compaction_exhausted", [
         kv("step", jnum(_int_to_float(step_idx))),
@@ -1108,7 +1135,10 @@ func loop_v2(
         retryable: false
       })
     },
-    Ok(compacted_msgs) => {
+    Ok(conv_compacted) => {
+    -- Re-prepend the pinned system message (split off above) for the provider call and
+    -- carry-forward, so the AILANG reference is ALWAYS present and never compacted.
+    let compacted_msgs = sys_prefix ++ conv_compacted;
     let stream_id = "step-${show(step_idx)}";
     let _ = emit_event(session_id, "thinking_stream_start", [
       kv("step", jnum(_int_to_float(step_idx))),
@@ -1191,7 +1221,8 @@ func loop_v2(
           cache_read:      new_cache_read,
           cache_creation:  new_cache_creation,
           cost_millicents: new_total_cost,
-          cost_warned_pct: new_warned_pct
+          cost_warned_pct: new_warned_pct,
+          last_input_tokens: result.input_tokens
         };
 
         -- M3 DP1a: response-intercept hook. Extensions can synthesize a

src/core/compaction.ail

@@ -22,7 +22,7 @@
 
 module src/core/compaction
 
-import std/ai (Message)
+import std/ai (Message, ToolCall)
 import std/string (length, substring)
 import std/list (foldl)
 import std/result (Result, Ok, Err)
@@ -31,18 +31,35 @@ import src/core/context_usage (context_limit_for)
 -- Token estimate for [Message]: sum all content-field lengths, divide by 4.
 -- Same approximation as estimate_tokens for [Msg] in context_usage.ail.
 -- contracts: SKIPPED — uses foldl (Z3 fragment requires non-higher-order)
+-- Sum the serialized size of a message's tool_calls (name + arguments). WriteFile/EditFile carry the FULL
+-- file content in `arguments`, re-sent every turn — counting only `content` (as before) massively undercounts
+-- those turns, so compaction fires too late -> context overflow (docx hit input 263259 > 262144).
+pure func tool_calls_chars(tcs: [ToolCall]) -> int {
+  foldl(\acc tc. acc + length(tc.name) + length(tc.arguments), 0, tcs)
+}
+
 export pure func estimate_tokens_messages(msgs: [Message]) -> int {
-  let total = foldl(\acc m. acc + length(m.content), 0, msgs);
-  (total + 3) / 4
+  let total = foldl(\acc m. acc + length(m.content) + tool_calls_chars(m.tool_calls), 0, msgs);
+  -- chars/7: MEASURED ratio for AILANG code + DOCX XML is ~7.5 chars/token (ollama
+  -- actual 74K vs old chars/4 estimate 138K at the same step -> the old /4 over-counted
+  -- ~1.9x, firing compaction at ~31% of the real 262144 window -> working-set thrash).
+  -- Slightly conservative (/7 > real 7.5) so we compact just BEFORE the true limit.
+  (total + 6) / 7
 }
 
 -- Context usage as integer percent (0-100). Returns 0 for unknown models.
 -- Uses integer arithmetic throughout to avoid float precision issues.
 -- contracts: SKIPPED — depends on foldl-based estimate_tokens_messages
 export pure func usage_percent(msgs: [Message], model: string) -> int {
   let limit = context_limit_for(model);
-  if limit == 0 then 0
-  else (estimate_tokens_messages(msgs) * 100) / limit
+  -- Reserve ~75k headroom for the model's OUTPUT. context_limit is the TOTAL input+output window,
+  -- so compaction must keep INPUT well below it. docx overflowed at input 263259 > 262144 because
+  -- usage was measured against the FULL window — compaction never reserved output room and let input
+  -- crowd the hard limit. 75k covers the 65536 output cap + estimate margin. Small (rotation)
+  -- contexts stay ~0% so this only bites large-context tasks (compacts them earlier, as intended).
+  let effective = limit - 75000;
+  if limit == 0 || effective <= 0 then 0
+  else (estimate_tokens_messages(msgs) * 100) / effective
 }
 
 -- Count tool-role messages in the list.
@@ -132,6 +149,25 @@ export pure func compact_step(msgs: [Message], model: string) -> Result[[Message
   else Ok(msgs)
 }
 
+-- M-MOTOKO-ACTUAL-COMPACT: drive elision off the PROVIDER's ACTUAL input_tokens (last step)
+-- instead of a chars-ratio estimate. The estimate failed both ways — chars/4 over-counted ~1.9x
+-- (premature thrash); chars/7 under-counted (never fired -> 260K overflow). The real count is
+-- ground truth. Proactive tiers (start gentle well before the limit so a one-step jump can't
+-- overflow): keep_last=10 at 60%, 5 at 75%, emergency at 85% of effective(=limit-75000). Elides
+-- only OLD tool_results (the measured 96K bloat = BashExec/cat XML dumps); recent working set stays.
+-- Falls back to the estimate-based compact_step when actual is unknown (step 0).
+export pure func compact_step_actual(msgs: [Message], model: string, actual_input: int) -> Result[[Message], string] {
+  if actual_input <= 0 then compact_step(msgs, model)
+  else {
+    let effective = context_limit_for(model) - 75000;
+    let pct = if effective > 0 then (actual_input * 100) / effective else 0;
+    if pct >= 85 then try_emergency_compaction(msgs, model)
+    else if pct >= 75 then Ok(elide_old_tool_results(msgs, 5))
+    else if pct >= 60 then Ok(elide_old_tool_results(msgs, 10))
+    else Ok(msgs)
+  }
+}
+
 -- Unknown model → usage 0% → no compaction.
 func unknown_model_skips_compaction() -> bool
   tests [((), true)]

src/core/rpc.ail

@@ -17,7 +17,9 @@ import src/core/test/stub_step (LiveAI)
 import std/io     (println, exit)
 import std/json   (Json, encode, jo, ja, kv, js, jb, jnum)
 import std/list   (foldl)
-import std/string (length)
+import std/string (length, substring, contains)
+import std/env    (getEnvOr)
+import std/math   (intToFloat)
 import std/fs     (readFile, fileExists)
 import src/core/types  (Msg, version)
 import src/core/prompts (with_cache_hint, with_agents_context)
@@ -193,7 +195,12 @@ export func run_with_config(cfg: RuntimeConfig, inv: InvocationConfig) -> () ! {
   ])));
 
   let hint = get_hint(task);
-  let system_md_path = cfg.agent.system_prompt;
+  -- System prompt is env-AUTHORITATIVE: the SYSTEM_MD env wins over any profile config
+  -- that may silently set system_prompt="" (the AI-101 failure where AILANG tasks lost
+  -- their language reference). One explicit source of truth, env over config.
+  let cfg_system_path = cfg.agent.system_prompt;
+  let env_system_path = getEnvOr("SYSTEM_MD", "");
+  let system_md_path = if env_system_path != "" then env_system_path else cfg_system_path;
   let raw_system =
     if system_md_path == "" then ""
     else if fileExists(system_md_path)
@@ -225,6 +232,27 @@ export func run_with_config(cfg: RuntimeConfig, inv: InvocationConfig) -> () ! {
   };
   let system = dispatch_build_system_prompt(ext_runtime, system_ctx, system_with_hint);
 
+  -- OBSERVABILITY + LOUD FAILURE (no silent fallbacks): record exactly what system prompt
+  -- the model receives on every run, and shout if a source was set but produced nothing.
+  -- After 4 days of guessing, "is the AILANG reference served?" is now one line in the log.
+  let sys_chars = length(system);
+  let sys_prev_end = if sys_chars < 160 then sys_chars else 160;
+  let _ = emit(encode(jo([
+    kv("type", js("system_prompt_built")),
+    kv("source", js(if env_system_path != "" then "env:SYSTEM_MD"
+                    else if cfg_system_path != "" then "config:agent.system_prompt"
+                    else "none")),
+    kv("chars", jnum(intToFloat(sys_chars))),
+    kv("has_ailang_reference", jb(contains(system, "AILANG LANGUAGE REFERENCE"))),
+    kv("preview", js(substring(system, 0, sys_prev_end)))])));
+  let _ =
+    if system_md_path != "" && raw_system == "" then {
+      let _ = emit(encode(jo([
+        kv("type", js("error")),
+        kv("message", js("SYSTEM PROMPT EMPTY: '${system_md_path}' was set but produced no content — model running WITHOUT its system prompt."))])));
+      ()
+    } else ();
+
   let init: [Msg] = [
     { role: "system", content: system, tool_calls: [], tool_call_id: "" },
     { role: "user", content: task, tool_calls: [], tool_call_id: "" }

src/tui/src/index.ts

@@ -735,6 +735,20 @@ async function main(): Promise<void> {
       process.env.SYSTEM_MD = materialized;
     }
   }
+  // SYSTEM_MD pointing OUTSIDE the workspace is silently unreadable: the AILANG runtime is
+  // FS-sandboxed to workdir (AILANG_FS_SANDBOX), so rpc.ail's readFile returns empty and the
+  // model runs with NO system prompt (observed via the system_prompt_built event: chars=0).
+  // Materialize an external path into the workspace — the same treatment --system-prompt gets —
+  // so any harness that writes a prompt to /tmp (e.g. the AILANG eval scripts) just works,
+  // instead of every caller having to remember to place it inside the workspace.
+  const sysMdPath = (process.env.SYSTEM_MD ?? "").trim();
+  if (sysMdPath !== "") {
+    const rel = path.relative(path.resolve(workdir), path.resolve(sysMdPath));
+    if (rel.startsWith("..") || path.isAbsolute(rel)) {
+      const materialized = materializeSystemPromptArg(sysMdPath, workdir);
+      if (materialized !== null) process.env.SYSTEM_MD = materialized;
+    }
+  }
   // M-MOTOKO-EVAL-HARNESS-HARDENING follow-up (2026-05-08): default
   // ENV_PORT to 0 = let the kernel pick a free port atomically when
   // startEnvServer binds. The wrapper used to do its own pick_free_port

src/tui/src/runtime-process-env.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from "@jest/globals";
+import { autoForwardedEnvKeys } from "./runtime-process.js";
+import { CORE_MAP, EXTENSION_MAPS } from "./config.js";
+
+// Drift-guard for the recurring "env var scrubbed by the childEnv allowlist → feature
+// silently off" bug (hit 5×: SYSTEM_MD, MOTOKO_REPO, MOTOKO_PERSIST_RETRIES,
+// AILANG_OLLAMA_MAX_TOKENS, AILANG_STDLIB_PATH). The allowlist must be DERIVED from the
+// config maps + the motoko/ailang namespaces, never hand-maintained as a second source of
+// truth. These tests fail loudly if forwarding ever stops covering a config-mapped var.
+describe("autoForwardedEnvKeys — env-forward drift guard", () => {
+  it("forwards every CORE_MAP env var (incl. SYSTEM_MD)", () => {
+    const keys = new Set(autoForwardedEnvKeys({}));
+    for (const { env } of Object.values(CORE_MAP)) {
+      expect(keys.has(env)).toBe(true);
+    }
+    expect(keys.has("SYSTEM_MD")).toBe(true); // agent.system_prompt — the 4-day bug
+  });
+
+  it("forwards every EXTENSION_MAPS env var", () => {
+    const keys = new Set(autoForwardedEnvKeys({}));
+    for (const map of Object.values(EXTENSION_MAPS)) {
+      for (const { env } of Object.values(map)) {
+        expect(keys.has(env)).toBe(true);
+      }
+    }
+  });
+
+  it("auto-forwards motoko/ailang-namespaced vars but not arbitrary host vars", () => {
+    const keys = new Set(
+      autoForwardedEnvKeys({
+        SYSTEM_MD: "x",
+        MOTOKO_BRAND_NEW_FEATURE: "y",
+        AILANG_BRAND_NEW_FLAG: "z",
+        HOME: "/home/nobody",
+        SECRET_TOKEN: "should-not-leak",
+      }),
+    );
+    expect(keys.has("MOTOKO_BRAND_NEW_FEATURE")).toBe(true);
+    expect(keys.has("AILANG_BRAND_NEW_FLAG")).toBe(true);
+    expect(keys.has("SECRET_TOKEN")).toBe(false);
+  });
+});

src/tui/src/runtime-process.ts

@@ -4,6 +4,26 @@ import * as path from "path";
 import * as readline from "readline";
 import { createOhMyPiSession } from "./ohMyPi/session-adapter.js";
 import { dispatchOhMyPiTool } from "./ohMyPi/dispatcher.js";
+import { CORE_MAP, EXTENSION_MAPS } from "./config.js";
+
+// autoForwardedEnvKeys — the SINGLE source of truth for which parent env vars reach the
+// AILANG runtime subprocess. The childEnv allowlist below scrubs everything not listed,
+// which has silently broken at least FIVE env-gated features whose var was forgotten:
+// SYSTEM_MD, MOTOKO_REPO, MOTOKO_PERSIST_RETRIES, AILANG_OLLAMA_MAX_TOKENS, AILANG_STDLIB_PATH
+// (AILANG_OLLAMA_MAX_TOKENS was even hand-added twice). Root cause: the allowlist was a second
+// source of truth that drifts from config.ts's CORE_MAP. Instead, derive forwarding from
+// CORE_MAP/EXTENSION_MAPS plus the motoko/ailang namespaces, so a new env-gated feature reaches
+// the .ail BY DEFAULT instead of failing silently. Exported for the drift-guard test.
+export function autoForwardedEnvKeys(env: NodeJS.ProcessEnv): string[] {
+  const keys = new Set<string>([
+    ...Object.values(CORE_MAP).map((e) => e.env),
+    ...Object.values(EXTENSION_MAPS).flatMap((m) => Object.values(m).map((e) => e.env)),
+  ]);
+  for (const k of Object.keys(env)) {
+    if (k.startsWith("MOTOKO_") || k.startsWith("AILANG_") || k === "SYSTEM_MD") keys.add(k);
+  }
+  return [...keys];
+}
 
 export interface DelegatedExecReq {
   cmd: string;
@@ -305,6 +325,12 @@ export class RuntimeProcess {
       GOOGLE_API_KEY: process.env.GOOGLE_API_KEY,
       EXA_API_KEY: process.env.EXA_API_KEY,
       CLICKSTACK_INGESTION_KEY: process.env.CLICKSTACK_INGESTION_KEY,
+      // SYSTEM_MD — path to the system-prompt file (for AILANG tasks, the language
+      // reference). rpc.ail reads it via getEnvOr("SYSTEM_MD"). Without forwarding it
+      // here the explicit allowlist scrubs it (same gotcha as MOTOKO_PERSIST_RETRIES /
+      // MOTOKO_REPO above), so the model runs with NO system prompt — silently, chars=0.
+      // This was invisible for days until the system_prompt_built event surfaced it.
+      SYSTEM_MD: process.env.SYSTEM_MD ?? "",
       AILANG_FS_SANDBOX: workdir,
       MOTOKO_STREAM_EVENTS: process.env.MOTOKO_STREAM_EVENTS ?? "1",
       // M-MOTOKO-HEADLESS (2026-05-08): when stdin is not a TTY, set
@@ -423,6 +449,18 @@ export class RuntimeProcess {
     if (openaiBaseUrl.trim() !== "") childEnv.OPENAI_BASE_URL = openaiBaseUrl;
     if (aiOptionsJson.trim() !== "") childEnv.MOTOKO_AI_OPTIONS_JSON = aiOptionsJson;
 
+    // SYSTEMIC ENV-FORWARD GUARD: auto-forward every config-mapped (CORE_MAP) and
+    // motoko/ailang-namespaced var that wasn't already set explicitly above. This is
+    // what kills the recurring "forgot to add the var to the allowlist → feature silently
+    // off" bug class (SYSTEM_MD, MOTOKO_REPO, MOTOKO_PERSIST_RETRIES, AILANG_OLLAMA_*,
+    // AILANG_STDLIB_PATH …). Explicit entries above win (they carry computed values /
+    // defaults); everything else now reaches the .ail by default instead of vanishing.
+    for (const k of autoForwardedEnvKeys(process.env)) {
+      if (!(k in childEnv) && process.env[k] !== undefined) {
+        childEnv[k] = process.env[k];
+      }
+    }
+
     // M-MOTOKO-EVAL-HARNESS-HARDENING gap #6 (2026-05-08): mirror the
      // requested profile dir from MOTOKO_REPO into <workdir>/.motoko/config
      // when the workdir doesn't already have one. AILANG_FS_SANDBOX