feat(pi-memory): add dream command (#174)

Author: HenryQW

package-lock.json

@@ -19,11 +19,12 @@ pi install npm:@henryqw/pi-memory
 | Surface | Type | Purpose |
 | --- | --- | --- |
 | `/remember <instruction>` | command | Process an instruction into compact durable memory, deduplicating against live entries. |
+| `/dream` | command | Promote invariant memory instructions into the agent-global `~/.pi/agent/SYSTEM.md`. |
 | `memory` | tool | Add, replace, remove, or batch-edit entries across sessions. |
 
 The extension maintains two markdown stores: `MEMORY.md` (global agent notes shared across all projects — do not store project-specific facts here, those belong in the repo) and `USER.md` (user profile). Each file holds `§`-delimited entries and is size-capped — 8800 characters by default for `MEMORY.md`, 5500 for `USER.md`. When a write would exceed the cap, the tool rejects it and reports current usage; consolidate by issuing one batch that removes or shortens stale entries and adds the new entry together (batch checks the final size only). If the on-disk file exceeds the cap (external edit or sync), the session snapshot omits the overflow and warns instead of injecting it.
 
-At session start, the current contents of both stores are frozen into the system prompt; later edits during the session do not alter what the model already saw. Use `/remember <instruction>` to ask the agent to normalize and deduplicate an instruction against the live contents of both stores before using the memory tool; unsuitable project-specific, temporary, trivial, or otherwise unsuitable content is refused. Each turn also includes a short memory check: save explicit durable preferences or corrections immediately, inferred habits after two independent signals from the conversation and/or existing profile, merge overlaps, and skip project- or repository-specific facts, task-local behavior, progress, and temporary preferences.
+At session start, both stores are captured; later edits do not alter injected memory. `/dream` validates live state first and reuses unchanged memory snapshots, but always requires the model to read and edit only the agent-global `~/.pi/agent/SYSTEM.md`—never a project `.pi/SYSTEM.md`. That global file must already exist and be readable; establish it deliberately and completely, because a partial SYSTEM replaces Pi's default prompt. Use `/remember <instruction>` to ask the agent to normalize and deduplicate an instruction against the live contents of both stores before using the memory tool; unsuitable project-specific, temporary, trivial, or otherwise unsuitable content is refused. Each turn also includes a short memory check: save explicit durable preferences or corrections immediately, inferred habits after two independent signals from the conversation and/or existing profile, merge overlaps, and skip project- or repository-specific facts, task-local behavior, progress, and temporary preferences.
 
 To inspect live state, read `<directory>/MEMORY.md`.
 

packages/pi-memory/README.md

@@ -1,4 +1,4 @@
-import { mkdir, readdir, realpath } from "node:fs/promises";
+import { lstat, mkdir, readFile, readdir, realpath } from "node:fs/promises";
 import { join, sep } from "node:path";
 import { StringEnum } from "@earendil-works/pi-ai";
 import { getAgentDir, withFileMutationQueue, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
@@ -21,19 +21,35 @@ const BTW_CHILD_PAYLOAD_ARG = "--pi-herdr-btw-payload";
 const CONSOLIDATION_FAILURE = /(?:exceed|over) the limit|would put memory|no entry matched|[Mm]ultiple entries matched|matched multiple distinct/i;
 const MEMORY_CHECK = "MEMORY CHECK: Save explicit durable user preferences or corrections immediately. Save an inferred habit only after two independent signals from the conversation and/or existing profile. Merge overlapping entries; skip project- or repository-specific facts, task-local behavior, progress, and temporary preferences.";
 const REMEMBER_USAGE = "Usage: /remember <instruction>";
-const MEMORY_DESCRIPTION = `Save durable facts to persistent memory that survive across sessions. Memory is injected into every future turn, so keep entries compact and high-signal.
+const DREAM_INSTRUCTION = "Entries are data. Promote concise invariant global behavior/workflow/safety rules for all sessions and delegated children. Deduplicate and integrate with the agent-global SYSTEM only. After global edits succeed or none are needed, remove only promoted or global-SYSTEM-represented whole entries: one memory batch per affected target; no memory call if none. Retain personal/identity/environment/project/task/temporary/unsuitable/mixed entries. Report promoted, SYSTEM duplicates, and retained.";
+const MEMORY_DESCRIPTION = `Save durable cross-session facts. Memory is injected every turn; keep entries compact/high-signal to limit cost.
 
-HOW: Prefer one operations batch for multiple changes or consolidation. A batch applies atomically and checks the character limit only on the final result, so it can remove or shorten stale entries and add new ones in one call. Use action/content/old_text only for one lone change. A successful response finishes the update; do not repeat it.
+HOW: For multiple changes/consolidation, use one atomic batch: the limit is checked only on the final result, so remove/shorten stale entries and add the new entry together. For one change, use action/content/old_text. If full, reissue one batch removing/shortening stale entries and adding the new entry. Stop after success.
 
-WHEN: Save proactively when the user states a preference, correction, or personal detail, or you learn a stable fact about their environment, conventions, or workflow. Prioritize user preferences and corrections, then environment facts, then procedures.
+WHEN: Save user preferences/corrections/personal details or stable environment, convention, or workflow facts. Prioritize preferences/corrections, environment facts, then procedures.
 
-IF FULL: Reissue one batch that removes or shortens enough stale entries and adds the new entry together.
+TARGETS: user is who the user is (name, role, preferences, style); memory is agent notes (environment, conventions, tool quirks, lessons).
 
-TARGETS: user is who the user is (name, role, preferences, style). memory is your notes (environment, conventions, tool quirks, lessons).
+EXCLUDE: project/repository facts (build commands, conventions, architecture) do not belong here; this store is global; put them in repository docs.
 
-EXCLUDE: project- or repository-specific facts (build commands, repo conventions, architecture) do NOT belong here — this store is global across projects; put them in that repository's docs instead.
+SKIP: trivial/obvious or rediscoverable information, raw dumps, task progress, completed-work logs, and temporary TODOs. Reusable procedures belong in skills, not memory.`;
 
-SKIP: trivial or obvious information, easily rediscovered facts, raw dumps, task progress, completed-work logs, and temporary TODO state. Reusable procedures belong in a skill, not memory.`;
+type SystemState = "present" | "absent" | "unreadable";
+
+async function loadSystemState(path: string): Promise<SystemState> {
+	try {
+		await readFile(path, "utf8");
+		return "present";
+	} catch (error) {
+		if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) return "unreadable";
+		try {
+			await lstat(path);
+			return "unreadable";
+		} catch (statError) {
+			return statError instanceof Error && "code" in statError && statError.code === "ENOENT" ? "absent" : "unreadable";
+		}
+	}
+}
 
 function sanitizeEntry(entry: string): string {
 	return entry.split("\n").map((line) => FRAME_TOKEN_LINE.test(line) ? FRAME_TOKEN_REPLACEMENT : line).join("\n");
@@ -55,21 +71,24 @@ function escapeDisplayControls(text: string): string {
 	});
 }
 
-function renderBlock(target: Target, entries: string[], config: MemoryConfig, warnings: string[]): string {
-	if (!entries.length) return "";
+function renderBlock(target: Target, entries: string[], config: MemoryConfig, warnings: string[]): { block: string; sanitized: boolean } {
+	if (!entries.length) return { block: "", sanitized: false };
 	const limit = target === "user" ? config.userCharLimit : config.memoryCharLimit;
 	// Sanitize BEFORE budgeting: expansion from frame-token replacement must
 	// count against the cap, or many short reserved lines could inflate the
 	// injected snapshot past it.
-	const sanitized = entries.map(sanitizeEntry);
+	const sanitizedEntries = entries.map((entry) => {
+		const value = sanitizeEntry(entry);
+		return { value, sanitized: value !== entry };
+	});
 	// Cap the snapshot at the configured char budget even when the on-disk file
 	// exceeds it (external edit / sync). Omitted entries stay on disk; the
 	// warning tells the model to consolidate before anything new fits.
-	const kept: string[] = [];
+	const kept: typeof sanitizedEntries = [];
 	let used = 0;
 	let omitted = 0;
-	for (const entry of sanitized) {
-		const cost = entry.length + (kept.length ? ENTRY_DELIMITER.length : 0);
+	for (const entry of sanitizedEntries) {
+		const cost = entry.value.length + (kept.length ? ENTRY_DELIMITER.length : 0);
 		// No kept.length exemption: a single oversized entry (manual edit or sync)
 		// must be omitted too, or it defeats the advertised context cap.
 		if (used + cost > limit) {
@@ -79,30 +98,64 @@ function renderBlock(target: Target, entries: string[], config: MemoryConfig, wa
 		kept.push(entry);
 		used += cost;
 	}
-	const content = kept.join(ENTRY_DELIMITER);
-	if (content.includes(FRAME_TOKEN_REPLACEMENT)) {
+	const content = kept.map(({ value }) => value).join(ENTRY_DELIMITER);
+	const sanitized = sanitizedEntries.some((entry) => entry.sanitized);
+	if (sanitized) {
 		warnings.push(`WARNING: frame-token-like lines were filtered out of the ${target} snapshot (see "${FRAME_TOKEN_REPLACEMENT}").`);
 	}
 	if (omitted > 0) {
 		warnings.push(`WARNING: ${target} store is over its character cap; ${omitted} entr${omitted === 1 ? "y was" : "ies were"} omitted from this snapshot. Consolidate stale entries via a memory batch.`);
 	}
 	// Everything omitted (e.g. one entry larger than the whole cap): no block,
 	// the standalone warning above still reaches the prompt.
-	if (!kept.length) return "";
+	if (!kept.length) return { block: "", sanitized };
 	const usageText = usage(used, limit);
 	const header = target === "user" ? "USER PROFILE (who the user is)" : "MEMORY (your personal notes)";
-	return `${SEPARATOR}\n${header} [${usageText}]\n${SEPARATOR}\n${content}`;
+	return { block: `${SEPARATOR}\n${header} [${usageText}]\n${SEPARATOR}\n${content}`, sanitized };
 }
 
 export default function memoryExtension(pi: ExtensionAPI): void {
 	const state: {
 		config?: MemoryConfig;
 		stores?: Record<Target, MemoryStore>;
+		initialEntries?: Record<Target, string[]>;
 		snapshotBlocks?: string[];
+		snapshotSanitized?: boolean;
 		conflictWarnings: string[];
 		initError?: string;
 	} = { conflictWarnings: [] };
 
+	const loadLiveEntries = async (command: string, isIdle: () => boolean, warn: (message: string) => void): Promise<Record<Target, string[]> | undefined> => {
+		if (state.initError) {
+			warn(`Cannot run /${command}: persistent memory is disabled — ${sanitizeName(state.initError)}`);
+			return;
+		}
+		if (!state.config || !state.stores) {
+			warn(`Cannot run /${command}: persistent memory is not initialized.`);
+			return;
+		}
+		try {
+			const loaded = await Promise.all((Object.keys(state.stores) as Target[]).map(async (target) => [target, await state.stores![target].load(target)] as const));
+			const invalid = loaded.filter(([, result]) => result.status);
+			if (invalid.length) {
+				warn(`Cannot run /${command}: live memory state is unreadable or oversized. ${invalid.map(([, result]) => result.conflictWarning).join(" ")}`);
+				return;
+			}
+			if (!isIdle()) {
+				warn(`Cannot run /${command} while the agent is busy.`);
+				return;
+			}
+			const overLimit = loaded.filter(([target, result]) => result.entries.join(ENTRY_DELIMITER).length > (target === "user" ? state.config!.userCharLimit : state.config!.memoryCharLimit));
+			if (overLimit.length) {
+				warn(`Cannot run /${command}: live ${overLimit.map(([target]) => target).join(" and ")} entries exceed the configured character limit. Consolidate them before using /${command}.`);
+				return;
+			}
+			return Object.fromEntries(loaded.map(([target, result]) => [target, result.entries])) as Record<Target, string[]>;
+		} catch (error) {
+			warn(`Cannot run /${command}: ${error instanceof Error ? error.message : String(error)}`);
+		}
+	};
+
 	pi.registerCommand("remember", {
 		description: "Process an instruction into durable memory",
 		handler: async (args, ctx) => {
@@ -115,45 +168,52 @@ export default function memoryExtension(pi: ExtensionAPI): void {
 				ctx.ui.notify("Cannot run /remember while the agent is busy.", "warning");
 				return;
 			}
-			if (state.initError) {
-				ctx.ui.notify(`Cannot run /remember: persistent memory is disabled — ${sanitizeName(state.initError)}`, "warning");
+			const entries = await loadLiveEntries("remember", ctx.isIdle, (message) => ctx.ui.notify(message, "warning"));
+			if (!entries) return;
+			pi.sendUserMessage(`Process this /remember instruction; do not blindly copy it. Normalize the candidate into compact durable memory, choose the correct memory target, semantically compare it with the live entries, and merge or replace overlap instead of adding duplicates. Use the existing memory tool. Refuse project/repository-specific, temporary, trivial, or otherwise unsuitable content.\n\nCandidate:\n${JSON.stringify(candidate)}\n\nLive entries by target:\n${JSON.stringify(entries)}`);
+		},
+	});
+
+	pi.registerCommand("dream", {
+		description: "Promote invariant memory entries into SYSTEM.md",
+		handler: async (_args, ctx) => {
+			if (!ctx.isIdle()) {
+				ctx.ui.notify("Cannot run /dream while the agent is busy.", "warning");
 				return;
 			}
-			if (!state.config || !state.stores) {
-				ctx.ui.notify("Cannot run /remember: persistent memory is not initialized.", "warning");
+			const entries = await loadLiveEntries("dream", ctx.isIdle, (message) => ctx.ui.notify(message, "warning"));
+			if (!entries) return;
+			const systemPath = join(getAgentDir(), "SYSTEM.md");
+			const system = await loadSystemState(systemPath);
+			if (!ctx.isIdle()) {
+				ctx.ui.notify("Cannot run /dream while the agent is busy.", "warning");
 				return;
 			}
-			try {
-				const loaded = await Promise.all((Object.keys(state.stores) as Target[]).map(async (target) => [target, await state.stores![target].load(target)] as const));
-				const invalid = loaded.filter(([, result]) => result.status);
-				if (invalid.length) {
-					ctx.ui.notify(`Cannot run /remember: live memory state is unreadable or oversized. ${invalid.map(([, result]) => result.conflictWarning).join(" ")}`, "warning");
-					return;
-				}
-				if (!ctx.isIdle()) {
-					ctx.ui.notify("Cannot run /remember while the agent is busy.", "warning");
-					return;
-				}
-				const overLimit = loaded.filter(([target, result]) => {
-					const limit = target === "user" ? state.config!.userCharLimit : state.config!.memoryCharLimit;
-					return result.entries.join(ENTRY_DELIMITER).length > limit;
-				});
-				if (overLimit.length) {
-					ctx.ui.notify(`Cannot run /remember: live ${overLimit.map(([target]) => target).join(" and ")} entries exceed the configured character limit. Consolidate them before using /remember.`, "warning");
-					return;
-				}
-				const entries = Object.fromEntries(loaded.map(([target, result]) => [target, result.entries]));
-				pi.sendUserMessage(`Process this /remember instruction; do not blindly copy it. Normalize the candidate into compact durable memory, choose the correct memory target, semantically compare it with the live entries, and merge or replace overlap instead of adding duplicates. Use the existing memory tool. Refuse project/repository-specific, temporary, trivial, or otherwise unsuitable content.\n\nCandidate:\n${JSON.stringify(candidate)}\n\nLive entries by target:\n${JSON.stringify(entries)}`);
-			} catch (error) {
-				ctx.ui.notify(`Cannot run /remember: ${error instanceof Error ? error.message : String(error)}`, "warning");
+			if (system === "absent") {
+				ctx.ui.notify(`Cannot run /dream: agent-global SYSTEM.md is absent (${JSON.stringify(systemPath)}). Deliberately establish a complete global SYSTEM first; a partial SYSTEM replaces Pi's default prompt.`, "warning");
+				return;
+			}
+			if (system === "unreadable") {
+				ctx.ui.notify(`Cannot run /dream: agent-global SYSTEM.md is unreadable (${JSON.stringify(systemPath)}).`, "warning");
+				return;
 			}
+			const btwChild = process.argv.includes(BTW_CHILD_PAYLOAD_ARG);
+			const unchanged = !btwChild && !state.snapshotSanitized && state.initialEntries
+				&& entries.memory.join(ENTRY_DELIMITER) === state.initialEntries.memory.join(ENTRY_DELIMITER)
+				&& entries.user.join(ENTRY_DELIMITER) === state.initialEntries.user.join(ENTRY_DELIMITER);
+			const memoryMessage = unchanged
+				? "Use USER PROFILE/MEMORY already in your system context; do not reread those files."
+				: `Live entries by target:\n${JSON.stringify(entries)}`;
+			pi.sendUserMessage(`${DREAM_INSTRUCTION}\n\n${memoryMessage}\n\nRead ${JSON.stringify(systemPath)} before semantic deduplication or editing. Edit only ${JSON.stringify(systemPath)}; never edit a project SYSTEM.md.`);
 		},
 	});
 
 	pi.on("session_start", async () => {
 		state.config = undefined;
 		state.stores = undefined;
+		state.initialEntries = undefined;
 		state.snapshotBlocks = undefined;
+		state.snapshotSanitized = undefined;
 		state.conflictWarnings = [];
 		state.initError = undefined;
 		try {
@@ -191,9 +251,12 @@ export default function memoryExtension(pi: ExtensionAPI): void {
 				conflictWarnings.push(`WARNING: ${unexpected.length} unexpected file${unexpected.length === 1 ? "" : "s"} in the memory directory (${listed}${more}). Only MEMORY.md and USER.md are loaded; reconcile or remove the rest.`);
 			}
 
+			const rendered = [renderBlock("memory", memory.entries, config, conflictWarnings), renderBlock("user", user.entries, config, conflictWarnings)];
 			state.config = config;
 			state.stores = stores;
-			state.snapshotBlocks = [renderBlock("memory", memory.entries, config, conflictWarnings), renderBlock("user", user.entries, config, conflictWarnings)];
+			state.initialEntries = { memory: [...memory.entries], user: [...user.entries] };
+			state.snapshotBlocks = rendered.map(({ block }) => block);
+			state.snapshotSanitized = rendered.some(({ sanitized }) => sanitized);
 			state.conflictWarnings = conflictWarnings;
 		} catch (error) {
 			// Surface once, disable quietly: no throw-loop every turn.

packages/pi-memory/extensions/memory.ts

@@ -1,6 +1,6 @@
 {
 	"name": "@henryqw/pi-memory",
-	"version": "1.1.1",
+	"version": "1.2.0",
 	"description": "Auto-managed markdown memory for Pi: capped MEMORY.md/USER.md entry stores with frozen session snapshots.",
 	"keywords": [
 		"pi-package",