feat(pi-memory): recommend periodic dream sessions (#178)

Author: HenryQW

package-lock.json

@@ -24,7 +24,7 @@ pi install npm:@henryqw/pi-memory
 
 The extension maintains two markdown stores: `MEMORY.md` (global agent notes shared across all projects — do not store project-specific facts here, those belong in the repo) and `USER.md` (user profile). Each file holds `§`-delimited entries and is size-capped — 8800 characters by default for `MEMORY.md`, 5500 for `USER.md`. When a write would exceed the cap, the tool rejects it and reports current usage; consolidate by issuing one batch that removes or shortens stale entries and adds the new entry together (batch checks the final size only). If the on-disk file exceeds the cap (external edit or sync), the session snapshot omits the overflow and warns instead of injecting it.
 
-At session start, both stores are captured; later edits do not alter injected memory. `/dream` validates live state first and reuses unchanged memory snapshots, but always requires the model to read and edit only the agent-global `~/.pi/agent/SYSTEM.md`—never a project `.pi/SYSTEM.md`. That global file must already exist and be readable; establish it deliberately and completely, because a partial SYSTEM replaces Pi's default prompt. Use `/remember <instruction>` to ask the agent to normalize and deduplicate an instruction against the live contents of both stores before using the memory tool; unsuitable project-specific, temporary, trivial, or otherwise unsuitable content is refused. Each turn also includes a short memory check: save explicit durable preferences or corrections immediately, inferred habits after two independent signals from the conversation and/or existing profile, merge overlaps, and skip project- or repository-specific facts, task-local behavior, progress, and temporary preferences.
+At session start, both stores are captured; later edits do not alter injected memory. Pi recommends `/dream` when memory is non-empty and no previous dream is recorded, the last dream was over 30 days ago, or either store is at least 70% full and the last dream was at least 7 days ago. `/dream` records its completed run time in `~/.pi/agent/config/pi-memory/dream.json`, validates live state first, and reuses unchanged memory snapshots, but always requires the model to read and edit only the agent-global `~/.pi/agent/SYSTEM.md`—never a project `.pi/SYSTEM.md`. That global file must already exist and be readable; establish it deliberately and completely, because a partial SYSTEM replaces Pi's default prompt. Use `/remember <instruction>` to ask the agent to normalize and deduplicate an instruction against the live contents of both stores before using the memory tool; unsuitable project-specific, temporary, trivial, or otherwise unsuitable content is refused. Each turn also includes a short memory check: save explicit durable preferences or corrections immediately, inferred habits after two independent signals from the conversation and/or existing profile, merge overlaps, and skip project- or repository-specific facts, task-local behavior, progress, and temporary preferences.
 
 To inspect live state, read `<directory>/MEMORY.md`.
 

packages/pi-memory/README.md

@@ -1,19 +1,23 @@
-import { lstat, mkdir, readFile, readdir, realpath } from "node:fs/promises";
+import { lstat, mkdir, open, readFile, readdir, realpath, rename, unlink } from "node:fs/promises";
 import { join, sep } from "node:path";
 import { StringEnum } from "@earendil-works/pi-ai";
 import { getAgentDir, withFileMutationQueue, type ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { Text } from "@earendil-works/pi-tui";
 import { lock } from "proper-lockfile";
 import { Type } from "typebox";
 import { configPath, loadMemoryConfig, type MemoryConfig } from "../src/config.ts";
-import { ENTRY_DELIMITER, MemoryStore, usage, type Target } from "../src/store.ts";
+import { ENTRY_DELIMITER, isReservedFrameLine, MemoryStore, usage, type Target } from "../src/store.ts";
 
 const SEPARATOR = "═".repeat(46);
 // Backups and the lock file live OUTSIDE config.directory (which may be
 // iCloud-synced) so the memory dir holds exactly MEMORY.md and USER.md (ADR 005).
 const BACKUP_DIR = () => join(getAgentDir(), "config", "pi-memory", "backups");
+const DREAM_STATE_PATH = () => join(getAgentDir(), "config", "pi-memory", "dream.json");
+const DREAM_AFTER_MS = 30 * 24 * 60 * 60 * 1000;
+const DREAM_FULL_COOLDOWN_MS = 7 * 24 * 60 * 60 * 1000;
+const DREAM_USAGE_PERCENT = 70;
+const DREAM_STATE_MAX_BYTES = 4 * 1024;
 // Defense-in-depth against snapshot frame spoofing by poisoned on-disk entries.
-const FRAME_TOKEN_LINE = /^\s*(?:═{3,}|MEMORY \(your personal notes|USER PROFILE \(who the user is)/;
 const FRAME_TOKEN_REPLACEMENT = "[filtered frame token]";
 const DISPLAY_CONTROL_CHARACTER = /[\p{Cc}\p{Cf}]/gu;
 // @henryqw/pi-herdr-btw does not export internal/core.ts from its package root.
@@ -51,8 +55,58 @@ async function loadSystemState(path: string): Promise<SystemState> {
 	}
 }
 
+async function loadLastDreamAt(): Promise<number | undefined> {
+	let handle: Awaited<ReturnType<typeof open>> | undefined;
+	try {
+		handle = await open(DREAM_STATE_PATH(), "r");
+		const buffer = Buffer.alloc(DREAM_STATE_MAX_BYTES + 1);
+		let total = 0;
+		while (total < buffer.length) {
+			const { bytesRead } = await handle.read(buffer, total, buffer.length - total, null);
+			if (bytesRead === 0) break;
+			total += bytesRead;
+		}
+		if (total > DREAM_STATE_MAX_BYTES) throw new Error(`Dream state file is too large: ${DREAM_STATE_PATH()}`);
+		const parsed: unknown = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(buffer.subarray(0, total)));
+		const lastDreamAt = parsed && typeof parsed === "object" && !Array.isArray(parsed)
+			? (parsed as Record<string, unknown>).lastDreamAt
+			: undefined;
+		const value = typeof lastDreamAt === "string" ? Date.parse(lastDreamAt) : Number.NaN;
+		if (!Number.isFinite(value) || value > Date.now()) throw new Error(`Invalid lastDreamAt in ${DREAM_STATE_PATH()}`);
+		return value;
+	} catch (error) {
+		if (error instanceof Error && "code" in error && error.code === "ENOENT") return;
+		throw error;
+	} finally {
+		await handle?.close();
+	}
+}
+
+async function saveLastDreamAt(): Promise<void> {
+	const path = DREAM_STATE_PATH();
+	const tempPath = `${path}.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2)}`;
+	let created = false;
+	try {
+		const handle = await open(tempPath, "wx", 0o600);
+		created = true;
+		try {
+			await handle.writeFile(`${JSON.stringify({ lastDreamAt: new Date().toISOString() }, null, 2)}\n`);
+		} finally {
+			await handle.close();
+		}
+		// rename replaces a destination symlink rather than following it.
+		await rename(tempPath, path);
+	} finally {
+		if (created) {
+			await unlink(tempPath).catch((error: NodeJS.ErrnoException) => {
+				if (error.code !== "ENOENT") throw error;
+			});
+		}
+	}
+}
+
 function sanitizeEntry(entry: string): string {
-	return entry.split("\n").map((line) => FRAME_TOKEN_LINE.test(line) ? FRAME_TOKEN_REPLACEMENT : line).join("\n");
+	return entry.split("\n").map((line) => isReservedFrameLine(line) ? FRAME_TOKEN_REPLACEMENT : line).join("\n");
 }
 
 // Strip control characters so externally-influenced names can't smuggle
@@ -123,6 +177,8 @@ export default function memoryExtension(pi: ExtensionAPI): void {
 		snapshotSanitized?: boolean;
 		conflictWarnings: string[];
 		initError?: string;
+		dreamPending?: boolean;
+		dreamSucceeded?: boolean;
 	} = { conflictWarnings: [] };
 
 	const loadLiveEntries = async (command: string, isIdle: () => boolean, warn: (message: string) => void): Promise<Record<Target, string[]> | undefined> => {
@@ -204,18 +260,53 @@ export default function memoryExtension(pi: ExtensionAPI): void {
 			const memoryMessage = unchanged
 				? "Use USER PROFILE/MEMORY already in your system context; do not reread those files."
 				: `Live entries by target:\n${JSON.stringify(entries)}`;
-			pi.sendUserMessage(`${DREAM_INSTRUCTION}\n\n${memoryMessage}\n\nRead ${JSON.stringify(systemPath)} before semantic deduplication or editing. Edit only ${JSON.stringify(systemPath)}; never edit a project SYSTEM.md.`);
+			state.dreamPending = true;
+			state.dreamSucceeded = false;
+			try {
+				pi.sendUserMessage(`${DREAM_INSTRUCTION}\n\n${memoryMessage}\n\nRead ${JSON.stringify(systemPath)} before semantic deduplication or editing. Edit only ${JSON.stringify(systemPath)}; never edit a project SYSTEM.md.`);
+			} catch (error) {
+				state.dreamPending = false;
+				throw error;
+			}
 		},
 	});
 
-	pi.on("session_start", async () => {
+	pi.on("agent_end", (event) => {
+		if (!state.dreamPending) return;
+		for (let index = event.messages.length - 1; index >= 0; index--) {
+			const message = event.messages[index];
+			if (message?.role !== "assistant") continue;
+			state.dreamSucceeded = message.stopReason === "stop";
+			break;
+		}
+	});
+
+	pi.on("agent_settled", async (_event, ctx) => {
+		if (!state.dreamPending) return;
+		const succeeded = state.dreamSucceeded;
+		state.dreamPending = false;
+		state.dreamSucceeded = false;
+		if (!succeeded) {
+			ctx.ui.notify("Dream did not complete; its timestamp was not updated.", "warning");
+			return;
+		}
+		try {
+			await saveLastDreamAt();
+		} catch (error) {
+			ctx.ui.notify(`Dream completed, but its timestamp could not be recorded: ${error instanceof Error ? error.message : String(error)}`, "warning");
+		}
+	});
+
+	pi.on("session_start", async (_event, ctx) => {
 		state.config = undefined;
 		state.stores = undefined;
 		state.initialEntries = undefined;
 		state.snapshotBlocks = undefined;
 		state.snapshotSanitized = undefined;
 		state.conflictWarnings = [];
 		state.initError = undefined;
+		state.dreamPending = false;
+		state.dreamSucceeded = false;
 		try {
 			await mkdir(BACKUP_DIR(), { recursive: true });
 			const config = loadMemoryConfig();
@@ -258,6 +349,24 @@ export default function memoryExtension(pi: ExtensionAPI): void {
 			state.snapshotBlocks = rendered.map(({ block }) => block);
 			state.snapshotSanitized = rendered.some(({ sanitized }) => sanitized);
 			state.conflictWarnings = conflictWarnings;
+
+			const memoryChars = memory.entries.join(ENTRY_DELIMITER).length;
+			const userChars = user.entries.join(ENTRY_DELIMITER).length;
+			const validWithinCap = !memory.status && !user.status
+				&& memoryChars <= config.memoryCharLimit && userChars <= config.userCharLimit;
+			if (!process.argv.includes(BTW_CHILD_PAYLOAD_ARG) && validWithinCap && (memory.entries.length || user.entries.length)) {
+				try {
+					const lastDreamAt = await loadLastDreamAt();
+					const age = lastDreamAt === undefined ? undefined : Date.now() - lastDreamAt;
+					const full = memoryChars * 100 >= config.memoryCharLimit * DREAM_USAGE_PERCENT
+						|| userChars * 100 >= config.userCharLimit * DREAM_USAGE_PERCENT;
+					if (age === undefined || age >= DREAM_AFTER_MS || (full && age >= DREAM_FULL_COOLDOWN_MS)) {
+						ctx.ui.notify("Memory dream recommended; run /dream.", "info");
+					}
+				} catch (error) {
+					ctx.ui.notify(`Cannot check dream reminder: ${error instanceof Error ? error.message : String(error)}`, "warning");
+				}
+			}
 		} catch (error) {
 			// Surface once, disable quietly: no throw-loop every turn.
 			state.initError = error instanceof Error ? error.message : String(error);

packages/pi-memory/extensions/memory.ts

@@ -1,6 +1,6 @@
 {
 	"name": "@henryqw/pi-memory",
-	"version": "1.2.0",
+	"version": "1.3.0",
 	"description": "Auto-managed markdown memory for Pi: capped MEMORY.md/USER.md entry stores with frozen session snapshots.",
 	"keywords": [
 		"pi-package",

packages/pi-memory/package.json

@@ -1,7 +1,13 @@
+import { createHash } from "node:crypto";
 import { copyFile, lstat, mkdir, open, rename, stat, writeFile, rm } from "node:fs/promises";
 import { dirname, join } from "node:path";
 
 export const ENTRY_DELIMITER: string = "\n§\n";
+const RESERVED_FRAME_LINE = /^\s*(?:═{3,}|MEMORY \(your personal notes|USER PROFILE \(who the user is)/;
+
+export function isReservedFrameLine(line: string): boolean {
+	return RESERVED_FRAME_LINE.test(line);
+}
 
 export type Target = "memory" | "user";
 
@@ -103,8 +109,8 @@ export class MemoryStore {
 	private readonly observedExisting = new Set<Target>();
 	private disappearanceDetected = false;
 	private unreadableReason: string | undefined;
-	// mtime/size of the last successfully loaded file, per target.
-	private readonly loadedFingerprints = new Map<Target, { mtimeMs: number; size: number }>();
+	// Metadata and content digest of the last successfully loaded file, per target.
+	private readonly loadedFingerprints = new Map<Target, { mtimeMs: number; size: number; digest: string }>();
 
 	constructor(config: StoreConfig) {
 		this.config = config;
@@ -183,6 +189,24 @@ export class MemoryStore {
 		return { entries: file.kind === "ok" ? parseEntries(file.raw) : [] };
 	}
 
+	private async digestFile(path: string): Promise<string> {
+		const handle = await open(path, "r");
+		try {
+			const hash = createHash("sha256");
+			const buffer = Buffer.alloc(64 * 1024);
+			let total = 0;
+			for (;;) {
+				const { bytesRead } = await handle.read(buffer, 0, buffer.length, null);
+				if (bytesRead === 0) return hash.digest("base64url");
+				total += bytesRead;
+				if (total > MAX_FILE_BYTES) throw new Error(`${path} grew over the ${MAX_FILE_BYTES.toLocaleString()}-byte limit during mutation.`);
+				hash.update(buffer.subarray(0, bytesRead));
+			}
+		} finally {
+			await handle.close();
+		}
+	}
+
 	/**
 	 * Returns file state: "absent" for a missing file, "unreadable" when the file
 	 * EXISTS but could not be read (permissions or invalid UTF-8), "oversized"
@@ -221,7 +245,11 @@ export class MemoryStore {
 			// by V1-plus-mutation.
 			try {
 				const st = await (this.config.statFn ?? stat)(this.pathFor(target));
-				this.loadedFingerprints.set(target, { mtimeMs: st.mtimeMs, size: st.size });
+				this.loadedFingerprints.set(target, {
+					mtimeMs: st.mtimeMs,
+					size: st.size,
+					digest: createHash("sha256").update(buffer.subarray(0, total)).digest("base64url"),
+				});
 			} catch {
 				this.loadedFingerprints.delete(target);
 			}
@@ -317,7 +345,8 @@ export class MemoryStore {
 				const fingerprint = this.loadedFingerprints.get(target);
 				if (fingerprint) {
 					const current = await (this.config.statFn ?? stat)(path);
-					if (current.mtimeMs !== fingerprint.mtimeMs || current.size !== fingerprint.size) {
+					if (current.mtimeMs !== fingerprint.mtimeMs || current.size !== fingerprint.size
+						|| await this.digestFile(path) !== fingerprint.digest) {
 						throw new Error(`${path} changed during this mutation (likely sync); retry to merge its content.`);
 					}
 				}
@@ -339,7 +368,7 @@ export class MemoryStore {
 		// included): anything the sanitizer would filter must be rejected here,
 		// or writes report success while vanishing from snapshots.
 		for (const line of normalized.split("\n")) {
-			if (/^\s*(?:═{3,}|MEMORY \(your personal notes|USER PROFILE \(who the user is)/.test(line)) {
+			if (isReservedFrameLine(line)) {
 				return "Content must not contain lines starting with '═' separators or the reserved headers 'MEMORY (your personal notes' / 'USER PROFILE (who the user is'.";
 			}
 		}