docs: document reproducibility, base-model benchmarking, and CLI training flags in changelog

Author: will-lamerton

CHANGELOG.md

@@ -1,5 +1,5 @@
 {
-	"$schema": "https://biomejs.dev/schemas/2.5.1/schema.json",
+	"$schema": "https://biomejs.dev/schemas/2.5.7/schema.json",
 	"vcs": {
 		"enabled": true,
 		"clientKind": "git",

biome.json

@@ -59,6 +59,8 @@ With sampling enabled, `--samples <n>` runs each test n times and records the pe
 nanotune benchmark --temperature 0.8 --samples 5
 ```
 
+`--temperature`, `--seed`, and `--samples` are rejected outright if they can't be parsed, rather than falling back to the default. A typo like `--samples 5x` fails with a message naming the flag, so a run never reports a score under settings you didn't ask for.
+
 ## Examples
 
 ```bash

docs/commands/benchmark.md

@@ -84,13 +84,15 @@ Export training data to a file. Uses the same formats as `data import`, selected
 | Flag | Description |
 |------|-------------|
 | `-y, --yes` | Skip the overwrite confirmation prompt |
+| `-e, --eval` | Export the validation set (`valid.jsonl`) instead of training data |
 
 JSONL and JSON exports preserve every example exactly, including multi-turn conversations and per-example context messages — feeding the output back into `data import` reproduces the original data. **CSV has no way to represent multi-turn examples**, so any example with more than one turn is skipped (not truncated) during a CSV export, and reported in the summary.
 
 If the target file already exists, you're asked to confirm before it's overwritten. Pass `--yes` to skip the prompt — this is also what lets `data export` run in a script or CI job.
 
 ```bash
 nanotune data export backup.jsonl --yes
+nanotune data export valid-backup.jsonl --eval --yes
 ```
 
 ## nanotune data list

docs/commands/data.md

@@ -1,6 +1,6 @@
 {
 	"name": "@nanocollective/nanotune",
-	"version": "1.6.0",
+	"version": "1.7.0",
 	"type": "module",
 	"description": "A simple, interactive CLI for fine-tuning small language models on Apple Silicon. No YAML configs, no complex flags - just an interactive CLI that guides you through the process. ⚒️",
 	"bin": {

package.json

@@ -82,7 +82,8 @@ dataCommand
 		'-y, --yes',
 		'Skip the overwrite confirmation prompt (for scripts and CI)',
 	)
-	.action(async (file: string, options: {yes?: boolean}) => {
+	.option('-e, --eval', 'Export the validation set instead of training data')
+	.action(async (file: string, options: {yes?: boolean; eval?: boolean}) => {
 		const {DataExportCommand} = await import('./commands/data/export.js');
 		// Without a TTY there is no way to answer the overwrite prompt, so require --yes.
 		if (!options.yes && !supportsRawMode()) {
@@ -91,7 +92,9 @@ dataCommand
 			process.exitCode = 1;
 			return;
 		}
-		render(<DataExportCommand file={file} yes={options.yes} />);
+		render(
+			<DataExportCommand file={file} yes={options.yes} isEval={options.eval} />,
+		);
 	});
 
 dataCommand

src/cli.tsx

@@ -309,6 +309,32 @@ export function BenchmarkCommand({options}: Props) {
 				return;
 			}
 
+			// Resolve the sampling flags before anything expensive. `--base` can
+			// spend minutes downloading and quantizing a base model, and failing
+			// a typo'd `--samples` only after that has finished wastes the whole
+			// run on something we could see immediately.
+			const sampling = resolveSamplingOptions({
+				temperature: options.temperature,
+				seed: options.seed,
+				samples: options.samples,
+			});
+
+			// Reject a mistyped flag rather than running a suite under settings
+			// the user didn't ask for.
+			if (sampling.errors.length > 0) {
+				setError(sampling.errors[0]);
+				setStatus('error');
+				return;
+			}
+
+			if (sampling.samples > 1 && sampling.temperature === 0) {
+				setError(
+					'Cannot use --samples with temperature 0 (greedy decoding produces identical outputs). Use --temperature 0.1 or higher for sampling.',
+				);
+				setStatus('error');
+				return;
+			}
+
 			// Find model
 			let modelPath: string | null;
 			if (options.base) {
@@ -452,22 +478,6 @@ export function BenchmarkCommand({options}: Props) {
 				}
 			}
 
-			// Build inference options from CLI flags or preset
-			const sampling = resolveSamplingOptions({
-				temperature: options.temperature,
-				seed: options.seed,
-				samples: options.samples,
-			});
-
-			// Validate sampling configuration
-			if (sampling.samples > 1 && sampling.temperature === 0) {
-				setError(
-					'Cannot use --samples with temperature 0 (greedy decoding produces identical outputs). Use --temperature 0.1 or higher for sampling.',
-				);
-				setStatus('error');
-				return;
-			}
-
 			let serverOptions: ServerOptions;
 			let generateOptions: GenerateOptions;
 

src/commands/benchmark.tsx

@@ -1,11 +1,12 @@
-import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import test from "ava";
 import { Text } from "ink";
 import { render } from "ink-testing-library";
 import { useKeyInput } from "../components/index.js";
 import { loadTrainingData } from "../lib/data.js";
 import { streamPreview } from "./chat.js";
+import { DataExportCommand } from "./data/export.js";
 import { DataImportCommand } from "./data/import.js";
 import { DataListCommand } from "./data/list.js";
 import { DataValidateCommand } from "./data/validate.js";
@@ -60,6 +61,19 @@ function writeExamples(lines: object[]) {
   );
 }
 
+function writeEvalExamples(lines: object[]) {
+  writeFileSync(
+    join(DATA_DIR, "valid.jsonl"),
+    `${lines.map((l) => JSON.stringify(l)).join("\n")}\n`,
+  );
+}
+
+const settle = () => new Promise((resolve) => setTimeout(resolve, 60));
+
+function userContent(example: { messages: { role: string; content: string }[] }) {
+  return example.messages.find((m) => m.role === "user")?.content;
+}
+
 function example(userInput: string) {
   return {
     messages: [
@@ -293,3 +307,82 @@ test("streamPreview clips a single very long line by characters", (t) => {
   t.true(truncated);
   t.is(text.length, 2000);
 });
+
+// ── data list edits the set it was opened on ──────────────────────────
+
+test.serial(
+  "DataListCommand with --eval edits valid.jsonl and leaves train.jsonl alone",
+  async (t) => {
+    // Regression: the edit path called updateTrainingExample/loadTrainingData
+    // without isEval, so editing a validation example overwrote the training
+    // example at the same index instead.
+    const originalTTY = process.stdin.isTTY;
+    try {
+      setupProject();
+      writeExamples([example("train-one")]);
+      writeEvalExamples([example("valid-one")]);
+      process.stdin.isTTY = true;
+
+      const instance = render(<DataListCommand isEval />);
+      await settle();
+      instance.stdin.write("e"); // enter edit mode
+      await settle();
+      instance.stdin.write("\r"); // submit user input unchanged
+      await settle();
+      instance.stdin.write("\r"); // submit assistant output unchanged
+      await settle();
+      instance.unmount();
+
+      t.is(userContent(loadTrainingData(false)[0]), "train-one");
+      t.is(userContent(loadTrainingData(true)[0]), "valid-one");
+      t.is(loadTrainingData(false).length, 1);
+      t.is(loadTrainingData(true).length, 1);
+    } finally {
+      process.stdin.isTTY = originalTTY;
+      teardown();
+    }
+  },
+);
+
+// ── data export honours --eval ────────────────────────────────────────
+
+test.serial("DataExportCommand exports training data by default", async (t) => {
+  try {
+    setupProject();
+    writeExamples([example("train-one"), example("train-two")]);
+    writeEvalExamples([example("valid-one")]);
+
+    await renderCommand(
+      <DataExportCommand file="out.jsonl" yes />,
+      "Export complete!",
+    );
+
+    const written = readFileSync(join(TEST_DIR, "out.jsonl"), "utf-8").trim();
+    t.is(written.split("\n").length, 2);
+    t.true(written.includes("train-one"));
+    t.false(written.includes("valid-one"));
+  } finally {
+    teardown();
+  }
+});
+
+test.serial("DataExportCommand with --eval exports the validation set", async (t) => {
+  try {
+    setupProject();
+    writeExamples([example("train-one"), example("train-two")]);
+    writeEvalExamples([example("valid-one")]);
+
+    const output = await renderCommand(
+      <DataExportCommand file="out.jsonl" yes isEval />,
+      "Export complete!",
+    );
+
+    t.true(output.includes("Export Validation Data"));
+    const written = readFileSync(join(TEST_DIR, "out.jsonl"), "utf-8").trim();
+    t.is(written.split("\n").length, 1);
+    t.true(written.includes("valid-one"));
+    t.false(written.includes("train-one"));
+  } finally {
+    teardown();
+  }
+});

src/commands/commands.spec.tsx

@@ -16,9 +16,10 @@ interface Props {
 	file: string;
 	/** Skip the overwrite confirmation — needed to run under CI or in a pipeline. */
 	yes?: boolean;
+	isEval?: boolean;
 }
 
-export function DataExportCommand({file, yes}: Props) {
+export function DataExportCommand({file, yes, isEval = false}: Props) {
 	const {exit} = useApp();
 	const filePath = resolve(process.cwd(), file);
 	const fileExists = existsSync(filePath);
@@ -27,7 +28,8 @@ export function DataExportCommand({file, yes}: Props) {
 	>(!fileExists || yes ? 'exporting' : 'confirm');
 	const [result, setResult] = useState<ExportResult | null>(null);
 	const [error, setError] = useState<string | null>(null);
-	const count = configExists() ? countExamples() : 0;
+	const count = configExists() ? countExamples(isEval) : 0;
+	const title = isEval ? 'Export Validation Data' : 'Export Training Data';
 
 	const doExport = useCallback(() => {
 		try {
@@ -37,7 +39,7 @@ export function DataExportCommand({file, yes}: Props) {
 				return;
 			}
 
-			const exportResult = exportData(filePath);
+			const exportResult = exportData(filePath, isEval);
 			if (
 				exportResult.exported === 0 &&
 				exportResult.errors[0]?.startsWith('Unsupported')
@@ -53,7 +55,7 @@ export function DataExportCommand({file, yes}: Props) {
 			setError(err instanceof Error ? err.message : 'Export failed');
 			setStatus('error');
 		}
-	}, [filePath]);
+	}, [filePath, isEval]);
 
 	useEffect(() => {
 		if (status === 'exporting') {
@@ -78,7 +80,7 @@ export function DataExportCommand({file, yes}: Props) {
 	if (!configExists()) {
 		return (
 			<Box flexDirection="column" padding={1}>
-				<Header title="Export Training Data" />
+				<Header title={title} />
 				<StatusMessage variant="error">
 					Not a Nanotune project. Run `nanotune init` first.
 				</StatusMessage>
@@ -88,7 +90,7 @@ export function DataExportCommand({file, yes}: Props) {
 
 	return (
 		<Box flexDirection="column" padding={1}>
-			<Header title="Export Training Data" />
+			<Header title={title} />
 
 			{status === 'confirm' && (
 				<Box flexDirection="column">

src/commands/data/export.tsx

@@ -56,8 +56,8 @@ export function DataListCommand({isEval = false}: Props) {
 					assistantOutput,
 				),
 			};
-			updateTrainingExample(editIndex, updated);
-			setData(loadTrainingData());
+			updateTrainingExample(editIndex, updated, isEval);
+			setData(loadTrainingData(isEval));
 			setExpandedIndex(null);
 			setEditIndex(null);
 			setEditError(null);
@@ -69,10 +69,13 @@ export function DataListCommand({isEval = false}: Props) {
 	};
 
 	const handleEditInputSubmit = (value: string) => {
-		if (value.trim()) {
-			setEditInput(value.trim());
-			setEditField('output');
+		if (!value.trim()) {
+			setEditError('Input cannot be empty');
+			return;
 		}
+		setEditError(null);
+		setEditInput(value.trim());
+		setEditField('output');
 	};
 
 	const handleEditOutputSubmit = (value: string) => {