diff --git a/.gitignore b/.gitignore index f4f81ae..3ce1225 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store .coverage +.probitas/ result dist/ src/cli/_embedded_templates.ts diff --git a/README.md b/README.md index ee536b0..23da9b5 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,9 @@ probitas run # Run scenarios with specific tag probitas run -s tag:api +# Re-run only failed scenarios from previous run +probitas run --failed + # Run with JSON output probitas run --reporter json diff --git a/assets/usage-run.txt b/assets/usage-run.txt index 04feae2..5119ca9 100644 --- a/assets/usage-run.txt +++ b/assets/usage-run.txt @@ -21,6 +21,8 @@ Options: -S, --sequential Run scenarios sequentially (alias for --max-concurrency=1) --max-failures Stop after N failures -f, --fail-fast Stop on first failure (alias for --max-failures=1) + -F, --failed Run only scenarios that failed in the previous run + Reads from .probitas/last-run.json --config Path to config file (deno.json/deno.jsonc) --env Load environment variables from file (default: .env) --no-env Skip loading .env file @@ -93,6 +95,11 @@ Examples: probitas run -f Stop on first failure (alias) probitas run --reload Reload dependencies before running + # Re-run failed scenarios + probitas run --failed Run only scenarios that failed previously + probitas run -F Run only scenarios that failed previously (short) + probitas run -F -s tag:api Run failed scenarios AND with @api tag + # Output control probitas run -v Verbose output probitas run --verbose Verbose output (same as -v) diff --git a/deno.json b/deno.json index eabbc36..104e2ea 100644 --- a/deno.json +++ b/deno.json @@ -106,6 +106,9 @@ "@std/testing/bdd": "jsr:@std/testing@^1.0.16/bdd", "@std/testing/mock": "jsr:@std/testing@^1.0.16/mock", "@std/testing/time": "jsr:@std/testing@^1.0.16/time", + "@std/text": "jsr:@std/text@^1.0.17", + "@std/text/closest-string": "jsr:@std/text@^1.0.17/closest-string", + "@std/text/levenshtein-distance": "jsr:@std/text@^1.0.17/levenshtein-distance", "jsr:@probitas/probitas@^0": "./src/mod.ts" } } diff --git a/deno.lock b/deno.lock index a7b51e3..b52c826 100644 --- a/deno.lock +++ b/deno.lock @@ -83,10 +83,12 @@ "jsr:@std/path@0.217": "0.217.0", "jsr:@std/path@1": "1.1.4", "jsr:@std/path@^1.1.4": "1.1.4", + "jsr:@std/regexp@^1.0.1": "1.0.1", "jsr:@std/streams@^1.0.14": "1.0.16", "jsr:@std/streams@^1.0.16": "1.0.16", "jsr:@std/streams@^1.0.9": "1.0.16", "jsr:@std/testing@^1.0.16": "1.0.16", + "jsr:@std/text@^1.0.17": "1.0.17", "npm:@aws-sdk/client-sqs@^3.700.0": "3.965.0", "npm:@bufbuild/protobuf@^2.7.0": "2.10.2", "npm:@connectrpc/connect-node@^2.1.1": "2.1.1_@bufbuild+protobuf@2.10.2_@connectrpc+connect@2.1.1__@bufbuild+protobuf@2.10.2", @@ -447,6 +449,9 @@ "jsr:@std/internal@^1.0.12" ] }, + "@std/regexp@1.0.1": { + "integrity": "5179d823465085c5480dafb44438466e83c424fadc61ba31f744050ecc0f596d" + }, "@std/streams@1.0.16": { "integrity": "85030627befb1767c60d4f65cb30fa2f94af1d6ee6e5b2515b76157a542e89c4", "dependencies": [ @@ -461,6 +466,12 @@ "jsr:@std/data-structures", "jsr:@std/internal@^1.0.12" ] + }, + "@std/text@1.0.17": { + "integrity": "4b2c4ef67ae5b6c1dfd447c81c83a43718f52e3c7e748d8b33f694aba9895f95", + "dependencies": [ + "jsr:@std/regexp" + ] } }, "npm": { @@ -1579,6 +1590,7 @@ "jsr:@std/path@^1.1.4", "jsr:@std/streams@^1.0.16", "jsr:@std/testing@^1.0.16", + "jsr:@std/text@^1.0.17", "npm:@faker-js/faker@^10.2.0" ] } diff --git a/src/cli/_templates/run.ts b/src/cli/_templates/run.ts index 8ea298d..3ef1855 100644 --- a/src/cli/_templates/run.ts +++ b/src/cli/_templates/run.ts @@ -66,6 +66,7 @@ runSubprocess(async (ipc, input) => { timeout, stepOptions, logLevel, + failedFilter, } = input; // Create abort controller for this run @@ -75,8 +76,8 @@ runSubprocess(async (ipc, input) => { await configureLogging(logLevel); try { - // Load scenarios from files - const scenarios = applySelectors( + // Load scenarios from files and apply selectors + let scenarios = applySelectors( await loadScenarios(filePaths, { onImportError: (file, err) => { const m = err instanceof Error ? err.message : String(err); @@ -86,6 +87,26 @@ runSubprocess(async (ipc, input) => { selectors, ); + // Apply failed filter if provided (for --failed flag) + if (failedFilter && failedFilter.length > 0) { + const failedSet = new Set( + failedFilter.map((f) => `${f.name}|${f.file}`), + ); + scenarios = scenarios.filter((s) => { + const scenarioFile = s.origin?.path ?? "unknown"; + // Check both absolute path and relative path matching + return failedSet.has(`${s.name}|${scenarioFile}`) || + failedFilter.some((f) => + f.name === s.name && scenarioFile.endsWith(f.file) + ); + }); + + logger.debug("Applied failed filter", { + filterCount: failedFilter.length, + matchedCount: scenarios.length, + }); + } + // Check if aborted during loading phase if (globalAbortController.signal.aborted) { throw new Error("Aborted during scenario loading"); @@ -96,6 +117,7 @@ runSubprocess(async (ipc, input) => { logger.info("No scenarios found after applying selectors", { filePaths, selectors, + failedFilter: failedFilter?.length ?? 0, }); } diff --git a/src/cli/_templates/run_protocol.ts b/src/cli/_templates/run_protocol.ts index 1110d76..757bf30 100644 --- a/src/cli/_templates/run_protocol.ts +++ b/src/cli/_templates/run_protocol.ts @@ -34,6 +34,16 @@ export type RunInput = | RunCommandInput | RunAbortInput; +/** + * Failed scenario identifier for filtering + */ +export interface FailedScenarioFilter { + /** Scenario name */ + readonly name: string; + /** File path (relative or absolute) */ + readonly file: string; +} + /** * Run scenarios command */ @@ -53,6 +63,8 @@ export interface RunCommandInput { readonly stepOptions?: StepOptions; /** Log level for subprocess logging */ readonly logLevel: LogLevel; + /** Filter to only run these failed scenarios (name + file pairs) */ + readonly failedFilter?: readonly FailedScenarioFilter[]; } /** diff --git a/src/cli/commands/list.ts b/src/cli/commands/list.ts index 76d14a9..7a258a0 100644 --- a/src/cli/commands/list.ts +++ b/src/cli/commands/list.ts @@ -12,6 +12,11 @@ import { fromErrorObject, isErrorObject } from "@core/errorutil/error-object"; import { unreachable } from "@core/errorutil/unreachable"; import { EXIT_CODE } from "../constants.ts"; import { findProbitasConfigFile, loadConfig } from "../config.ts"; +import { + createUnknownArgHandler, + extractKnownOptions, + formatUnknownArgError, +} from "../unknown_args.ts"; import { createDiscoveryProgress, writeStatus } from "../progress.ts"; import { configureLogging, @@ -29,6 +34,36 @@ import { const logger = getLogger(["probitas", "cli", "list"]); +/** + * parseArgs configuration for the list command + */ +const PARSE_ARGS_CONFIG = { + string: ["config", "include", "exclude", "selector", "env"], + boolean: [ + "help", + "json", + "no-env", + "reload", + "quiet", + "verbose", + "debug", + ], + collect: ["include", "exclude", "selector"], + alias: { + h: "help", + s: "selector", + r: "reload", + v: "verbose", + q: "quiet", + d: "debug", + }, + default: { + include: undefined, + exclude: undefined, + selector: undefined, + }, +} as const; + /** * Execute the list command * @@ -46,34 +81,33 @@ export async function listCommand( // Extract deno options first (before parseArgs) const { denoArgs, remainingArgs } = extractDenoOptions(args); + // Setup unknown argument handler + const knownOptions = extractKnownOptions(PARSE_ARGS_CONFIG); + const { handler: unknownHandler, result: unknownResult } = + createUnknownArgHandler({ + knownOptions, + commandName: "list", + }); + // Parse command-line arguments const parsed = parseArgs(remainingArgs, { - string: ["config", "include", "exclude", "selector", "env"], - boolean: [ - "help", - "json", - "no-env", - "reload", - "quiet", - "verbose", - "debug", - ], - collect: ["include", "exclude", "selector"], - alias: { - h: "help", - s: "selector", - r: "reload", - v: "verbose", - q: "quiet", - d: "debug", - }, - default: { - include: undefined, - exclude: undefined, - selector: undefined, - }, + ...PARSE_ARGS_CONFIG, + unknown: unknownHandler, }); + // Check for unknown arguments before showing help + if (unknownResult.hasErrors) { + for (const unknown of unknownResult.unknownArgs) { + console.error( + formatUnknownArgError(unknown, { + knownOptions, + commandName: "list", + }), + ); + } + return EXIT_CODE.USAGE_ERROR; + } + // Show help if requested if (parsed.help) { try { diff --git a/src/cli/commands/run.ts b/src/cli/commands/run.ts index 34797f2..7679aec 100644 --- a/src/cli/commands/run.ts +++ b/src/cli/commands/run.ts @@ -10,6 +10,11 @@ import { unreachable } from "@core/errorutil/unreachable"; import { getLogger, type LogLevel } from "@logtape/logtape"; import { DEFAULT_TIMEOUT, EXIT_CODE } from "../constants.ts"; import { findProbitasConfigFile, loadConfig } from "../config.ts"; +import { + createUnknownArgHandler, + extractKnownOptions, + formatUnknownArgError, +} from "../unknown_args.ts"; import { discoverScenarioFiles } from "@probitas/discover"; import type { StepOptions } from "@probitas/core"; import type { Reporter, RunResult } from "@probitas/runner"; @@ -29,13 +34,62 @@ import { deserializeRunResult, deserializeScenarioResult, deserializeStepResult, + type FailedScenarioFilter, isRunOutput, type RunCommandInput, type RunOutput, } from "../_templates/run_protocol.ts"; +import { loadLastRunState, saveLastRunState } from "../state.ts"; const logger = getLogger(["probitas", "cli", "run"]); +/** + * parseArgs configuration for the run command + */ +const PARSE_ARGS_CONFIG = { + string: [ + "reporter", + "config", + "max-concurrency", + "max-failures", + "include", + "exclude", + "selector", + "timeout", + "env", + ], + boolean: [ + "help", + "no-color", + "no-timeout", + "no-env", + "reload", + "quiet", + "verbose", + "debug", + "sequential", + "fail-fast", + "failed", + ], + collect: ["include", "exclude", "selector"], + alias: { + h: "help", + s: "selector", + S: "sequential", + f: "fail-fast", + F: "failed", + v: "verbose", + q: "quiet", + d: "debug", + r: "reload", + }, + default: { + include: undefined, + exclude: undefined, + selector: undefined, + }, +} as const; + /** * Execute the run command * @@ -55,49 +109,33 @@ export async function runCommand( // Extract deno options first (before parseArgs) const { denoArgs, remainingArgs } = extractDenoOptions(args); + // Setup unknown argument handler + const knownOptions = extractKnownOptions(PARSE_ARGS_CONFIG); + const { handler: unknownHandler, result: unknownResult } = + createUnknownArgHandler({ + knownOptions, + commandName: "run", + }); + // Parse command-line arguments const parsed = parseArgs(remainingArgs, { - string: [ - "reporter", - "config", - "max-concurrency", - "max-failures", - "include", - "exclude", - "selector", - "timeout", - "env", - ], - boolean: [ - "help", - "no-color", - "no-timeout", - "no-env", - "reload", - "quiet", - "verbose", - "debug", - "sequential", - "fail-fast", - ], - collect: ["include", "exclude", "selector"], - alias: { - h: "help", - s: "selector", - S: "sequential", - f: "fail-fast", - v: "verbose", - q: "quiet", - d: "debug", - r: "reload", - }, - default: { - include: undefined, - exclude: undefined, - selector: undefined, - }, + ...PARSE_ARGS_CONFIG, + unknown: unknownHandler, }); + // Check for unknown arguments before showing help + if (unknownResult.hasErrors) { + for (const unknown of unknownResult.unknownArgs) { + console.error( + formatUnknownArgError(unknown, { + knownOptions, + commandName: "run", + }), + ); + } + return EXIT_CODE.USAGE_ERROR; + } + // Show help if requested if (parsed.help) { try { @@ -186,6 +224,29 @@ export async function runCommand( // Get selectors (will be applied in subprocess) const selectors = parsed.selector ?? config?.selectors ?? []; + // Handle --failed flag: load previous run state and build filter + let failedFilter: FailedScenarioFilter[] | undefined; + if (parsed.failed) { + const lastRunState = await loadLastRunState(cwd); + if (!lastRunState) { + console.warn( + "No previous run state found. Running all matching scenarios.", + ); + } else if (lastRunState.failed.length === 0) { + console.log("No failed scenarios from previous run."); + return EXIT_CODE.SUCCESS; + } else { + failedFilter = lastRunState.failed.map((f) => ({ + name: f.name, + file: f.file, + })); + logger.debug("Loaded failed filter from previous run", { + count: failedFilter.length, + scenarios: failedFilter, + }); + } + } + // Parse options const maxConcurrency = parsed.sequential ? 1 @@ -216,6 +277,7 @@ export async function runCommand( const runResult = await runWithSubprocess(scenarioFiles, { reporter, selectors, + failedFilter, maxConcurrency: maxConcurrency ?? 0, maxFailures: maxFailures ?? 0, timeout, @@ -233,6 +295,9 @@ export async function runCommand( skipped: runResult.skipped, }); + // Save run state for --failed flag support + await saveLastRunState(cwd, runResult); + return runResult.failed > 0 ? EXIT_CODE.FAILURE : EXIT_CODE.SUCCESS; } catch (err: unknown) { logger.error("Unexpected error in run command", { error: err }); @@ -252,6 +317,7 @@ async function runWithSubprocess( options: { reporter: Reporter; selectors: readonly string[]; + failedFilter?: readonly FailedScenarioFilter[]; maxConcurrency: number; maxFailures: number; timeout: number; @@ -265,6 +331,7 @@ async function runWithSubprocess( const { reporter, selectors, + failedFilter, maxConcurrency, maxFailures, timeout, @@ -290,6 +357,7 @@ async function runWithSubprocess( type: "run", filePaths, selectors, + failedFilter, maxConcurrency, maxFailures, timeout, diff --git a/src/cli/state.ts b/src/cli/state.ts new file mode 100644 index 0000000..3393f23 --- /dev/null +++ b/src/cli/state.ts @@ -0,0 +1,148 @@ +/** + * State persistence for CLI commands + * + * Manages the `.probitas/` directory and stores run state between executions. + * + * @module + */ + +import { join } from "@std/path"; +import { relative } from "@std/path/relative"; +import type { RunResult } from "@probitas/runner"; + +/** + * A scenario that failed in a previous run + */ +export interface FailedScenario { + /** Scenario name */ + readonly name: string; + /** Relative file path from project root */ + readonly file: string; + /** Error message (optional, for display purposes) */ + readonly error?: string; +} + +/** + * State of the last run + */ +export interface LastRunState { + /** Schema version for forward compatibility */ + readonly version: 1; + /** ISO timestamp of when the run completed */ + readonly timestamp: string; + /** List of failed scenarios */ + readonly failed: readonly FailedScenario[]; +} + +const STATE_DIR_NAME = ".probitas"; +const LAST_RUN_FILE = "last-run.json"; +const CURRENT_VERSION = 1; + +/** + * Get the state directory path, creating it if it doesn't exist + * + * @param cwd - Project root directory + * @returns Path to the state directory + */ +export async function getStateDir(cwd: string): Promise { + const stateDir = join(cwd, STATE_DIR_NAME); + try { + await Deno.mkdir(stateDir, { recursive: true }); + } catch (error) { + // Ignore if already exists + if (!(error instanceof Deno.errors.AlreadyExists)) { + throw error; + } + } + return stateDir; +} + +/** + * Save the last run state to disk + * + * Extracts failed scenarios from the run result and persists them + * to `.probitas/last-run.json`. + * + * @param cwd - Project root directory + * @param result - Run result from scenario execution + */ +export async function saveLastRunState( + cwd: string, + result: RunResult, +): Promise { + const stateDir = await getStateDir(cwd); + const statePath = join(stateDir, LAST_RUN_FILE); + + // Extract failed scenarios from result + const failed: FailedScenario[] = []; + for (const s of result.scenarios) { + if (s.status !== "failed") continue; + + // Now TypeScript knows s.status is "failed" and s.error exists + const metadata = s.metadata; + const filePath = metadata.origin?.path ?? "unknown"; + const relativeFile = filePath !== "unknown" + ? relative(cwd, filePath) + : "unknown"; + + failed.push({ + name: metadata.name, + file: relativeFile, + error: s.error instanceof Error + ? s.error.message + : typeof s.error === "string" + ? s.error + : undefined, + }); + } + + const state: LastRunState = { + version: CURRENT_VERSION, + timestamp: new Date().toISOString(), + failed, + }; + + await Deno.writeTextFile(statePath, JSON.stringify(state, null, 2) + "\n"); +} + +/** + * Load the last run state from disk + * + * @param cwd - Project root directory + * @returns The last run state, or undefined if no state file exists or it's invalid + */ +export async function loadLastRunState( + cwd: string, +): Promise { + const statePath = join(cwd, STATE_DIR_NAME, LAST_RUN_FILE); + + try { + const content = await Deno.readTextFile(statePath); + const state = JSON.parse(content); + + // Validate version + if (state.version !== CURRENT_VERSION) { + return undefined; + } + + // Basic validation of required fields + if ( + typeof state.timestamp !== "string" || + !Array.isArray(state.failed) + ) { + return undefined; + } + + return state as LastRunState; + } catch (error) { + // File doesn't exist or can't be read + if (error instanceof Deno.errors.NotFound) { + return undefined; + } + // Invalid JSON or other error - treat as missing state + if (error instanceof SyntaxError) { + return undefined; + } + throw error; + } +} diff --git a/src/cli/state_test.ts b/src/cli/state_test.ts new file mode 100644 index 0000000..318adce --- /dev/null +++ b/src/cli/state_test.ts @@ -0,0 +1,313 @@ +/** + * Tests for state persistence module + * + * @requires --allow-read Permission to read state files + * @requires --allow-write Permission to write state files + * @module + */ + +import { assertEquals, assertExists } from "@std/assert"; +import { describe, it } from "@std/testing/bdd"; +import { sandbox } from "@lambdalisue/sandbox"; +import { join } from "@std/path"; +import type { RunResult, ScenarioResult } from "@probitas/runner"; +import type { ScenarioMetadata } from "@probitas/core"; +import { + getStateDir, + type LastRunState, + loadLastRunState, + saveLastRunState, +} from "./state.ts"; + +/** + * Helper to create ScenarioMetadata for tests + */ +function createMetadata( + name: string, + options?: { + tags?: string[]; + origin?: { path: string }; + }, +): ScenarioMetadata { + return { + name, + tags: options?.tags ?? [], + steps: [], + origin: options?.origin, + }; +} + +/** + * Helper to create a passed ScenarioResult + */ +function createPassedScenario( + name: string, + options?: { + tags?: string[]; + origin?: { path: string }; + duration?: number; + }, +): ScenarioResult { + return { + status: "passed", + metadata: createMetadata(name, options), + steps: [], + duration: options?.duration ?? 100, + }; +} + +/** + * Helper to create a failed ScenarioResult + */ +function createFailedScenario( + name: string, + error: unknown, + options?: { + tags?: string[]; + origin?: { path: string }; + duration?: number; + }, +): ScenarioResult { + return { + status: "failed", + metadata: createMetadata(name, options), + steps: [], + duration: options?.duration ?? 100, + error, + }; +} + +describe("state persistence", () => { + describe("getStateDir", () => { + it("creates .probitas directory if it doesn't exist", async () => { + await using sbox = await sandbox(); + + const stateDir = await getStateDir(sbox.path); + + assertEquals(stateDir, join(sbox.path, ".probitas")); + const stat = await Deno.stat(stateDir); + assertEquals(stat.isDirectory, true); + }); + + it("returns existing .probitas directory", async () => { + await using sbox = await sandbox(); + await Deno.mkdir(join(sbox.path, ".probitas")); + + const stateDir = await getStateDir(sbox.path); + + assertEquals(stateDir, join(sbox.path, ".probitas")); + }); + }); + + describe("saveLastRunState", () => { + it("saves failed scenarios to last-run.json", async () => { + await using sbox = await sandbox(); + + const result: RunResult = { + total: 3, + passed: 1, + failed: 2, + skipped: 0, + duration: 1000, + scenarios: [ + createPassedScenario("Passing Test", { + origin: { path: join(sbox.path, "tests/pass.probitas.ts") }, + }), + createFailedScenario("Failing Test 1", new Error("Test failed"), { + tags: ["api"], + origin: { path: join(sbox.path, "tests/fail1.probitas.ts") }, + duration: 200, + }), + createFailedScenario("Failing Test 2", "String error", { + origin: { path: join(sbox.path, "tests/fail2.probitas.ts") }, + duration: 300, + }), + ], + }; + + await saveLastRunState(sbox.path, result); + + const statePath = join(sbox.path, ".probitas", "last-run.json"); + const content = await Deno.readTextFile(statePath); + const state = JSON.parse(content) as LastRunState; + + assertEquals(state.version, 1); + assertExists(state.timestamp); + assertEquals(state.failed.length, 2); + assertEquals(state.failed[0].name, "Failing Test 1"); + assertEquals(state.failed[0].file, "tests/fail1.probitas.ts"); + assertEquals(state.failed[0].error, "Test failed"); + assertEquals(state.failed[1].name, "Failing Test 2"); + assertEquals(state.failed[1].file, "tests/fail2.probitas.ts"); + assertEquals(state.failed[1].error, "String error"); + }); + + it("saves empty failed array when all tests pass", async () => { + await using sbox = await sandbox(); + + const result: RunResult = { + total: 1, + passed: 1, + failed: 0, + skipped: 0, + duration: 100, + scenarios: [ + createPassedScenario("Passing Test", { + origin: { path: join(sbox.path, "tests/pass.probitas.ts") }, + }), + ], + }; + + await saveLastRunState(sbox.path, result); + + const state = await loadLastRunState(sbox.path); + assertExists(state); + assertEquals(state.failed.length, 0); + }); + + it("handles scenarios without origin path", async () => { + await using sbox = await sandbox(); + + const result: RunResult = { + total: 1, + passed: 0, + failed: 1, + skipped: 0, + duration: 100, + scenarios: [ + createFailedScenario("No Origin", new Error("Failed")), + ], + }; + + await saveLastRunState(sbox.path, result); + + const state = await loadLastRunState(sbox.path); + assertExists(state); + assertEquals(state.failed[0].file, "unknown"); + }); + }); + + describe("loadLastRunState", () => { + it("returns undefined when no state file exists", async () => { + await using sbox = await sandbox(); + + const state = await loadLastRunState(sbox.path); + + assertEquals(state, undefined); + }); + + it("loads valid state from disk", async () => { + await using sbox = await sandbox(); + await Deno.mkdir(join(sbox.path, ".probitas")); + + const savedState: LastRunState = { + version: 1, + timestamp: "2024-01-15T10:30:00.000Z", + failed: [ + { name: "Test 1", file: "tests/test1.probitas.ts" }, + { + name: "Test 2", + file: "tests/test2.probitas.ts", + error: "Some error", + }, + ], + }; + await Deno.writeTextFile( + join(sbox.path, ".probitas", "last-run.json"), + JSON.stringify(savedState), + ); + + const state = await loadLastRunState(sbox.path); + + assertExists(state); + assertEquals(state.version, 1); + assertEquals(state.timestamp, "2024-01-15T10:30:00.000Z"); + assertEquals(state.failed.length, 2); + assertEquals(state.failed[0].name, "Test 1"); + assertEquals(state.failed[1].error, "Some error"); + }); + + it("returns undefined for invalid version", async () => { + await using sbox = await sandbox(); + await Deno.mkdir(join(sbox.path, ".probitas")); + + const invalidState = { + version: 999, + timestamp: "2024-01-15T10:30:00.000Z", + failed: [], + }; + await Deno.writeTextFile( + join(sbox.path, ".probitas", "last-run.json"), + JSON.stringify(invalidState), + ); + + const state = await loadLastRunState(sbox.path); + + assertEquals(state, undefined); + }); + + it("returns undefined for malformed JSON", async () => { + await using sbox = await sandbox(); + await Deno.mkdir(join(sbox.path, ".probitas")); + + await Deno.writeTextFile( + join(sbox.path, ".probitas", "last-run.json"), + "{ invalid json", + ); + + const state = await loadLastRunState(sbox.path); + + assertEquals(state, undefined); + }); + + it("returns undefined for missing required fields", async () => { + await using sbox = await sandbox(); + await Deno.mkdir(join(sbox.path, ".probitas")); + + const invalidState = { + version: 1, + // missing timestamp and failed + }; + await Deno.writeTextFile( + join(sbox.path, ".probitas", "last-run.json"), + JSON.stringify(invalidState), + ); + + const state = await loadLastRunState(sbox.path); + + assertEquals(state, undefined); + }); + }); + + describe("round-trip", () => { + it("saves and loads state correctly", async () => { + await using sbox = await sandbox(); + + const result: RunResult = { + total: 2, + passed: 1, + failed: 1, + skipped: 0, + duration: 500, + scenarios: [ + createPassedScenario("Pass", { duration: 100 }), + createFailedScenario("Fail", new Error("Expected failure"), { + tags: ["integration"], + origin: { path: join(sbox.path, "e2e/fail.probitas.ts") }, + duration: 400, + }), + ], + }; + + await saveLastRunState(sbox.path, result); + const loaded = await loadLastRunState(sbox.path); + + assertExists(loaded); + assertEquals(loaded.version, 1); + assertEquals(loaded.failed.length, 1); + assertEquals(loaded.failed[0].name, "Fail"); + assertEquals(loaded.failed[0].file, "e2e/fail.probitas.ts"); + assertEquals(loaded.failed[0].error, "Expected failure"); + }); + }); +}); diff --git a/src/cli/unknown_args.ts b/src/cli/unknown_args.ts new file mode 100644 index 0000000..c8adc7e --- /dev/null +++ b/src/cli/unknown_args.ts @@ -0,0 +1,245 @@ +/** + * Unknown CLI arguments detection and hint generation + * + * @module + */ + +import { closestString } from "@std/text/closest-string"; +import { levenshteinDistance } from "@std/text/levenshtein-distance"; + +/** + * Maximum Levenshtein distance for suggesting similar options + */ +const MAX_SUGGESTION_DISTANCE = 3; + +/** + * Represents an unknown argument detected during parsing + */ +export interface UnknownArg { + /** The full argument string (e.g., "--tag" or "--tag=foo") */ + arg: string; + /** The option name without dashes (e.g., "tag") */ + key: string; + /** The value if provided with = (e.g., "foo" for "--tag=foo") */ + value: unknown; +} + +/** + * Configuration for the unknown argument handler + */ +export interface UnknownArgHandlerOptions { + /** List of known option names (without dashes) */ + knownOptions: readonly string[]; + /** The command name for help text (e.g., "run" or "list") */ + commandName: string; +} + +/** + * Result from unknown argument collection + */ +export interface UnknownArgResult { + /** List of unknown arguments detected */ + unknownArgs: UnknownArg[]; + /** Whether any unknown arguments were found */ + hasErrors: boolean; +} + +/** + * Creates a handler for detecting unknown arguments in parseArgs + * + * The returned object contains: + * - `handler`: The callback to pass to parseArgs `unknown` option + * - `result`: The collection of unknown arguments after parsing + * + * @example Usage with parseArgs + * ```ts + * import { parseArgs } from "@std/cli"; + * import { createUnknownArgHandler, formatUnknownArgError } from "./unknown_args.ts"; + * + * const knownOptions = ["help", "verbose", "selector"]; + * const { handler, result } = createUnknownArgHandler({ + * knownOptions, + * commandName: "run", + * }); + * + * const args = ["--unknown", "--help"]; + * const parsed = parseArgs(args, { + * boolean: ["help"], + * unknown: handler, + * }); + * + * if (result.hasErrors) { + * for (const unknown of result.unknownArgs) { + * console.error(formatUnknownArgError(unknown, { + * knownOptions, + * commandName: "run", + * })); + * } + * } + * ``` + */ +export function createUnknownArgHandler( + _options: UnknownArgHandlerOptions, +): { + handler: (arg: string, key?: string, value?: unknown) => boolean; + result: UnknownArgResult; +} { + const result: UnknownArgResult = { + unknownArgs: [], + hasErrors: false, + }; + + const handler = (arg: string, key?: string, value?: unknown): boolean => { + // Only handle option arguments (starting with -) + if (!arg.startsWith("-")) { + return true; // Allow positional arguments + } + + // key is undefined for malformed arguments, extract from arg + const actualKey = key ?? extractKeyFromArg(arg); + + result.unknownArgs.push({ + arg, + key: actualKey, + value, + }); + result.hasErrors = true; + + // Return false to exclude from parse result + return false; + }; + + return { handler, result }; +} + +/** + * Extracts the option key from an argument string + */ +function extractKeyFromArg(arg: string): string { + // Handle --option=value or --option + const match = arg.match(/^--?([^=]+)/); + return match?.[1] ?? arg; +} + +/** + * Generates a contextual hint for an unknown argument + * + * Provides helpful suggestions for common mistakes like: + * - --tag → suggests -s 'tag:' + * - --name → suggests -s 'name:' + * - --filter → suggests -s '' + * - Typos → suggests closest known option + */ +export function generateHint( + unknown: UnknownArg, + options: UnknownArgHandlerOptions, +): string { + const { key, value } = unknown; + const { knownOptions, commandName } = options; + + // Check for tag/tags pattern + if (key === "tag" || key === "tags") { + const tagValue = value ?? ""; + return `Did you mean '-s "tag:${tagValue}"'? Use the selector option to filter by tag.`; + } + + // Check for name/names pattern + if (key === "name" || key === "names") { + const nameValue = value ?? ""; + return `Did you mean '-s "name:${nameValue}"'? Use the selector option to filter by name.`; + } + + // Check for filter-like options + if (key === "filter" || key === "select" || key === "match") { + const filterValue = value ?? ""; + return `Did you mean '-s "${filterValue}"'? Use the selector option to filter scenarios.`; + } + + // Check for similar options using Levenshtein distance + const similar = findSimilarOption(key, knownOptions); + if (similar) { + return `Did you mean '--${similar}'?`; + } + + // Fallback to generic help message + return `Run 'probitas ${commandName} --help' for available options.`; +} + +/** + * Finds a similar known option using Levenshtein distance + * + * Returns the closest match if within MAX_SUGGESTION_DISTANCE, otherwise undefined. + * This is the same approach used by Cliffy for production-quality suggestions. + */ +export function findSimilarOption( + unknown: string, + knownOptions: readonly string[], +): string | undefined { + if (knownOptions.length === 0) { + return undefined; + } + + // Use closestString from @std/text (same as Cliffy) + const closest = closestString(unknown, knownOptions as string[]); + + // Only suggest if within threshold + const distance = levenshteinDistance(unknown, closest); + if (distance <= MAX_SUGGESTION_DISTANCE) { + return closest; + } + + return undefined; +} + +/** + * Formats an error message for an unknown argument + */ +export function formatUnknownArgError( + unknown: UnknownArg, + options: UnknownArgHandlerOptions, +): string { + const hint = generateHint(unknown, options); + return `Unknown option: ${unknown.arg}\n${hint}`; +} + +/** + * Extracts all known options from parseArgs configuration + * + * Combines string options, boolean options, and their aliases. + * Also handles --no-* variants for boolean options. + */ +export function extractKnownOptions(config: { + string?: readonly string[]; + boolean?: readonly string[]; + alias?: Record; +}): string[] { + const known = new Set(); + + // Add string options + for (const opt of config.string ?? []) { + known.add(opt); + } + + // Add boolean options and their --no-* variants + for (const opt of config.boolean ?? []) { + known.add(opt); + // Boolean options also accept --no-