Skip to content

Commit 3758389

Browse files
leifericfclaude
andcommitted
fix(cli): batch UX improvements for CLI help and feedback
- --reanalyze flag: add default hint (only unanalyzed files) - do-update: add next-step hint when analyze=true - list-databases --delete: warn that analysis data is lost - watch --concurrency: use specific description, not shared flag - introspect epilog: correct target default from examples to all - ask --continue-from: add placement guidance - benchmark canary warning: use [CANARY WARNING] prefix - benchmark pre-run: show mode (fast/full/no-judge) in log - benchmark checkpoint error: show (changed) not raw hashes, add recovery hint - benchmark --resume: detect when path is consumed as checkpoint ID - benchmark cost warning: standardize to [COST WARNING] prefix Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 4efb9e1 commit 3758389

3 files changed

Lines changed: 38 additions & 14 deletions

File tree

src/noumenon/benchmark.clj

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1026,8 +1026,8 @@
10261026
cost-ans (llm/estimate-cost model est-in est-out)
10271027
cost-jdg (llm/estimate-cost (or judge-model model) est-in est-out)
10281028
total (+ cost-ans cost-jdg)]
1029-
(log! (str "WARNING: Benchmarks are expensive. "
1030-
total-stages " stages × ~"
1029+
(log! (str "[COST WARNING] Benchmarks are expensive. "
1030+
total-stages " stages x ~"
10311031
(:input avg-tokens-per-stage) " input + ~"
10321032
(:output avg-tokens-per-stage) " output tokens/stage"))
10331033
(when (pos? total)
@@ -1201,7 +1201,7 @@
12011201
" run-id=" run-id
12021202
" details=" (pr-str (:details eval-result))))
12031203
(when (= :warn (:status eval-result))
1204-
(log! "WARNING: All canary questions scored :wrong. Results may be unreliable — check model/provider configuration."))))
1204+
(log! "[CANARY WARNING] All canary questions failed — results may be unreliable. Check that analyze has been run and the correct model is configured."))))
12051205

12061206
(when-not @stop-flag
12071207
(run-pairs! rest-pairs shared concurrency))))
@@ -1480,6 +1480,10 @@
14801480
" questions=" (count questions)
14811481
" layers=" (str/join "," (map name layers))
14821482
" stages=" total
1483+
" mode=" (cond
1484+
(:deterministic-only mode) "fast"
1485+
(:skip-judge mode) "no-judge"
1486+
:else "full")
14831487
(when (:skip-judge mode) " skip-judge")
14841488
(when (:deterministic-only mode) " deterministic-only")
14851489
(when resuming?

src/noumenon/cli.clj

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@
113113

114114
(def ^:private reanalyze-flag
115115
{:flag "--reanalyze" :key :reanalyze :parse :string
116-
:desc "Re-analyze files: all, prompt-changed, model-changed, stale"
116+
:desc "Re-analyze files: all, prompt-changed, model-changed, stale (default: only unanalyzed files)"
117117
:error-missing :missing-reanalyze-value})
118118

119119
(def ^:private analyze-flags
@@ -150,7 +150,10 @@
150150
:desc "Also run LLM analysis on changed files"}
151151
model-flag
152152
(assoc provider-flag :valid all-valid-providers)
153-
db-dir-flag concurrency-flag]
153+
db-dir-flag
154+
{:flag "--concurrency" :key :concurrency :parse :range-int :min 1 :max 20
155+
:desc "Parallel workers for import/enrich, 1-20 (default: 8)"
156+
:error-invalid :invalid-concurrency :error-missing :missing-concurrency-value}]
154157
:initial {}
155158
:positionals {:required 1 :error :no-repo-path :keys [:repo-path]}})
156159

@@ -223,7 +226,7 @@
223226
:error-invalid :invalid-max-iterations
224227
:error-missing :missing-max-iterations-value}
225228
{:flag "--continue-from" :key :continue-from :parse :string
226-
:desc "Session ID from a budget-exhausted run — resumes the agent"}
229+
:desc "Session ID from a budget-exhausted run — resumes the agent (place before <repo-path>)"}
227230
db-dir-flag]
228231
verbose-flags))
229232
:initial {:subcommand "ask"}
@@ -325,7 +328,7 @@
325328
"introspect" {:spec introspect-command-spec
326329
:summary "Autonomous self-improvement loop (optimize prompts via benchmark)"
327330
:usage "introspect [options] <repo-path>"
328-
:epilog "Runs an autonomous loop: propose prompt change, evaluate via benchmark,\nkeep if improved, revert if not. Uses an LLM to propose improvements and\nthe agent benchmark to evaluate them.\n\nTargets (comma-separated): examples (default), system-prompt, rules, code, train.\nThe :code target requires passing lint and compilation. The :train target retrains\nthe on-device ML model. Example: --target examples,system-prompt\nUse --max-hours or --max-cost for overnight runs."}
331+
:epilog "Runs an autonomous loop: propose prompt change, evaluate via benchmark,\nkeep if improved, revert if not. Uses an LLM to propose improvements and\nthe agent benchmark to evaluate them.\n\nTargets (comma-separated): examples, system-prompt, rules, code, train\n(default: all — LLM chooses based on benchmark results).\nThe :code target requires passing lint and compilation. The :train target retrains\nthe on-device ML model. Example: --target examples,system-prompt\nUse --max-hours or --max-cost for overnight runs."}
329332
"reseed" {:spec {:flags [db-dir-flag]
330333
:initial {:subcommand "reseed"}
331334
:positionals {:required 0 :error nil :keys []}}
@@ -511,6 +514,11 @@
511514
(assoc result :layers (mapv keyword (str/split layers-str #",")))
512515
result)]
513516
(cond
517+
(and (= :no-repo-path (:error result))
518+
(:resume result)
519+
(str/starts-with? (str (:resume result)) "/"))
520+
(assoc result :error :resume-consumed-repo-path)
521+
514522
(:error result) result
515523
;; --fast: deterministic only, full layer only (cheapest mode)
516524
(:fast result) (-> result

src/noumenon/main.clj

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,10 @@
201201
sync-opts (cond-> (build-sync-opts opts)
202202
analyze (assoc :meta-db (d/db meta-conn)))
203203
result (sync/update-repo! conn repo-path repo-uri sync-opts)]
204-
(when-not analyze
204+
(if analyze
205+
(log! (str "Next: run '" cli/program-name " ask -q \"...\" " repo-path
206+
"' or '" cli/program-name " query <query-name> " repo-path
207+
"' to explore the updated graph."))
205208
(log! (str "Next: run '" cli/program-name " analyze " repo-path
206209
"' to enrich with semantic metadata.")))
207210
{:exit 0 :result result}))))
@@ -397,6 +400,7 @@
397400
(do (print-error! (str "Database \"" db-name "\" not found.")) {:exit 1})
398401
(do (db/delete-db client db-name)
399402
(log! (str "Deleted database \"" db-name "\"."))
403+
(log! "WARNING: All analysis data has been destroyed. Re-running analyze may be expensive.")
400404
(log! (str "Re-import: " cli/program-name " import <repo-path>"))
401405
{:exit 0})))
402406
(let [names (db/list-db-dirs db-dir)]
@@ -441,17 +445,24 @@
441445
:rubric-hash "Rubric"
442446
:answer-prompt-hash "Answer prompt"})
443447

448+
(def ^:private hash-fields
449+
#{:question-set-hash :rubric-hash :answer-prompt-hash})
450+
444451
(defn- format-compat-error
445452
"Format a checkpoint compatibility error message."
446453
[mismatches]
447454
(str "Incompatible checkpoint. The benchmark configuration has changed "
448-
"since this checkpoint was created. Start a fresh run without --resume.\n"
455+
"since this checkpoint was created.\n"
449456
"Mismatched fields:\n"
450457
(str/join "\n"
451-
(map #(str " " (get compat-field-labels (:field %) (name (:field %)))
452-
": checkpoint=" (:checkpoint %)
453-
" current=" (:current %))
454-
mismatches))))
458+
(map (fn [{:keys [field checkpoint current]}]
459+
(let [label (get compat-field-labels field (name field))]
460+
(if (hash-fields field)
461+
(str " " label ": (changed)")
462+
(str " " label ": checkpoint=" checkpoint
463+
" current=" current))))
464+
mismatches))
465+
"\nStart a fresh run: " cli/program-name " benchmark <repo-path>"))
455466

456467
(defn- do-benchmark-resume
457468
"Handle --resume path for benchmark. Returns {:exit n}."
@@ -684,6 +695,7 @@
684695
:unknown-subcommand #(str "Unknown subcommand: " (:subcommand %)
685696
". Run '" cli/program-name " --help' for available subcommands.")
686697
:no-repo-path "Missing <repo-path> argument."
698+
:resume-consumed-repo-path "Missing <repo-path> argument. Did --resume consume your repo-path? Place --resume after <repo-path>."
687699
:query-missing-args "Missing <query-name> and <repo-path> arguments."
688700
:missing-db-dir-value "Missing value for --db-dir."
689701
:missing-delete-value "Missing database name for --delete."
@@ -724,7 +736,7 @@
724736
#{:no-args :unknown-subcommand})
725737

726738
(def ^:private errors-with-subcommand-usage
727-
#{:no-repo-path :missing-db-dir-value :unknown-flag
739+
#{:no-repo-path :resume-consumed-repo-path :missing-db-dir-value :unknown-flag
728740
:ask-missing-question :ask-missing-args :query-missing-args
729741
:missing-param-value :invalid-param-value
730742
:invalid-concurrency :missing-concurrency-value

0 commit comments

Comments
 (0)