fix(bench): count partially-completed questions in multi-layer runs

When budget exhaustion interrupts a multi-layer benchmark (e.g.
--layers raw,full --max-questions 1), questions that completed some
layers but not all were excluded from results entirely, producing
misleading output like "questions=0 full=0.00" even though a question
scored correct on the full layer.

Now stages->results includes questions with at least one fully-scored
layer, and aggregate-scores only counts questions that have a score
for each layer when computing per-layer means.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: leifericf

src/noumenon/benchmark.clj

@@ -533,7 +533,8 @@
          mean       (fn [xs] (if (seq xs) (/ (reduce + xs) (count xs)) 0.0))
          layer-key  (fn [layer] (keyword (str (name layer) "-score")))
          layer-mean (fn [layer rs]
-                      (mean (mapv #(score-value (get % (layer-key layer))) rs)))
+                      (let [scored (filterv #(contains? % (layer-key layer)) rs)]
+                        (mean (mapv #(score-value (get % (layer-key layer))) scored))))
          by-cat     (group-by :category results)
          det-rs     (filterv #(= :deterministic (:scoring %)) results)
          llm-rs     (filterv #(not= :deterministic (:scoring %)) results)
@@ -548,9 +549,11 @@
              {:question-count      (count results)
               :canonical           canonical?
               :deterministic-count (count det-rs)
-              :deterministic-mean  (mean (mapv #(score-value (get % (layer-key primary-layer))) det-rs))
+              :deterministic-mean  (let [scored (filterv #(contains? % (layer-key primary-layer)) det-rs)]
+                                     (mean (mapv #(score-value (get % (layer-key primary-layer))) scored)))
               :llm-judged-count    (count llm-rs)
-              :llm-judged-mean     (mean (mapv #(score-value (get % (layer-key primary-layer))) llm-rs))
+              :llm-judged-mean     (let [scored (filterv #(contains? % (layer-key primary-layer)) llm-rs)]
+                                     (mean (mapv #(score-value (get % (layer-key primary-layer))) scored)))
               :per-category        (into {}
                                          (map (fn [[cat rs]]
                                                 [cat (reduce (fn [m layer]
@@ -952,16 +955,32 @@
                    :completed-at (java.util.Date.)})
               (throw e))))))))
 
+(defn- layer-stage-keys
+  "Return stage keys for a single [question, layer] pair under the given mode."
+  [question layer mode]
+  (let [qid            (:id question)
+        skip-judge?    (:skip-judge mode)
+        deterministic? (= :deterministic (:scoring question))]
+    (cond-> [[qid layer :answer]]
+      (or (not skip-judge?) deterministic?)
+      (conj [qid layer :judge]))))
+
 (defn stages->results
-  "Convert stages map to result seq. Only includes questions with all expected stages complete.
+  "Convert stages map to result seq. Includes questions with at least one layer
+   fully scored (answer + judge complete). Layers that haven't finished are omitted
+   from the result map rather than counted as wrong.
    Results include per-layer scores keyed as :<layer>-score, :<layer>-reasoning, :<layer>-answer."
   ([questions stages] (stages->results questions stages nil))
   ([questions stages mode]
    (let [layers (resolve-layers (or mode {}))]
      (for [q questions
-           :let [qid      (:id q)
-                 exp-keys (if mode (mode-stage-keys q mode) (stage-keys q layers))]
-           :when (every? #(contains? stages %) exp-keys)]
+           :let [qid             (:id q)
+                 complete-layers (filterv
+                                  (fn [layer]
+                                    (every? #(contains? stages %)
+                                            (layer-stage-keys q layer mode)))
+                                  layers)]
+           :when (seq complete-layers)]
        (reduce (fn [result layer]
                  (let [judge-key [qid layer :judge]
                        judge  (get-in stages [judge-key :result])
@@ -977,7 +996,7 @@
                 :category   (:category q)
                 :scoring    (:scoring q)
                 :query-name (:query-name q)}
-               layers)))))
+               complete-layers)))))
 
 ;; --- Rate limiting ---
 

test/noumenon/benchmark_test.clj

@@ -341,12 +341,20 @@
     (is (= "ra" (:raw-answer (first results))))
     (is (= :partial (:raw-score (first results))))))
 
-(deftest stages->results-incomplete-question-excluded
-  (let [qs     [{:id :q01 :category :single-hop :query-name "test"}]
-        stages {[:q01 :full :answer] {:status :ok :result "qa"}
-                [:q01 :full :judge]  {:status :ok :result {:score :correct :reasoning "good"}}}
-        results (vec (bench/stages->results qs stages))]
-    (is (= 0 (count results)))))
+(deftest stages->results-partial-layers-included
+  (testing "Question with only :full complete is included with :full-score only"
+    (let [qs     [{:id :q01 :category :single-hop :query-name "test"}]
+          stages {[:q01 :full :answer] {:status :ok :result "qa"}
+                  [:q01 :full :judge]  {:status :ok :result {:score :correct :reasoning "good"}}}
+          results (vec (bench/stages->results qs stages))]
+      (is (= 1 (count results)))
+      (is (= :correct (:full-score (first results))))
+      (is (nil? (:raw-score (first results))))))
+  (testing "Question with no complete layers is excluded"
+    (let [qs     [{:id :q01 :category :single-hop :query-name "test"}]
+          stages {[:q01 :full :answer] {:status :ok :result "qa"}}
+          results (vec (bench/stages->results qs stages))]
+      (is (= 0 (count results))))))
 
 (deftest stages->results-nil-judge-uses-wrong-when-judge-present
   (let [qs     [{:id :q01 :category :single-hop :query-name "test"}]