AMPR-185 #532: ampere-eval 3 — Meter & Tolerance grading interface

Add Reading, Tolerance, Meter (fun interface), MeterError (sealed),
OutcomeMeter, CompositeMeter (WeightedMeter), and JudgeMeter (JudgeClient
SAM + PlaybackRelay seam) under link.socket.ampere.eval.meter.
Includes 26 commonTest tests covering all tasks (3.1–3.5).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: wow-miley

ampere-eval/src/commonMain/kotlin/link/socket/ampere/eval/meter/CompositeMeter.kt

@@ -0,0 +1,60 @@
+package link.socket.ampere.eval.meter
+
+import link.socket.ampere.eval.trace.Trace
+
+/**
+ * A [Meter] paired with a relative [weight] and a [required] flag.
+ *
+ * If [required] is `true`, a failed or missing reading from this meter causes the
+ * [CompositeMeter] to fail immediately.
+ */
+data class WeightedMeter(
+    val meter: Meter,
+    val weight: Double,
+    val required: Boolean = false,
+)
+
+/**
+ * Aggregates child meters into a single weighted-mean [Reading].
+ *
+ * - Aggregate score = weighted mean of successful child scores.
+ * - A required child that returns `Result.failure` propagates the failure immediately.
+ * - A required child whose [Reading.passed] is `false` forces the composite to fail
+ *   even if the weighted mean clears [tolerance].
+ */
+class CompositeMeter(
+    val meterId: String,
+    private val tolerance: Tolerance,
+    private val children: List<WeightedMeter>,
+) : Meter {
+
+    override suspend fun measure(trace: Trace): Result<Reading> {
+        val childReadings = mutableListOf<Pair<WeightedMeter, Reading>>()
+
+        for (weighted in children) {
+            val result = weighted.meter.measure(trace)
+            when {
+                result.isFailure && weighted.required -> return result
+                result.isSuccess -> childReadings.add(weighted to result.getOrThrow())
+            }
+        }
+
+        if (childReadings.isEmpty()) {
+            return Result.failure(MeterError.NoReadings(meterId))
+        }
+
+        val totalWeight = childReadings.sumOf { (w, _) -> w.weight }
+        val aggregateScore = if (totalWeight > 0) {
+            childReadings.sumOf { (w, r) -> w.weight * r.score } / totalWeight
+        } else {
+            0.0
+        }
+
+        val anyRequiredFailed = childReadings.any { (w, r) -> w.required && !r.passed }
+        val passed = !anyRequiredFailed && tolerance.passes(aggregateScore)
+
+        return Result.success(
+            Reading(score = aggregateScore, passed = passed, meterId = meterId),
+        )
+    }
+}

ampere-eval/src/commonMain/kotlin/link/socket/ampere/eval/meter/JudgeMeter.kt

@@ -0,0 +1,72 @@
+package link.socket.ampere.eval.meter
+
+import link.socket.ampere.agents.domain.routing.CognitiveRelay
+import link.socket.ampere.eval.trace.Trace
+
+/**
+ * Performs an LLM completion call (prompt → text) on behalf of a [JudgeMeter].
+ *
+ * In production this wraps the live provider client. In CI a stubbed lambda returns
+ * a fixed response, and routing is served by a [link.socket.ampere.eval.relay.PlaybackRelay]
+ * wired as the [JudgeMeter.relay] — making the judge deterministic under replay.
+ */
+fun interface JudgeClient {
+    suspend fun call(prompt: String): Result<String>
+}
+
+/**
+ * LLM-grades a [Trace] against a [rubric], parsing a normalized score from the response.
+ *
+ * The [relay] resolves routing (which model to call) — swapping in a
+ * [link.socket.ampere.eval.relay.PlaybackRelay] makes the judge deterministic in CI.
+ * The [client] performs the actual completion; for tests inject a stubbed lambda.
+ *
+ * The judge response must contain `"Score: X.X"` (case-insensitive) where `X.X` is
+ * a value in `0.0..1.0`. Anything else is a [MeterError.MalformedJudgeResponse].
+ */
+class JudgeMeter(
+    val meterId: String,
+    private val tolerance: Tolerance,
+    private val rubric: String,
+    val relay: CognitiveRelay,
+    private val client: JudgeClient,
+) : Meter {
+
+    override suspend fun measure(trace: Trace): Result<Reading> {
+        if (trace.events.isEmpty()) {
+            return Result.failure(MeterError.EmptyTrace(meterId))
+        }
+        val prompt = buildPrompt(trace)
+        return client.call(prompt).fold(
+            onSuccess = { response ->
+                val score = parseScore(response)
+                    ?: return Result.failure(MeterError.MalformedJudgeResponse(meterId, response))
+                Result.success(
+                    Reading(score = score, passed = tolerance.passes(score), meterId = meterId),
+                )
+            },
+            onFailure = { Result.failure(it) },
+        )
+    }
+
+    private fun buildPrompt(trace: Trace): String {
+        val eventLines = trace.events.joinToString("\n") { "  [${it.index}] ${it.type} at ${it.timestamp}" }
+        return """Grade the trajectory below against the rubric.
+Respond with exactly: "Score: X.X" (0.0 to 1.0) then your reasoning.
+
+Rubric: $rubric
+
+Trajectory (${trace.events.size} events):
+$eventLines"""
+    }
+
+    companion object {
+        private val SCORE_REGEX = Regex("""(?i)score:\s*(\d+(?:\.\d+)?)""")
+
+        fun parseScore(response: String): Double? =
+            SCORE_REGEX.find(response)
+                ?.groupValues?.get(1)
+                ?.toDoubleOrNull()
+                ?.takeIf { it in 0.0..1.0 }
+    }
+}

ampere-eval/src/commonMain/kotlin/link/socket/ampere/eval/meter/Meter.kt

@@ -0,0 +1,27 @@
+package link.socket.ampere.eval.meter
+
+import kotlinx.serialization.Serializable
+import link.socket.ampere.eval.trace.Trace
+
+@Serializable
+data class Reading(
+    val score: Double,
+    val passed: Boolean,
+    val meterId: String,
+    val detail: Map<String, String> = emptyMap(),
+)
+
+data class Tolerance(val minScore: Double) {
+    fun passes(score: Double) = score >= minScore
+}
+
+fun interface Meter {
+    suspend fun measure(trace: Trace): Result<Reading>
+}
+
+sealed class MeterError(message: String, cause: Throwable? = null) : Exception(message, cause) {
+    class EmptyTrace(meterId: String) : MeterError("[$meterId] trace has no events")
+    class NoReadings(meterId: String) : MeterError("[$meterId] no child meters produced a reading")
+    class MalformedJudgeResponse(meterId: String, response: String) :
+        MeterError("[$meterId] judge response could not be parsed: «$response»")
+}

ampere-eval/src/commonMain/kotlin/link/socket/ampere/eval/meter/OutcomeMeter.kt

@@ -0,0 +1,34 @@
+package link.socket.ampere.eval.meter
+
+import link.socket.ampere.eval.trace.Trace
+import link.socket.ampere.eval.trace.TraceEvent
+
+/**
+ * Scores a [Trace] by inspecting its terminal event against a [predicate].
+ *
+ * Score is `1.0` on a match, `0.0` on a mismatch; [tolerance] determines pass/fail.
+ * A mismatch surfaces the terminal event type in [Reading.detail].
+ */
+class OutcomeMeter(
+    val meterId: String,
+    private val tolerance: Tolerance,
+    private val predicate: (TraceEvent) -> Boolean,
+) : Meter {
+
+    override suspend fun measure(trace: Trace): Result<Reading> {
+        if (trace.events.isEmpty()) {
+            return Result.failure(MeterError.EmptyTrace(meterId))
+        }
+        val terminal = trace.events.last()
+        val matched = predicate(terminal)
+        val score = if (matched) 1.0 else 0.0
+        val detail = if (!matched) {
+            mapOf("terminal_type" to terminal.type, "match" to "false")
+        } else {
+            emptyMap()
+        }
+        return Result.success(
+            Reading(score = score, passed = tolerance.passes(score), meterId = meterId, detail = detail),
+        )
+    }
+}

ampere-eval/src/commonTest/kotlin/link/socket/ampere/eval/meter/CompositeMeterTest.kt

@@ -0,0 +1,132 @@
+package link.socket.ampere.eval.meter
+
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertFalse
+import kotlin.test.assertIs
+import kotlin.test.assertTrue
+import kotlinx.coroutines.test.runTest
+import link.socket.ampere.eval.trace.Trace
+
+/** AMPR-185 task 3.3 validation. */
+class CompositeMeterTest {
+
+    private val alwaysPass = Tolerance(minScore = 0.0)
+
+    @Test
+    fun `aggregate score equals the weighted mean of children`() = runTest {
+        val composite = CompositeMeter(
+            meterId = "composite",
+            tolerance = alwaysPass,
+            children = listOf(
+                WeightedMeter(meter = fixedMeter("a", 0.8), weight = 1.0),
+                WeightedMeter(meter = fixedMeter("b", 0.4), weight = 3.0),
+            ),
+        )
+        val reading = composite.measure(anyTrace()).getOrThrow()
+
+        // (1.0 * 0.8 + 3.0 * 0.4) / 4.0 = (0.8 + 1.2) / 4.0 = 0.5
+        assertEquals(0.5, reading.score)
+    }
+
+    @Test
+    fun `passes when all required children pass`() = runTest {
+        val composite = CompositeMeter(
+            meterId = "composite",
+            tolerance = Tolerance(0.5),
+            children = listOf(
+                WeightedMeter(meter = fixedMeter("a", 1.0), weight = 1.0, required = true),
+                WeightedMeter(meter = fixedMeter("b", 1.0), weight = 1.0, required = true),
+            ),
+        )
+        val reading = composite.measure(anyTrace()).getOrThrow()
+
+        assertTrue(reading.passed)
+    }
+
+    @Test
+    fun `one failing required child fails the composite`() = runTest {
+        val composite = CompositeMeter(
+            meterId = "composite",
+            tolerance = Tolerance(0.0),
+            children = listOf(
+                WeightedMeter(meter = fixedMeter("a", 1.0), weight = 1.0, required = true),
+                WeightedMeter(meter = fixedMeter("b", 0.0), weight = 1.0, required = true),
+            ),
+        )
+        val reading = composite.measure(anyTrace()).getOrThrow()
+
+        assertFalse(reading.passed)
+    }
+
+    @Test
+    fun `required child failure propagates immediately as Result failure`() = runTest {
+        val boom = MeterError.EmptyTrace("boom")
+        val failingMeter = Meter { _ -> Result.failure(boom) }
+        val composite = CompositeMeter(
+            meterId = "composite",
+            tolerance = alwaysPass,
+            children = listOf(
+                WeightedMeter(meter = failingMeter, weight = 1.0, required = true),
+            ),
+        )
+        val result = composite.measure(anyTrace())
+
+        assertTrue(result.isFailure)
+        assertEquals(boom, result.exceptionOrNull())
+    }
+
+    @Test
+    fun `non-required child failure is skipped`() = runTest {
+        val failingMeter = Meter { _ -> Result.failure(MeterError.EmptyTrace("optional")) }
+        val composite = CompositeMeter(
+            meterId = "composite",
+            tolerance = alwaysPass,
+            children = listOf(
+                WeightedMeter(meter = failingMeter, weight = 1.0, required = false),
+                WeightedMeter(meter = fixedMeter("b", 0.6), weight = 1.0),
+            ),
+        )
+        val reading = composite.measure(anyTrace()).getOrThrow()
+
+        assertEquals(0.6, reading.score)
+    }
+
+    @Test
+    fun `all children failing returns NoReadings`() = runTest {
+        val failingMeter = Meter { _ -> Result.failure(MeterError.EmptyTrace("x")) }
+        val composite = CompositeMeter(
+            meterId = "composite",
+            tolerance = alwaysPass,
+            children = listOf(WeightedMeter(meter = failingMeter, weight = 1.0, required = false)),
+        )
+        val result = composite.measure(anyTrace())
+
+        assertTrue(result.isFailure)
+        assertIs<MeterError.NoReadings>(result.exceptionOrNull())
+    }
+
+    // region — fixtures
+
+    private fun anyTrace() = Trace(
+        id = "t",
+        runId = "r",
+        arcId = "a",
+        createdAt = 0L,
+        events = listOf(
+            link.socket.ampere.eval.trace.TraceEvent(
+                index = 0,
+                timestamp = 1L,
+                type = "Anything",
+                payload = kotlinx.serialization.json.buildJsonObject {},
+            ),
+        ),
+    )
+
+    private fun fixedMeter(id: String, score: Double): Meter {
+        val reading = Reading(score = score, passed = score >= 0.5, meterId = id)
+        return Meter { _ -> Result.success(reading) }
+    }
+
+    // endregion
+}