Merge pull request #48 from peopleworks/verdict-that-one-source-decides

Reconcile the verdict with the measurement (#32)

Author: peopleworks

Docs/CALIBRATION.md

@@ -73,6 +73,14 @@ Across these groups the median score runs from 7.2 (**es-wikipedia**) down to 4.
 | 95 | 0 / 90 | 0% | 0% – 4.1% |
 | 100 | 0 / 90 | 0% | 0% – 4.1% |
 
+## What the product does with this number
+
+The tool speaks at **25/100** and nowhere else, taking the boundary from the table above rather than from anybody's judgement. Below it a document gets its score and the reason it gets nothing more: a low score is not evidence that a person wrote something, since a detector that detects nothing also returns a low score, and this project has deliberately never measured how much machine writing it catches. The boundary moves when this page moves — including upward if a larger corpus turns out to be less flattering.
+
+Above it there is **one** verdict, not a scale of them. This corpus can place a boundary and can say nothing whatever about how much further past it a score has travelled: no text known to be human came close to the upper reaches, and grading "moderate" against "strong" would need machine-written text, which the opening of this page argues against collecting. Interfaces do shade a high score more urgently than a low one, and those shades are a display convention — they are not on this page because nothing measured them.
+
+A language absent from the corpus entirely gets no verdict at all, whatever it scores. A language present but too thin to bound its own rate borrows this boundary and carries its own figure beside it, so the reader weighs the rate that was measured for the writing in front of them rather than the pooled one.
+
 ## Which rules misfire
 
 Every rule below fired on text no machine wrote, so each hit is a false positive by construction — there is no judgement call to make. This is the most immediately useful thing the exercise produces: it turns "our rules probably have false positives somewhere" into a ranked list.

src/SignsOfAI.Core/Model/AnalysisResult.cs

@@ -93,11 +93,7 @@ public sealed record AnalysisResult
     /// person goes through the interface's localiser or the report's own resources, both of which
     /// take their boundary from <see cref="VerdictBands"/> exactly as this does.
     /// </summary>
-    public string Verdict => VerdictBands.Emphasis(OverallScore) switch
-    {
-        VerdictEmphasis.High => "Strong signs of AI writing",
-        VerdictEmphasis.Elevated => "Moderate signs of AI writing",
-        VerdictEmphasis.Present => "Light signs of AI writing",
-        _ => "Reads mostly human",
-    };
+    public string Verdict => VerdictBands.Holds(OverallScore)
+        ? "Signs of AI writing"
+        : "No signs above the measured boundary";
 }

src/SignsOfAI.Core/Reporting/EvidenceReport.cs

@@ -499,10 +499,8 @@ private static void HowOftenWrong(StringBuilder sb, ReportText text, string? lan
 
     private static string Verdict(ReportText text, double score) => text.Get(score switch
     {
-        _ when VerdictBands.Emphasis(score) is VerdictEmphasis.High => ReportMessages.VerdictStrong,
-        _ when VerdictBands.Emphasis(score) is VerdictEmphasis.Elevated => ReportMessages.VerdictModerate,
-        _ when VerdictBands.Emphasis(score) is VerdictEmphasis.Present => ReportMessages.VerdictLight,
-        _ => ReportMessages.VerdictMinimal,
+        _ when VerdictBands.Holds(score) => ReportMessages.VerdictSigns,
+        _ => ReportMessages.VerdictNone,
     }).Text;
 
     private static string LanguageName(ReportText text, string language) => text.Get(

src/SignsOfAI.Core/Reporting/ReportMessages.cs

@@ -29,10 +29,27 @@ public static class ReportMessages
     public const string SectionObservations = "section.observations";
     public const string SectionErrorRate = "section.error-rate";
     public const string SectionUnreadable = "section.unreadable";
-    public const string VerdictStrong = "verdict.strong";
-    public const string VerdictModerate = "verdict.moderate";
-    public const string VerdictLight = "verdict.light";
-    public const string VerdictMinimal = "verdict.minimal";
+    /// <summary>
+    /// Above the measured boundary. One wording, not three.
+    ///
+    /// "Strong", "Moderate" and "Light" read as three measured degrees, and they never were: the
+    /// corpus locates the boundary and says nothing whatever about 45 or 70, since no text known to
+    /// be human came within twenty points of either. Separating degrees would need machine-written
+    /// text, and Docs/CALIBRATION.md argues against ever collecting it. Printing the three words
+    /// with a footnote admitting they are unmeasured does not fix that — the footnote is read once
+    /// and the heading is read every time.
+    /// </summary>
+    public const string VerdictSigns = "verdict.signs";
+
+    /// <summary>
+    /// Below it. A statement about this tool, never about the writer.
+    ///
+    /// The report used to say "Reads mostly human" and the interface "Minimal signs of AI writing" —
+    /// the same state, two claims, and the first one is not ours to make. A detector that detects
+    /// nothing also returns a low score, and this project has deliberately never measured how much
+    /// machine writing it catches, so a low score is evidence about the boundary and nothing else.
+    /// </summary>
+    public const string VerdictNone = "verdict.none";
     public const string AnalysisScoreWithVerdict = "analysis.score.with-verdict";
     public const string AnalysisScoreWithoutVerdict = "analysis.score.without-verdict";
     public const string AnalysisNoVerdict = "analysis.no-verdict";
@@ -107,10 +124,8 @@ public static class ReportMessages
         [SectionObservations] = 0,
         [SectionErrorRate] = 0,
         [SectionUnreadable] = 0,
-        [VerdictStrong] = 0,
-        [VerdictModerate] = 0,
-        [VerdictLight] = 0,
-        [VerdictMinimal] = 0,
+        [VerdictSigns] = 0,
+        [VerdictNone] = 0,
         [AnalysisScoreWithVerdict] = 2,
         [AnalysisScoreWithoutVerdict] = 1,
         [AnalysisNoVerdict] = 0,
@@ -195,10 +210,8 @@ public static class ReportMessages
         [SectionObservations] = "Found, but at a rate people write at",
         [SectionErrorRate] = "How often this is wrong",
         [SectionUnreadable] = "Could not be read",
-        [VerdictStrong] = "Strong signs of AI writing",
-        [VerdictModerate] = "Moderate signs of AI writing",
-        [VerdictLight] = "Light signs of AI writing",
-        [VerdictMinimal] = "Reads mostly human",
+        [VerdictSigns] = "Signs of AI writing",
+        [VerdictNone] = "No signs above the measured boundary",
         [AnalysisScoreWithVerdict] = "**{0}/100 — {1}**",
         [AnalysisScoreWithoutVerdict] = "**{0}/100**",
         [AnalysisNoVerdict] = "*Below the threshold this build can support, so no verdict is given. A low score is not evidence that a person wrote this.*",
@@ -264,7 +277,7 @@ public static class ReportMessages
         FallbackMarker, FallbackSummary, FallbackLanguage, DefaultTitle,
         SectionAnalysis, SectionCheckable, SectionCharacters, SectionCitations, SectionSignals,
         SectionObservations, SectionErrorRate, SectionUnreadable,
-        VerdictStrong, VerdictModerate, VerdictLight, VerdictMinimal,
+        VerdictSigns, VerdictNone,
         AnalysisNoVerdict,
         LanguageEnglish, LanguageSpanish, LanguageOther,
         CaveatUncalibrated, CaveatAggregateNoThreshold, CaveatLanguageUnmeasured,

src/SignsOfAI.Core/Reporting/report.en.json

@@ -18,10 +18,8 @@
     "section.observations": { "text": "Found, but at a rate people write at" },
     "section.error-rate": { "text": "How often this is wrong" },
     "section.unreadable": { "text": "Could not be read" },
-    "verdict.strong": { "text": "Strong signs of AI writing" },
-    "verdict.moderate": { "text": "Moderate signs of AI writing" },
-    "verdict.light": { "text": "Light signs of AI writing" },
-    "verdict.minimal": { "text": "Reads mostly human" },
+    "verdict.signs": { "text": "Signs of AI writing" },
+    "verdict.none": { "text": "No signs above the measured boundary" },
     "analysis.score.with-verdict": { "text": "**{0}/100 — {1}**" },
     "analysis.score.without-verdict": { "text": "**{0}/100**" },
     "analysis.no-verdict": { "text": "*Below the threshold this build can support, so no verdict is given. A low score is not evidence that a person wrote this.*" },

src/SignsOfAI.Core/Reporting/report.es.json

@@ -54,21 +54,13 @@
       "text": "No se pudieron leer",
       "sourceHash": "51a02791dcb61eb8846a00d3f0263a9da516b30b6a92a3b08e34d65af4cef259"
     },
-    "verdict.strong": {
-      "text": "Señales fuertes de escritura con IA",
-      "sourceHash": "ee17bc5f7ea18e88a5a4a535d8a7d8f86fbaf3559c0b1927c71eb299ca138eef"
+    "verdict.signs": {
+      "text": "Señales de escritura con IA",
+      "sourceHash": "a03ec88d694c5dee62e6b04c9b46b89cdcb6e5d48b2971dba539fe7a5fbef0fc"
     },
-    "verdict.moderate": {
-      "text": "Señales moderadas de escritura con IA",
-      "sourceHash": "78304806a705ef996e3115ba84e0256f7838debf03afccb023e91d299d628a5b"
-    },
-    "verdict.light": {
-      "text": "Señales leves de escritura con IA",
-      "sourceHash": "99844fce9fa8df4247389342a41e53d0d231b950cc700f2458207dfef79d5d2e"
-    },
-    "verdict.minimal": {
-      "text": "Parece escrito mayormente por una persona",
-      "sourceHash": "4dd5c224203bf31595b61831cbd46ec9f2da352ce359b92ce46262e86cda64ce"
+    "verdict.none": {
+      "text": "Sin señales por encima del umbral medido",
+      "sourceHash": "dfdd3c06acb001454e7ec45ae2ffce857964a2531bb41686fc9f61c6abc16003"
     },
     "analysis.no-verdict": {
       "text": "*Por debajo del umbral que esta compilación puede respaldar, no se emite ningún veredicto. Una puntuación baja no demuestra que una persona haya escrito este texto.*",

src/SignsOfAI.UI/Services/Loc.cs

@@ -249,12 +249,10 @@ private async Task LogAsync(string message)
     /// The one-line verdict for an overall score, in the interface's language. The boundary comes
     /// from <see cref="VerdictBands"/>; this only chooses the words for it.
     /// </summary>
-    public string Verdict(double score) => VerdictBands.Emphasis(score) switch
+    public string Verdict(double score) => VerdictBands.Holds(score) switch
     {
-        VerdictEmphasis.High => this["verdict.strong"],
-        VerdictEmphasis.Elevated => this["verdict.moderate"],
-        VerdictEmphasis.Present => this["verdict.light"],
-        _ => this["verdict.minimal"],
+        true => this["verdict.signs"],
+        false => this["verdict.none"],
     };
 
     /// <summary>How to name the language the analyzer settled on. The engine only knows EN and ES.</summary>

src/SignsOfAI.UI/wwwroot/i18n/en.json

@@ -46,10 +46,8 @@
   "sev.low": "Low",
   "sev.medium": "Medium",
   "sev.high": "High",
-  "verdict.strong": "Strong signs of AI writing",
-  "verdict.moderate": "Moderate signs of AI writing",
-  "verdict.light": "Light signs of AI writing",
-  "verdict.minimal": "Minimal signs of AI writing",
+  "verdict.signs": "Signs of AI writing",
+  "verdict.none": "No signs above the measured boundary",
   "home.pagetitle": "Signs of AI Writing — detect & de-AI-ify your text",
   "home.tagline": "Paste, upload, or just start typing. We flag the tells of AI writing — overused words, rhetorical crutches, robotic rhythm — <strong>live, as you write</strong>, and show you how to fix each one.",
   "home.privacy": "Analysis runs entirely in your browser. Your text never leaves your device.",

src/SignsOfAI.UI/wwwroot/i18n/es.json

@@ -46,10 +46,8 @@
   "sev.low": "Baja",
   "sev.medium": "Media",
   "sev.high": "Alta",
-  "verdict.strong": "Señales fuertes de escritura IA",
-  "verdict.moderate": "Señales moderadas de escritura IA",
-  "verdict.light": "Señales leves de escritura IA",
-  "verdict.minimal": "Señales mínimas de escritura IA",
+  "verdict.signs": "Señales de escritura con IA",
+  "verdict.none": "Sin señales por encima del umbral medido",
   "home.pagetitle": "Señales de escritura IA — detecta y humaniza tu texto",
   "home.tagline": "Pega, sube un archivo o simplemente empieza a escribir. Marcamos las señales de escritura IA —palabras sobreusadas, muletillas retóricas, ritmo robótico— <strong>en vivo, mientras escribes</strong>, y te mostramos cómo corregir cada una.",
   "home.privacy": "El análisis se ejecuta por completo en tu navegador. Tu texto nunca sale de tu dispositivo.",

tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs

@@ -272,7 +272,7 @@ public void An_unmeasured_language_gets_no_aggregate_threshold_or_verdict()
 
         Assert.Contains("never been measured for language code pt", report);
         Assert.Contains("aggregate result from other languages is not a substitute", report);
-        Assert.DoesNotContain("Strong signs of AI writing", report);
+        Assert.DoesNotContain("Signs of AI writing", report);
 
         if (PublishedCalibration.Current is { } calibration)
         {
@@ -328,11 +328,13 @@ public void A_report_language_without_the_mandatory_core_is_rejected()
     [Fact]
     public void Withholds_the_verdict_below_the_threshold_it_can_support()
     {
-        // "Reads mostly human" printed above "treat the score as saying nothing" is a page arguing
-        // with itself, and the reader keeps whichever half suits them.
+        // A verdict printed above "treat the score as saying nothing" is a page arguing with itself,
+        // and the reader keeps whichever half suits them. Below the boundary the report prints the
+        // score and the reason, and no verdict line of either kind.
         var report = Report();
 
-        Assert.DoesNotContain("Reads mostly human", report);
+        Assert.DoesNotContain("No signs above the measured boundary", report);
+        Assert.DoesNotContain("Signs of AI writing", report);
         Assert.Contains("A low score is not evidence that a person wrote this", report);
     }