diff --git a/Docs/CALIBRATION.md b/Docs/CALIBRATION.md index f44e709..9f76d68 100644 --- a/Docs/CALIBRATION.md +++ b/Docs/CALIBRATION.md @@ -10,6 +10,7 @@ It is **not an accuracy figure**. Accuracy needs machine-written text to measure - **Corpus** `signsofai-human-baseline`, fingerprint `123fa5b9ebca3f29` - **Texts** 90 (280,221 words) +- **Lengths measured** 662 – 9,328 words (median 2,772) - **Engine** SignsOfAI.Core 0.4.0 - **Run** 2026-08-24 - **Target false-positive rate** 5% @@ -20,6 +21,8 @@ Every text here was published before generative models could have written it. Th **At a threshold of 25/100, this tool flags at most 5% of writing known to be human** — 0 of 90 texts in this corpus, an observed 0% with a 95% interval of 0% – 4.1%. +**It covers documents of 662 words and up, because that is what was measured.** Nothing shorter was: the corpus has no text below that length, so the boundary below is not supported there and the tool withholds its verdict rather than extrapolating. That is a statement about coverage, not about where the tool breaks — though the direction of the length effect *has* been measured, and it goes the wrong way: the same documents flagged 0 of 32 whole and 6 of 32 as 400-word excerpts of themselves (`Docs/PARAPHRASE.md`, section *Length*). Lowering this floor means measuring short writing people actually composed at that length, not slicing long documents into pieces. + Read the interval, not the percentage. On a small corpus an observed rate is compatible with a much wider range, and the recommendation below is made from the **upper** end of that range rather than the flattering one — so it stays cautious while the corpus is thin and tightens on its own as it grows. ## By language diff --git a/Docs/Calibration/README.md b/Docs/Calibration/README.md index ba03b6a..80d4032 100644 --- a/Docs/Calibration/README.md +++ b/Docs/Calibration/README.md @@ -27,6 +27,23 @@ exist, which is a stronger guarantee than any classifier can offer about anythin No text is admitted on the grounds that it "reads human". That judgement is the thing being measured and cannot also be the thing doing the measuring. +## What the corpus does not cover, and what it costs + +Every text here is **662 words or longer** — that is the shortest one, and the Wikipedia fetcher skips +anything under 700 by design. The threshold is therefore supported over that range and nowhere else, +so since #59 the engine **withholds its verdict below 662 words** rather than extrapolating onto a +population it never sampled. + +That is not a small exclusion. It is most of how the tool is used: somebody pastes a paragraph. And +the direction of the error is known — the same documents flag 0 of 32 whole and 6 of 32 as 400-word +excerpts of themselves (`Docs/PARAPHRASE.md`, section *Length*), so short text drifts toward the +machine rather than merely getting noisier. + +**The most wanted contribution is therefore short complete texts published before 2022**: encyclopedia +stubs, short news pieces, abstracts — writing somebody *composed* at that length. A window cut out of +a longer document is not the same population and must not be used: it has the sentence rhythm of a +fragment, which is the very thing being measured. See issue #66. + ## The texts are not in this repository `Docs/Calibration/texts/` is git-ignored. Licences differ per source, the bulk would dwarf the code, diff --git a/src/SignsOfAI.Cli/Program.cs b/src/SignsOfAI.Cli/Program.cs index 745416f..767f0f4 100644 --- a/src/SignsOfAI.Cli/Program.cs +++ b/src/SignsOfAI.Cli/Program.cs @@ -364,10 +364,13 @@ static void PrintReport(string path, AnalysisResult r, int top, bool useColor) string Col(string s, int code) => useColor ? $"[{code}m{s}" : s; string Bold(string s) => useColor ? $"{s}" : s; - int scoreColor = VerdictBands.Emphasis(r.OverallScore) switch + // Green says "nothing here"; the terminal has no way to un-say it four lines later. A document + // outside what was measured gets grey, because the honest colour for a refusal is not a result. + int scoreColor = VerdictBands.Emphasis(r.OverallScore, r.Language, r.Statistics.WordCount) switch { VerdictEmphasis.High => 31, VerdictEmphasis.Elevated or VerdictEmphasis.Present => 33, + VerdictEmphasis.Unmeasured => 90, _ => 32, }; Console.WriteLine(); @@ -377,6 +380,11 @@ static void PrintReport(string path, AnalysisResult r, int top, bool useColor) Console.WriteLine($" words {r.Statistics.WordCount} · sentences {r.Statistics.SentenceCount} · " + $"burstiness {r.Statistics.Burstiness:0.00} · lexical diversity {r.Statistics.LexicalDiversity:0.00}"); + if (!VerdictBands.Measured(r.Statistics.WordCount)) + Console.WriteLine(Col( + $" The boundary was measured only on texts of {VerdictBands.MinimumWords:N0} words and " + + "longer, so no verdict is given here. The findings below are unaffected.", 90)); + var cats = r.CategoryScores.Where(c => c.FindingCount > 0).ToList(); if (cats.Count > 0) Console.WriteLine(" " + string.Join(" ", cats.Select(c => $"{c.Category} {c.FindingCount}"))); diff --git a/src/SignsOfAI.Core/Calibration/CalibrationModel.cs b/src/SignsOfAI.Core/Calibration/CalibrationModel.cs index 42b43c4..f809feb 100644 --- a/src/SignsOfAI.Core/Calibration/CalibrationModel.cs +++ b/src/SignsOfAI.Core/Calibration/CalibrationModel.cs @@ -77,6 +77,25 @@ public sealed record StratumCalibration public required int TotalWords { get; init; } + /// + /// The length of the shortest and longest text in this group, in words. + /// + /// Not decoration on the table. The threshold below is only supported over the lengths that were + /// actually measured, and this group's shortest text is where that support stops — the whole of + /// issue #59 is that a boundary fitted here was being spent on a pasted paragraph a quarter of + /// its length. + /// + public required int ShortestWords { get; init; } + + /// + public required int LongestWords { get; init; } + + /// + /// The median length, which is the honest middle of a range this skewed: the corpus runs from 712 + /// words to 9,772, and quoting the mean would put the centre where few of the texts actually are. + /// + public required double MedianWords { get; init; } + public required double MedianScore { get; init; } /// The score nine in ten of these human texts stay below. diff --git a/src/SignsOfAI.Core/Calibration/Calibrator.cs b/src/SignsOfAI.Core/Calibration/Calibrator.cs index 4726024..0dba072 100644 --- a/src/SignsOfAI.Core/Calibration/Calibrator.cs +++ b/src/SignsOfAI.Core/Calibration/Calibrator.cs @@ -72,6 +72,7 @@ public static StratumCalibration Measure( return new StratumCalibration { Name = name, Count = 0, TotalWords = 0, + ShortestWords = 0, LongestWords = 0, MedianWords = 0, MedianScore = 0, NinetiethScore = 0, HighestScore = 0, Thresholds = [], ThresholdForTarget = null, }; @@ -98,6 +99,9 @@ public static StratumCalibration Measure( Name = name, Count = samples.Count, TotalWords = samples.Sum(s => s.WordCount), + ShortestWords = samples.Min(s => s.WordCount), + LongestWords = samples.Max(s => s.WordCount), + MedianWords = Quantile([.. samples.Select(s => (double)s.WordCount).Order()], 0.50), MedianScore = Quantile(scores, 0.50), NinetiethScore = Quantile(scores, 0.90), HighestScore = scores[^1], diff --git a/src/SignsOfAI.Core/Calibration/PublishedCalibration.cs b/src/SignsOfAI.Core/Calibration/PublishedCalibration.cs index dc833f9..bd6ec08 100644 --- a/src/SignsOfAI.Core/Calibration/PublishedCalibration.cs +++ b/src/SignsOfAI.Core/Calibration/PublishedCalibration.cs @@ -50,6 +50,25 @@ public sealed record PublishedCalibration public double RateHigh { get; init; } + /// + /// The length of the shortest and longest text the threshold above was measured on, in words as + /// the analyzer counts them. + /// + /// Recorded because a bound measured on one population must not be spent on another, and length + /// is such a population: the shipped boundary was fitted on texts of 662 words and up, and was + /// being applied to a pasted paragraph. reads + /// and withholds the verdict below it; nothing reads + /// yet, and it is here so the range on the page is a range rather + /// than half of one. See issue #59. + /// + /// Null in snapshots written before this field existed. That case does not gate — see + /// for why it differs from the language rule. + /// + public int? ShortestWords { get; init; } + + /// + public int? LongestWords { get; init; } + /// /// The rules most often seen on human writing, worst first. Printed alongside a report's findings /// so a reader can see whether the evidence they are holding leans on a rule that is known to be diff --git a/src/SignsOfAI.Core/Calibration/published-calibration.json b/src/SignsOfAI.Core/Calibration/published-calibration.json index 73bdcd3..d87f4e3 100644 --- a/src/SignsOfAI.Core/Calibration/published-calibration.json +++ b/src/SignsOfAI.Core/Calibration/published-calibration.json @@ -1 +1 @@ -{"corpusId":"signsofai-human-baseline","texts":90,"measuredOn":"2026-08-24","engine":"0.4.0","recommendedThreshold":25,"flaggedAtThreshold":0,"rateLow":0,"rateHigh":0.04093562618368095,"noisiestRules":[{"ruleId":"stat.burstiness","textShare":0.2777777777777778},{"ruleId":"rhet.in-terms-of","textShare":0.1},{"ruleId":"rhet.not-only-but","textShare":0.08888888888888889},{"ruleId":"rhet.in-order-to","textShare":0.07777777777777778},{"ruleId":"lex.furthermore","textShare":0.07777777777777778},{"ruleId":"lex.robust","textShare":0.07777777777777778},{"ruleId":"lex.just","textShare":0.07777777777777778},{"ruleId":"lex.simply","textShare":0.07777777777777778}],"languages":[{"language":"en","texts":65,"recommendedThreshold":null,"bestBound":0.05580153215404492},{"language":"es","texts":25,"recommendedThreshold":null,"bestBound":0.13319225276039096}]} +{"corpusId":"signsofai-human-baseline","texts":90,"measuredOn":"2026-08-24","engine":"0.4.0","recommendedThreshold":25,"flaggedAtThreshold":0,"rateLow":0,"rateHigh":0.04093562618368095,"shortestWords":662,"longestWords":9328,"noisiestRules":[{"ruleId":"stat.burstiness","textShare":0.2777777777777778},{"ruleId":"rhet.in-terms-of","textShare":0.1},{"ruleId":"rhet.not-only-but","textShare":0.08888888888888889},{"ruleId":"rhet.in-order-to","textShare":0.07777777777777778},{"ruleId":"lex.furthermore","textShare":0.07777777777777778},{"ruleId":"lex.robust","textShare":0.07777777777777778},{"ruleId":"lex.just","textShare":0.07777777777777778},{"ruleId":"lex.simply","textShare":0.07777777777777778}],"languages":[{"language":"en","texts":65,"recommendedThreshold":null,"bestBound":0.05580153215404492},{"language":"es","texts":25,"recommendedThreshold":null,"bestBound":0.13319225276039096}]} diff --git a/src/SignsOfAI.Core/Model/AnalysisResult.cs b/src/SignsOfAI.Core/Model/AnalysisResult.cs index ccad11e..82a6379 100644 --- a/src/SignsOfAI.Core/Model/AnalysisResult.cs +++ b/src/SignsOfAI.Core/Model/AnalysisResult.cs @@ -85,6 +85,17 @@ public sealed record AnalysisResult /// public CitationReport Citations { get; init; } = CitationReport.Empty; + /// + /// Whether this build will say anything at all about this document — score, language and length + /// together. + /// + /// Derived here so that every surface asks the same question of the same three facts. The last + /// time each host decided for itself, one engine gave three answers about the same text; see + /// , whose whole existence is that failure. + /// + public bool HasVerdict => + VerdictBands.Holds(OverallScore, Language, Statistics.WordCount); + /// /// Human-readable one-line verdict derived from , in English. /// @@ -92,8 +103,15 @@ public sealed record AnalysisResult /// tool's payload — where a stable string is more use than a translated one. Anything shown to a /// person goes through the interface's localiser or the report's own resources, both of which /// take their boundary from exactly as this does. + /// + /// Four states, not two, and the order matters: the reasons this build cannot speak are checked + /// before the reading it would otherwise give. Collapsing "we did not measure anything this + /// short" into "no signs above the measured boundary" would turn a refusal into a finding, which + /// is the failure this whole property exists to avoid. /// - public string Verdict => VerdictBands.Holds(OverallScore) - ? "Signs of AI writing" + public string Verdict => + !VerdictBands.Measured(Statistics.WordCount) ? "No verdict: below the measured length" + : !VerdictBands.Measured(Language) ? "No verdict: language not measured" + : VerdictBands.Holds(OverallScore) ? "Signs of AI writing" : "No signs above the measured boundary"; } diff --git a/src/SignsOfAI.Core/Model/VerdictBands.cs b/src/SignsOfAI.Core/Model/VerdictBands.cs index 09fd5fa..15d160d 100644 --- a/src/SignsOfAI.Core/Model/VerdictBands.cs +++ b/src/SignsOfAI.Core/Model/VerdictBands.cs @@ -30,6 +30,52 @@ public static class VerdictBands /// public static bool Holds(double score) => Threshold is { } threshold && score >= threshold; + /// + /// The shortest text the boundary was ever measured on, or null when the embedded calibration + /// predates this field. + /// + /// This is a statement about **coverage, not about reliability**. It does not claim the tool + /// breaks below this length; it says nothing was measured there, which is a different and much + /// weaker claim — and the only one the corpus can support. The 25/100 boundary was fitted on 90 + /// texts whose shortest is 662 words and whose median is 2,772, and it was being applied to a + /// pasted paragraph with nothing on the page to say so. See issue #59. + /// + /// The number comes from the analyzer's own word count, not from a naive split on whitespace — + /// the two disagree by about 7% on this corpus. Deriving the floor with one counter and comparing + /// against another would silently rescale the gate, which is the same trap + /// documents for its rates. + /// + /// It is an observation rather than a fitted parameter, and that is the whole point of choosing + /// it: no grid of lengths, no windows cut out of longer documents, no subset selected to make a + /// number come out. Every earlier attempt at this measured a rate against synthetic short text + /// and inherited the problem it was fixing — a 400-word window sliced out of a paper is not a + /// paragraph somebody *composed* at 400 words, and a floor fitted on the first does not describe + /// the second. + /// + /// The way to lower it is to measure shorter writing: complete texts, published before 2022, at + /// the lengths people actually paste. That is issue #66, and every one of them extends this + /// downward by evidence rather than by decision. + /// + public static int? MinimumWords => PublishedCalibration.Current?.ShortestWords; + + /// + /// Whether a document of this length sits inside what the boundary was measured on. + /// + /// **There is deliberately no ceiling.** The asymmetry is measured, not assumed: shortening a + /// text moves its score toward the machine — 0 of 32 documents flagged whole, 6 of the same 32 + /// flagged as 400-word excerpts of themselves (`Docs/PARAPHRASE.md`, section *Length*) — while + /// nothing suggests a thesis longer than the corpus is at risk. Silencing the long end too would + /// withhold a verdict for a symmetry nobody has evidence for. + /// + /// A null minimum does **not** gate. That is deliberately unlike the language condition, where + /// absence from the corpus is a positive fact each snapshot records. Here null means the snapshot + /// is older than the field, not that the corpus had no lengths; going silent on it would stop + /// every fork carrying a 0.4.0 snapshot from speaking at all, which is a change driven by a + /// missing field rather than by evidence. + /// + public static bool Measured(int wordCount) => + MinimumWords is not { } floor || wordCount >= floor; + /// /// The same question for a document in a named language, which is stricter and has to be. /// @@ -48,6 +94,16 @@ public static class VerdictBands /// public static bool Holds(double score, string? language) => Holds(score) && Measured(language); + /// + /// The full question, and the one every surface should ask: score, language and length together. + /// + /// The three conditions are the same rule applied three times — *a bound measured on one + /// population must not be spent on another* — and they are answered in one place because the + /// last time this was decided in eight, one engine gave three answers about the same text. + /// + public static bool Holds(double score, string? language, int wordCount) => + Holds(score, language) && Measured(wordCount); + /// Whether the corpus contains this language at all, however thinly. public static bool Measured(string? language) => PublishedCalibration.Current?.For(language) is not null; @@ -70,6 +126,17 @@ public static bool Measured(string? language) => >= 45 => VerdictEmphasis.Elevated, _ => VerdictEmphasis.Present, }; + + /// + /// The same, for a document whose language and length are known. + /// + /// Colour is part of the verdict whatever the design system pretends. A page that withholds a + /// verdict in words and paints the score red anyway has given the verdict — louder, and without + /// the sentence that qualifies it. + /// + public static VerdictEmphasis Emphasis(double score, string? language, int wordCount) => + !Measured(wordCount) || !Measured(language) ? VerdictEmphasis.Unmeasured + : Emphasis(score); } /// @@ -87,4 +154,16 @@ public enum VerdictEmphasis Elevated, High, + + /// + /// Outside what was measured — a language the corpus never contained, or a document shorter than + /// anything the boundary was fitted on. + /// + /// Separate from , and the separation is the point. Both withhold a verdict, but + /// is a reading — the tool looked and found little, and a reassuring colour is + /// honest for it. This one is a refusal to read, and painting a 72/100 passage green because the + /// verdict was withheld would state the opposite of what was withheld, in the loudest channel on + /// the page. See issue #59. + /// + Unmeasured, } diff --git a/src/SignsOfAI.Core/Reporting/EvidenceReport.cs b/src/SignsOfAI.Core/Reporting/EvidenceReport.cs index a751521..fe4ff49 100644 --- a/src/SignsOfAI.Core/Reporting/EvidenceReport.cs +++ b/src/SignsOfAI.Core/Reporting/EvidenceReport.cs @@ -81,7 +81,15 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options = sb.AppendLine(); if (!VerdictHolds(result)) { - AppendBlock(sb, text, ReportMessages.AnalysisNoVerdict); + // Which of the reasons, because they are not the same statement. "Below the threshold" + // is a reading — the tool looked and found little. "Shorter than anything measured" is + // a refusal to read at all, and a reader who is told the first when the second is true + // will take away a reassurance nobody offered. + AppendBlock(sb, text, + VerdictBands.Measured(result.Statistics.WordCount) + ? ReportMessages.AnalysisNoVerdict + : ReportMessages.AnalysisNoVerdictShort, + Num(result.Statistics.WordCount), Num(VerdictBands.MinimumWords ?? 0)); sb.AppendLine(); } @@ -491,8 +499,12 @@ public static string FolderToHtml( /// it decides when the tool speaks, and it is published, measured and printed on the page beside /// the language's own figure. See issue #32. /// - private static bool VerdictHolds(AnalysisResult result) => - VerdictBands.Holds(result.OverallScore, result.Language); + /// + /// Length joined score and language in #59, and it arrives through + /// rather than being asked here, so that the CLI's JSON, the MCP payload, the interface and this + /// page cannot drift into disagreeing about the same document. + /// + private static bool VerdictHolds(AnalysisResult result) => result.HasVerdict; /// /// The sentence that has to appear on every report. Written from the embedded calibration so it diff --git a/src/SignsOfAI.Core/Reporting/ReportMessages.cs b/src/SignsOfAI.Core/Reporting/ReportMessages.cs index f7bc97d..e7d345e 100644 --- a/src/SignsOfAI.Core/Reporting/ReportMessages.cs +++ b/src/SignsOfAI.Core/Reporting/ReportMessages.cs @@ -53,6 +53,13 @@ public static class ReportMessages public const string AnalysisScoreWithVerdict = "analysis.score.with-verdict"; public const string AnalysisScoreWithoutVerdict = "analysis.score.without-verdict"; public const string AnalysisNoVerdict = "analysis.no-verdict"; + + /// + /// The other reason a verdict is withheld: the document is shorter than anything the + /// boundary was measured on. A different sentence from on + /// purpose — that one reports a reading, this one reports a refusal to give one. + /// + public const string AnalysisNoVerdictShort = "analysis.no-verdict-short"; public const string AnalysisFactsCitationOne = "analysis.facts.citation.one"; public const string AnalysisFactsCitationOther = "analysis.facts.citation.other"; public const string AnalysisFactsArtifactOne = "analysis.facts.artifact.one"; @@ -150,6 +157,7 @@ public static class ReportMessages [AnalysisScoreWithVerdict] = 2, [AnalysisScoreWithoutVerdict] = 1, [AnalysisNoVerdict] = 0, + [AnalysisNoVerdictShort] = 2, // {0} words here, {1} shortest measured [AnalysisFactsCitationOne] = 1, [AnalysisFactsCitationOther] = 1, [AnalysisFactsArtifactOne] = 1, @@ -239,6 +247,7 @@ public static class ReportMessages [AnalysisScoreWithVerdict] = "**{0}/100 — {1}**", [AnalysisScoreWithoutVerdict] = "**{0}/100**", [AnalysisNoVerdict] = "*Below the threshold this build can support, so no verdict is given. A low score is not evidence that a person wrote this.*", + [AnalysisNoVerdictShort] = "*No verdict is given: this document is {0} words, and the boundary above was measured only on texts of {1} words and longer. Nothing this short was measured, so the score below stands on its own — it is neither evidence that a machine wrote this nor evidence that a person did.*", [AnalysisFactsCitationOne] = "**Checkable facts found: {0} source contradiction. These did not move the score.**", [AnalysisFactsCitationOther] = "**Checkable facts found: {0} source contradictions. These did not move the score.**", [AnalysisFactsArtifactOne] = "**Checkable facts found: {0} unusual character. These did not move the score.**", @@ -305,7 +314,7 @@ public static class ReportMessages SectionAnalysis, SectionCheckable, SectionCharacters, SectionCitations, SectionSignals, SectionObservations, SectionErrorRate, SectionUnreadable, VerdictSigns, VerdictNone, - AnalysisNoVerdict, + AnalysisNoVerdict, AnalysisNoVerdictShort, LanguageEnglish, LanguageSpanish, LanguageOther, CaveatUncalibrated, CaveatAggregateNoThreshold, CaveatLanguageUnmeasured, CaveatLanguageNoThreshold, CaveatLanguageMeasured, CaveatAggregateMeasured, diff --git a/src/SignsOfAI.Core/Reporting/report.en.json b/src/SignsOfAI.Core/Reporting/report.en.json index 6a177a6..0bacb5e 100644 --- a/src/SignsOfAI.Core/Reporting/report.en.json +++ b/src/SignsOfAI.Core/Reporting/report.en.json @@ -1,81 +1,236 @@ { "language": "en", - "translators": ["SignsOfAI maintainers"], + "translators": [ + "SignsOfAI maintainers" + ], "messages": { - "fallback.marker": { "text": "This block has not been translated yet; it is shown in English." }, - "fallback.summary": { "text": "This report contains {0} block(s) not yet translated. Each is marked and shown in English." }, - "fallback.language": { "text": "This report is not available in {0}, so the whole of it is shown in English. Nothing has been withheld or shortened, but a reader who cannot read English cannot read the part that limits the score, and that part is the point of the page." }, - "analysis.no-rule-pack": { "text": "> **There is no rule pack for {0} yet, so this text was examined with the English one.** Treat the score as saying nothing at all: the tells this tool knows are English ones, and few of them can fire on writing in another language — so a low number here means nothing was looked for, not that nothing was found. Rule packs are JSON files anyone can contribute." }, - "default.title": { "text": "Writing analysis report" }, - "meta.document": { "text": "**Document:** {0}" }, - "meta.generated": { "text": "**Generated:** {0} · **Engine:** SignsOfAI {1}" }, - "meta.folder": { "text": "**Folder:** {0}" }, - "section.analysis": { "text": "What the analysis says" }, - "section.checkable": { "text": "Checkable facts" }, - "section.characters": { "text": "Characters found in the file" }, - "section.citations": { "text": "What the document says about its own sources" }, - "section.signals": { "text": "Signals counted" }, - "section.observations": { "text": "Found, but at a rate people write at" }, - "section.error-rate": { "text": "How often this is wrong" }, - "section.unreadable": { "text": "Could not be read" }, - "verdict.signs": { "text": "Signs of AI writing" }, - "verdict.none": { "text": "No signs above the measured boundary" }, - "analysis.score.with-verdict": { "text": "**{0}/100 — {1}**" }, - "analysis.score.without-verdict": { "text": "**{0}/100**" }, - "analysis.no-verdict": { "text": "*Below the threshold this build can support, so no verdict is given. A low score is not evidence that a person wrote this.*" }, - "analysis.facts.citation.one": { "text": "**Checkable facts found: {0} source contradiction. These did not move the score.**" }, - "analysis.facts.citation.other": { "text": "**Checkable facts found: {0} source contradictions. These did not move the score.**" }, - "analysis.facts.artifact.one": { "text": "**Checkable facts found: {0} unusual character. These did not move the score.**" }, - "analysis.facts.artifact.other": { "text": "**Checkable facts found: {0} unusual characters. These did not move the score.**" }, - "analysis.facts.both.one-one": { "text": "**Checkable facts found: {0} source contradiction, {1} unusual character. These did not move the score.**" }, - "analysis.facts.both.one-other": { "text": "**Checkable facts found: {0} source contradiction, {1} unusual characters. These did not move the score.**" }, - "analysis.facts.both.other-one": { "text": "**Checkable facts found: {0} source contradictions, {1} unusual character. These did not move the score.**" }, - "analysis.facts.both.other-other": { "text": "**Checkable facts found: {0} source contradictions, {1} unusual characters. These did not move the score.**" }, - "analysis.counts.one": { "text": "- {0} signal counted" }, - "analysis.counts.other": { "text": "- {0} signals counted" }, - "analysis.counts-with-observations.one": { "text": "- {0} signal counted, plus {1} found at a rate people write at, which count for nothing" }, - "analysis.counts-with-observations.other": { "text": "- {0} signals counted, plus {1} found at a rate people write at, which count for nothing" }, - "analysis.language-stats": { "text": "- Analysed as {0} · {1} words · {2} sentences · sentence-length variability {3}" }, - "language.en": { "text": "English" }, - "language.es": { "text": "Spanish" }, - "language.other": { "text": "language code {0}" }, - "caveat.uncalibrated": { "text": "> **This build has not been calibrated.** No false-positive rate has been measured for it, so the score above should not be used to support a decision about a person." }, - "caveat.aggregate-no-threshold": { "text": "> **No threshold is supported yet.** This build was measured against {0} texts, too few to bound its false-positive rate, so no score on this page should be used to support a decision about a person." }, - "caveat.language-unmeasured": { "text": "> **This build has never been measured for {0}.** It has no false-positive rate or supported threshold for writing in this language, and the aggregate result from other languages is not a substitute. No score on this page should be used to support a decision about a person." }, - "caveat.language-no-threshold": { "text": "> **No threshold is supported for this language yet.** The corpus holds {0} texts in it — too few to bound how often this build is wrong about writing in it, so no score on this page should be used to support a decision about a person. The best bound these texts support is {1}, and the overall figure is not a substitute for it." }, - "caveat.language-measured": { "text": "> **A score is not proof.** On {0} texts in this language, published before generative models existed, this build's false-positive rate at a threshold of {1}/100 was under {2} — the upper end of a 95% interval, not a guarantee, and measured on published articles rather than student work. Below that threshold, treat the score as saying nothing." }, - "caveat.aggregate-measured": { "text": "> **A score is not proof.** On {0} texts published before generative models existed, this build's false-positive rate at a threshold of {1}/100 was under {2} — the upper end of a 95% interval, not a guarantee, and measured on published articles rather than student work. Below that threshold, treat the score as saying nothing." }, - "checkable.intro": { "text": "These are not judgements about the writing and they did not move the score. Each is either present in the file or it is not." }, - "characters.explanation": { "text": "Several of these have ordinary explanations — word processors insert soft hyphens and unusual spaces on their own, and any copy-paste can carry them. Invisible characters and letters borrowed from another alphabet are harder to arrive at by accident, though pasting text can do it. This table says what is in the file, not how it got there." }, - "characters.ordered": { "text": "Listed with the characters hardest to arrive at by accident first, then in the order they appear in the file." }, - "characters.table-header": { "text": "| Character | Codepoint | Line | Column |" }, - "common.more-rows": { "text": "… and {0} more." }, - "citations.issues-note": { "text": "> None of this needed the internet: the document disagrees with itself. It is a question to ask, not a conclusion — the answer is usually one sentence." }, - "citations.no-issues-note": { "text": "> Nothing here is a finding. It describes what could and could not be checked." }, - "signals.none": { "text": "None." }, - "signals.ordered": { "text": "Ordered by how much weight each one carries, heaviest first, rather than by where it appears in the text." }, - "signals.more": { "text": "… and {0} more, none of them carrying more weight than what is shown above." }, - "observations.intro": { "text": "Measured against writing published before generative models existed. Shown because they are real, and counted for nothing because they are ordinary." }, - "observations.row.one": { "text": "- {0} — {1} occurrence" }, - "observations.row.other": { "text": "- {0} — {1} occurrences" }, - "privacy.document": { "text": "*This report was produced on the device that ran the analysis and contains material from the document it describes. It is yours to keep or to send; nothing here was uploaded anywhere.*" }, - "folder.summary.one": { "text": "{0} file scanned." }, - "folder.summary.other": { "text": "{0} files scanned." }, - "folder.summary-unreadable.one": { "text": "{0} file scanned, {1} unreadable." }, - "folder.summary-unreadable.other": { "text": "{0} files scanned, {1} unreadable." }, - "folder.reading-order": { "text": "> **This is a reading order, not a ranking.** A higher score means look sooner, and nothing more. Nothing on this page establishes that anyone did anything." }, - "folder.table-header": { "text": "| File | Score | Signals | Words |" }, - "folder.unreadable-row": { "text": "- {0} — {1}" }, - "privacy.folder": { "text": "*Produced on the device that scanned the folder. It names your students' files, so treat it as you would the coursework itself; nothing here was uploaded anywhere.*" }, - "how.uncalibrated": { "text": "This build ships no calibration, so nothing is known about how often it is wrong. That is itself the most important thing on this page." }, - "how.language-unmeasured": { "text": "This build has never been measured on writing in {0}. No language-specific false-positive rate or threshold exists, and the aggregate result from other languages is not a substitute." }, - "how.language-no-threshold": { "text": "Measured against **{0} texts in this language**, published before generative models existed, on {2} with engine {3}. That sample is too small to support a threshold; the best upper bound it supports is **{1}**, and the overall figure is not a substitute." }, - "how.language-measured": { "text": "Measured against **{0} texts in this language**, published before generative models existed, on {3} with engine {4}. At **{1}/100**, the upper end of the measured 95% false-positive interval was **{2}** — an interval, not a guarantee." }, - "how.aggregate-intro": { "text": "Measured against **{0} texts published before generative models existed**, so their authorship rests on their dates rather than on anybody's judgement. Measured on {1} with engine {2}." }, - "how.aggregate-threshold": { "text": "At **{0}/100**, {1} of those {2} were flagged — an observed {3}, with a 95% interval of {4} – {5}." }, - "how.read-interval": { "text": "Read the interval, not the observed rate. {0} out of {1} is not a false-positive rate you can round down." }, - "how.noisy-intro": { "text": "The rules seen most often on that human writing, worst first — if the evidence above leans on one of these, weigh it accordingly:" }, - "how.noisy-rule": { "text": "- `{0}` — {1} of human texts" }, - "how.limitation": { "text": "What this does **not** tell you: how much machine-written text it catches. That is the other half of the picture and it is deliberately not measured here, because any collection of machine-written text samples whichever models were convenient that month. A tool that flags nothing has a perfect false-positive rate." } + "fallback.marker": { + "text": "This block has not been translated yet; it is shown in English." + }, + "fallback.summary": { + "text": "This report contains {0} block(s) not yet translated. Each is marked and shown in English." + }, + "fallback.language": { + "text": "This report is not available in {0}, so the whole of it is shown in English. Nothing has been withheld or shortened, but a reader who cannot read English cannot read the part that limits the score, and that part is the point of the page." + }, + "analysis.no-rule-pack": { + "text": "> **There is no rule pack for {0} yet, so this text was examined with the English one.** Treat the score as saying nothing at all: the tells this tool knows are English ones, and few of them can fire on writing in another language — so a low number here means nothing was looked for, not that nothing was found. Rule packs are JSON files anyone can contribute." + }, + "default.title": { + "text": "Writing analysis report" + }, + "meta.document": { + "text": "**Document:** {0}" + }, + "meta.generated": { + "text": "**Generated:** {0} · **Engine:** SignsOfAI {1}" + }, + "meta.folder": { + "text": "**Folder:** {0}" + }, + "section.analysis": { + "text": "What the analysis says" + }, + "section.checkable": { + "text": "Checkable facts" + }, + "section.characters": { + "text": "Characters found in the file" + }, + "section.citations": { + "text": "What the document says about its own sources" + }, + "section.signals": { + "text": "Signals counted" + }, + "section.observations": { + "text": "Found, but at a rate people write at" + }, + "section.error-rate": { + "text": "How often this is wrong" + }, + "section.unreadable": { + "text": "Could not be read" + }, + "verdict.signs": { + "text": "Signs of AI writing" + }, + "verdict.none": { + "text": "No signs above the measured boundary" + }, + "analysis.score.with-verdict": { + "text": "**{0}/100 — {1}**" + }, + "analysis.score.without-verdict": { + "text": "**{0}/100**" + }, + "analysis.no-verdict": { + "text": "*Below the threshold this build can support, so no verdict is given. A low score is not evidence that a person wrote this.*" + }, + "analysis.no-verdict-short": { + "text": "*No verdict is given: this document is {0} words, and the boundary above was measured only on texts of {1} words and longer. Nothing this short was measured, so the score below stands on its own — it is neither evidence that a machine wrote this nor evidence that a person did.*" + }, + "analysis.facts.citation.one": { + "text": "**Checkable facts found: {0} source contradiction. These did not move the score.**" + }, + "analysis.facts.citation.other": { + "text": "**Checkable facts found: {0} source contradictions. These did not move the score.**" + }, + "analysis.facts.artifact.one": { + "text": "**Checkable facts found: {0} unusual character. These did not move the score.**" + }, + "analysis.facts.artifact.other": { + "text": "**Checkable facts found: {0} unusual characters. These did not move the score.**" + }, + "analysis.facts.both.one-one": { + "text": "**Checkable facts found: {0} source contradiction, {1} unusual character. These did not move the score.**" + }, + "analysis.facts.both.one-other": { + "text": "**Checkable facts found: {0} source contradiction, {1} unusual characters. These did not move the score.**" + }, + "analysis.facts.both.other-one": { + "text": "**Checkable facts found: {0} source contradictions, {1} unusual character. These did not move the score.**" + }, + "analysis.facts.both.other-other": { + "text": "**Checkable facts found: {0} source contradictions, {1} unusual characters. These did not move the score.**" + }, + "analysis.counts.one": { + "text": "- {0} signal counted" + }, + "analysis.counts.other": { + "text": "- {0} signals counted" + }, + "analysis.counts-with-observations.one": { + "text": "- {0} signal counted, plus {1} found at a rate people write at, which count for nothing" + }, + "analysis.counts-with-observations.other": { + "text": "- {0} signals counted, plus {1} found at a rate people write at, which count for nothing" + }, + "analysis.language-stats": { + "text": "- Analysed as {0} · {1} words · {2} sentences · sentence-length variability {3}" + }, + "language.en": { + "text": "English" + }, + "language.es": { + "text": "Spanish" + }, + "language.other": { + "text": "language code {0}" + }, + "caveat.uncalibrated": { + "text": "> **This build has not been calibrated.** No false-positive rate has been measured for it, so the score above should not be used to support a decision about a person." + }, + "caveat.aggregate-no-threshold": { + "text": "> **No threshold is supported yet.** This build was measured against {0} texts, too few to bound its false-positive rate, so no score on this page should be used to support a decision about a person." + }, + "caveat.language-unmeasured": { + "text": "> **This build has never been measured for {0}.** It has no false-positive rate or supported threshold for writing in this language, and the aggregate result from other languages is not a substitute. No score on this page should be used to support a decision about a person." + }, + "caveat.language-no-threshold": { + "text": "> **No threshold is supported for this language yet.** The corpus holds {0} texts in it — too few to bound how often this build is wrong about writing in it, so no score on this page should be used to support a decision about a person. The best bound these texts support is {1}, and the overall figure is not a substitute for it." + }, + "caveat.language-measured": { + "text": "> **A score is not proof.** On {0} texts in this language, published before generative models existed, this build's false-positive rate at a threshold of {1}/100 was under {2} — the upper end of a 95% interval, not a guarantee, and measured on published articles rather than student work. Below that threshold, treat the score as saying nothing." + }, + "caveat.aggregate-measured": { + "text": "> **A score is not proof.** On {0} texts published before generative models existed, this build's false-positive rate at a threshold of {1}/100 was under {2} — the upper end of a 95% interval, not a guarantee, and measured on published articles rather than student work. Below that threshold, treat the score as saying nothing." + }, + "checkable.intro": { + "text": "These are not judgements about the writing and they did not move the score. Each is either present in the file or it is not." + }, + "characters.explanation": { + "text": "Several of these have ordinary explanations — word processors insert soft hyphens and unusual spaces on their own, and any copy-paste can carry them. Invisible characters and letters borrowed from another alphabet are harder to arrive at by accident, though pasting text can do it. This table says what is in the file, not how it got there." + }, + "characters.ordered": { + "text": "Listed with the characters hardest to arrive at by accident first, then in the order they appear in the file." + }, + "characters.table-header": { + "text": "| Character | Codepoint | Line | Column |" + }, + "common.more-rows": { + "text": "… and {0} more." + }, + "citations.issues-note": { + "text": "> None of this needed the internet: the document disagrees with itself. It is a question to ask, not a conclusion — the answer is usually one sentence." + }, + "citations.no-issues-note": { + "text": "> Nothing here is a finding. It describes what could and could not be checked." + }, + "signals.none": { + "text": "None." + }, + "signals.ordered": { + "text": "Ordered by how much weight each one carries, heaviest first, rather than by where it appears in the text." + }, + "signals.more": { + "text": "… and {0} more, none of them carrying more weight than what is shown above." + }, + "observations.intro": { + "text": "Measured against writing published before generative models existed. Shown because they are real, and counted for nothing because they are ordinary." + }, + "observations.row.one": { + "text": "- {0} — {1} occurrence" + }, + "observations.row.other": { + "text": "- {0} — {1} occurrences" + }, + "privacy.document": { + "text": "*This report was produced on the device that ran the analysis and contains material from the document it describes. It is yours to keep or to send; nothing here was uploaded anywhere.*" + }, + "folder.summary.one": { + "text": "{0} file scanned." + }, + "folder.summary.other": { + "text": "{0} files scanned." + }, + "folder.summary-unreadable.one": { + "text": "{0} file scanned, {1} unreadable." + }, + "folder.summary-unreadable.other": { + "text": "{0} files scanned, {1} unreadable." + }, + "folder.reading-order": { + "text": "> **This is a reading order, not a ranking.** A higher score means look sooner, and nothing more. Nothing on this page establishes that anyone did anything." + }, + "folder.table-header": { + "text": "| File | Score | Signals | Words |" + }, + "folder.unreadable-row": { + "text": "- {0} — {1}" + }, + "privacy.folder": { + "text": "*Produced on the device that scanned the folder. It names your students' files, so treat it as you would the coursework itself; nothing here was uploaded anywhere.*" + }, + "how.uncalibrated": { + "text": "This build ships no calibration, so nothing is known about how often it is wrong. That is itself the most important thing on this page." + }, + "how.language-unmeasured": { + "text": "This build has never been measured on writing in {0}. No language-specific false-positive rate or threshold exists, and the aggregate result from other languages is not a substitute." + }, + "how.language-no-threshold": { + "text": "Measured against **{0} texts in this language**, published before generative models existed, on {2} with engine {3}. That sample is too small to support a threshold; the best upper bound it supports is **{1}**, and the overall figure is not a substitute." + }, + "how.language-measured": { + "text": "Measured against **{0} texts in this language**, published before generative models existed, on {3} with engine {4}. At **{1}/100**, the upper end of the measured 95% false-positive interval was **{2}** — an interval, not a guarantee." + }, + "how.aggregate-intro": { + "text": "Measured against **{0} texts published before generative models existed**, so their authorship rests on their dates rather than on anybody's judgement. Measured on {1} with engine {2}." + }, + "how.aggregate-threshold": { + "text": "At **{0}/100**, {1} of those {2} were flagged — an observed {3}, with a 95% interval of {4} – {5}." + }, + "how.read-interval": { + "text": "Read the interval, not the observed rate. {0} out of {1} is not a false-positive rate you can round down." + }, + "how.noisy-intro": { + "text": "The rules seen most often on that human writing, worst first — if the evidence above leans on one of these, weigh it accordingly:" + }, + "how.noisy-rule": { + "text": "- `{0}` — {1} of human texts" + }, + "how.limitation": { + "text": "What this does **not** tell you: how much machine-written text it catches. That is the other half of the picture and it is deliberately not measured here, because any collection of machine-written text samples whichever models were convenient that month. A tool that flags nothing has a perfect false-positive rate." + } } } diff --git a/src/SignsOfAI.Core/Reporting/report.es.json b/src/SignsOfAI.Core/Reporting/report.es.json index 750ab32..c2b0519 100644 --- a/src/SignsOfAI.Core/Reporting/report.es.json +++ b/src/SignsOfAI.Core/Reporting/report.es.json @@ -1,6 +1,8 @@ { "language": "es", - "translators": ["Equipo de SignsOfAI"], + "translators": [ + "Equipo de SignsOfAI" + ], "messages": { "fallback.marker": { "text": "Este bloque aún no está traducido; se muestra en inglés.", @@ -78,6 +80,10 @@ "text": "*Por debajo del umbral que esta compilación puede respaldar, no se emite ningún veredicto. Una puntuación baja no demuestra que una persona haya escrito este texto.*", "sourceHash": "b249a9a70a6e6c6acbd87df70e9eb60558eddb6bb39e8ac49af95db5c44811e8" }, + "analysis.no-verdict-short": { + "text": "*No se da veredicto: este documento tiene {0} palabras, y la frontera de arriba se midió solo sobre textos de {1} palabras o más. Nada tan corto se midió, así que la puntuación de abajo queda sola — no es prueba de que lo escribiera una máquina ni de que lo escribiera una persona.*", + "sourceHash": "ec90d082e44b2e0eec6defaae43905deb0bb90ee53884fb4e40ff06dc4a92f35" + }, "language.en": { "text": "inglés", "sourceHash": "ba118bf7fc9c1aedc1edb28a0aa86e0b43b681f222af6616e13c43be87815b06" diff --git a/src/SignsOfAI.UI/Components/LiveRewritePanel.razor b/src/SignsOfAI.UI/Components/LiveRewritePanel.razor index ab85da6..646ac44 100644 --- a/src/SignsOfAI.UI/Components/LiveRewritePanel.razor +++ b/src/SignsOfAI.UI/Components/LiveRewritePanel.razor @@ -269,11 +269,16 @@ if (cursor < Text.Length) yield return new Seg(Text[cursor..], null); } - private static string ScoreClass(double score) => VerdictBands.Emphasis(score) switch - { - VerdictEmphasis.High => "danger", - VerdictEmphasis.Elevated => "warn", - VerdictEmphasis.Present => "notice", - _ => "good", - }; + // The document's own language and length, so the two numbers here are coloured by the same rule + // as the score above them. Below the measured length neither band means anything, and a green + // "after" number would read as this panel certifying a rewrite the engine declines to judge. + private string ScoreClass(double score) => + VerdictBands.Emphasis(score, Result?.Language, Result?.Statistics.WordCount ?? 0) switch + { + VerdictEmphasis.High => "danger", + VerdictEmphasis.Elevated => "warn", + VerdictEmphasis.Present => "notice", + VerdictEmphasis.Unmeasured => "unmeasured", + _ => "good", + }; } diff --git a/src/SignsOfAI.UI/Pages/Batch.razor b/src/SignsOfAI.UI/Pages/Batch.razor index f30b20e..0b90029 100644 --- a/src/SignsOfAI.UI/Pages/Batch.razor +++ b/src/SignsOfAI.UI/Pages/Batch.razor @@ -83,7 +83,8 @@ else @if (r.Score is { } s) { - @s + @s } else { @@ -101,7 +102,12 @@ else } @code { - private sealed record Row(string Name, string Path, int? Words, int? Score, int? Findings, string? Error); + private sealed record Row( + string Name, string Path, int? Words, int? Score, int? Findings, string? Error, + // Carried so the pill can be coloured by the same rule as every other surface: a language the + // corpus never contained and a document below the measured length both mean "no verdict", and + // Measured(null) is false, so passing nothing here would grey out the whole table. + string? Language = null); private readonly List _rows = []; @@ -195,7 +201,7 @@ else file.Name, file.Path, result.Statistics.WordCount, (int)Math.Round(result.OverallScore), - result.Signals.Count, null)); + result.Signals.Count, null, result.Language)); } // Render each row as it lands. Two hundred files behind one silent await looks @@ -216,10 +222,16 @@ else // This said 40 where every other surface said 45 — the drift the single source exists to stop, // and it had already happened: one document could be coloured two ways by the same build. - private static string BandClass(int score) => VerdictBands.Emphasis(score) switch - { - VerdictEmphasis.High => "high", - VerdictEmphasis.Elevated => "medium", - _ => "low", - }; + // + // The row carries its own word count because a folder of submissions is exactly where short ones + // turn up, and a triage list that paints an unmeasured document green has answered a question it + // was not asked. + private static string BandClass(int score, int? words, string? language) => + VerdictBands.Emphasis(score, language, words ?? int.MaxValue) switch + { + VerdictEmphasis.High => "high", + VerdictEmphasis.Elevated => "medium", + VerdictEmphasis.Unmeasured => "unmeasured", + _ => "low", + }; } diff --git a/src/SignsOfAI.UI/Pages/Home.razor b/src/SignsOfAI.UI/Pages/Home.razor index 49405e8..95823ea 100644 --- a/src/SignsOfAI.UI/Pages/Home.razor +++ b/src/SignsOfAI.UI/Pages/Home.razor @@ -207,7 +207,7 @@ var r = _result;
-
@Math.Round(r.OverallScore) @@ -215,9 +215,17 @@
-

@L.Verdict(r.OverallScore)

+

@L.Verdict(r)

@L.P(r.Signals.Count, "home.signals") · @L["home.analyzedas"] @L.TextLanguageName(r.Language)

+ @* Why nothing is being claimed, at the moment the reader would otherwise assume the + silence means "clean". The evidence below is unaffected: only the claim is. *@ + @if (!VerdictBands.Measured(r.Statistics.WordCount)) + { +

+ @L.F("home.unmeasured.length", r.Statistics.WordCount, VerdictBands.MinimumWords ?? 0) +

+ }
@foreach (var c in r.CategoryScores.Where(c => c.FindingCount > 0)) { @@ -308,9 +316,9 @@ @if (_humanizedResult is not null) { - @Math.Round(r.OverallScore) + @Math.Round(r.OverallScore) → - @Math.Round(_humanizedResult.OverallScore) + @Math.Round(_humanizedResult.OverallScore) @{ var delta = r.OverallScore - _humanizedResult.OverallScore; } @if (delta > 0.5) { −@Math.Round(delta) } @@ -684,8 +692,8 @@ else { var data = new ShareCardData( Score: r.OverallScore, - ScoreColor: ScoreHex(r.OverallScore), - Verdict: L.Verdict(r.OverallScore), + ScoreColor: ScoreHex(r), + Verdict: L.Verdict(r), Signals: r.Signals.Count, Language: L.TextLanguageName(r.Language), Categories: r.CategoryScores.Where(c => c.FindingCount > 0) @@ -720,21 +728,27 @@ else $"signsofai-report-{stamp}.html", EvidenceReport.ToHtml(r, options)); } - private static string ScoreClass(double score) => VerdictBands.Emphasis(score) switch - { - VerdictEmphasis.High => "danger", - VerdictEmphasis.Elevated => "warn", - VerdictEmphasis.Present => "notice", - _ => "good", - }; + // Colour is part of the verdict. Painting a 72/100 passage green because the verdict was + // withheld would state the opposite of what was withheld, in the loudest channel on the page. + private static string ScoreClass(AnalysisResult r) => + VerdictBands.Emphasis(r.OverallScore, r.Language, r.Statistics.WordCount) switch + { + VerdictEmphasis.High => "danger", + VerdictEmphasis.Elevated => "warn", + VerdictEmphasis.Present => "notice", + VerdictEmphasis.Unmeasured => "unmeasured", + _ => "good", + }; - private static string ScoreHex(double score) => VerdictBands.Emphasis(score) switch - { - VerdictEmphasis.High => "#dc2626", - VerdictEmphasis.Elevated => "#ea580c", - VerdictEmphasis.Present => "#ca8a04", - _ => "#16a34a", - }; + private static string ScoreHex(AnalysisResult r) => + VerdictBands.Emphasis(r.OverallScore, r.Language, r.Statistics.WordCount) switch + { + VerdictEmphasis.High => "#dc2626", + VerdictEmphasis.Elevated => "#ea580c", + VerdictEmphasis.Present => "#ca8a04", + VerdictEmphasis.Unmeasured => "#6b7280", + _ => "#16a34a", + }; private const string SampleEn = "In today's digital age, we must delve into the rich tapestry of modern innovation. It's worth " + diff --git a/src/SignsOfAI.UI/Services/Loc.cs b/src/SignsOfAI.UI/Services/Loc.cs index a433597..c579669 100644 --- a/src/SignsOfAI.UI/Services/Loc.cs +++ b/src/SignsOfAI.UI/Services/Loc.cs @@ -246,14 +246,19 @@ private async Task LogAsync(string message) public string Sev(Severity severity) => this["sev." + severity.ToString().ToLowerInvariant()]; /// - /// The one-line verdict for an overall score, in the interface's language. The boundary comes - /// from ; this only chooses the words for it. + /// The one-line verdict for a document, in the interface's language. The boundary comes from + /// ; this only chooses the words for it. + /// + /// It takes the result rather than the score because two of the four answers are not about the + /// score at all: a language the corpus never contained and a document shorter than anything the + /// boundary was measured on both mean *this build has nothing to say*, which is a different + /// sentence from "no signs above the boundary" and must not borrow its reassurance. See #59. /// - public string Verdict(double score) => VerdictBands.Holds(score) switch - { - true => this["verdict.signs"], - false => this["verdict.none"], - }; + public string Verdict(AnalysisResult result) => + !VerdictBands.Measured(result.Statistics.WordCount) ? this["verdict.unmeasured.length"] + : !VerdictBands.Measured(result.Language) ? this["verdict.unmeasured.language"] + : VerdictBands.Holds(result.OverallScore) ? this["verdict.signs"] + : this["verdict.none"]; /// How to name the language the analyzer settled on. The engine only knows EN and ES. public string TextLanguageName(string code) => this[code == "es" ? "lang.spanish" : "lang.english"]; diff --git a/src/SignsOfAI.UI/wwwroot/css/app.css b/src/SignsOfAI.UI/wwwroot/css/app.css index e43aa16..e9fcca3 100644 --- a/src/SignsOfAI.UI/wwwroot/css/app.css +++ b/src/SignsOfAI.UI/wwwroot/css/app.css @@ -214,6 +214,9 @@ button:disabled { opacity: .45; cursor: not-allowed; } .score-ring.notice { --ring: var(--notice); } .score-ring.warn { --ring: var(--warn); } .score-ring.danger { --ring: var(--danger); } +/* Outside what was measured. Grey rather than green: the page is refusing to answer, and green is + an answer. See VerdictEmphasis.Unmeasured. */ +.score-ring.unmeasured { --ring: var(--text-muted); } .score-inner { width: 88px; height: 88px; border-radius: 50%; background: var(--surface); display: grid; place-items: center; text-align: center; @@ -226,6 +229,8 @@ button:disabled { opacity: .45; cursor: not-allowed; } .verdict h2.notice { color: var(--notice); } .verdict h2.warn { color: var(--warn); } .verdict h2.danger { color: var(--danger); } +.verdict h2.unmeasured { color: var(--text-muted); } +.hint.unmeasured { margin-top: .4rem; max-width: 46rem; } .verdict p { margin: 0 0 .6rem; color: var(--text-muted); font-size: .9rem; } .cat-chips { display: flex; gap: .4rem; flex-wrap: wrap; } @@ -1084,6 +1089,7 @@ button.ghost.sm { padding: .35rem .7rem; font-size: .82rem; } .score-pill.high { background: color-mix(in srgb, var(--danger) 16%, transparent); color: var(--danger); } .score-pill.medium { background: color-mix(in srgb, var(--warn) 18%, transparent); color: var(--warn); } .score-pill.low { background: color-mix(in srgb, var(--ok) 16%, transparent); color: var(--ok); } +.score-pill.unmeasured { background: var(--surface-2); color: var(--text-muted); } /* One-time model download (desktop). The bar only appears once a total is known; when the server sends no Content-Length the megabytes counter carries the message on its own. */ @@ -1156,3 +1162,6 @@ button.ghost.sm { padding: .35rem .7rem; font-size: .82rem; } .dl-steps { margin: .6rem 0 .8rem; padding-left: 1.2rem; font-size: .88rem; line-height: 1.6; } .dl-steps li { margin-bottom: .2rem; } .dl-steps code { font-size: .84em; } + +/* The before/after pair of the rewriter, and the folder-scan pill, share the refusal colour. */ +.sc-num.unmeasured { color: var(--text-muted); } diff --git a/src/SignsOfAI.UI/wwwroot/i18n/en.json b/src/SignsOfAI.UI/wwwroot/i18n/en.json index 2b3cb12..72d9bad 100644 --- a/src/SignsOfAI.UI/wwwroot/i18n/en.json +++ b/src/SignsOfAI.UI/wwwroot/i18n/en.json @@ -31,6 +31,7 @@ "batch.col.score": "Score", "batch.col.findings": "Signals", "batch.col.note": "Note", + "batch.unmeasured": "Shorter than anything the boundary was measured on — no verdict", "dl.pagetitle": "Windows app — Signs of AI Writing", "dl.h1": "Signs of AI Writing for Windows", "dl.tagline": "The same tool as this page, in a window: it opens the file formats a browser tab cannot, reads a whole folder at once, and measures on your own machine.", @@ -92,6 +93,8 @@ "sev.high": "High", "verdict.signs": "Signs of AI writing", "verdict.none": "No signs above the measured boundary", + "verdict.unmeasured.length": "No verdict at this length", + "verdict.unmeasured.language": "No verdict in this language", "home.pagetitle": "Signs of AI Writing — detect & de-AI-ify your text", "home.tagline": "We mark the signs of AI writing and show you the evidence, not just a percentage.", "home.privacy": "Analysis runs entirely in your browser. Your text never leaves your device.", @@ -149,6 +152,7 @@ "home.saved": "Saved in this browser", "home.signals.one": "{0} signal found", "home.signals.other": "{0} signals found", + "home.unmeasured.length": "This text is {0} words. The boundary was measured only on texts of {1} words and longer, so no verdict is given — the score is neither evidence that a machine wrote this nor evidence that a person did. Everything below is unaffected.", "home.analyzedas": "analyzed as", "home.humanize": "Humanize with AI", "home.humanizing": "Humanizing…", diff --git a/src/SignsOfAI.UI/wwwroot/i18n/es.json b/src/SignsOfAI.UI/wwwroot/i18n/es.json index bf2fc9d..55b7702 100644 --- a/src/SignsOfAI.UI/wwwroot/i18n/es.json +++ b/src/SignsOfAI.UI/wwwroot/i18n/es.json @@ -31,6 +31,7 @@ "batch.col.score": "Puntuación", "batch.col.findings": "Señales", "batch.col.note": "Nota", + "batch.unmeasured": "Más corto que nada sobre lo que se midió la frontera — sin veredicto", "dl.pagetitle": "App de Windows — Señales de escritura IA", "dl.h1": "Señales de escritura IA para Windows", "dl.tagline": "La misma herramienta de esta página, en una ventana: abre los formatos que una pestaña no puede, lee una carpeta entera de una vez y mide en tu propia máquina.", @@ -92,6 +93,8 @@ "sev.high": "Alta", "verdict.signs": "Señales de escritura con IA", "verdict.none": "Sin señales por encima del umbral medido", + "verdict.unmeasured.length": "Sin veredicto a esta longitud", + "verdict.unmeasured.language": "Sin veredicto en este idioma", "home.pagetitle": "Señales de escritura IA — detecta y humaniza tu texto", "home.tagline": "Marcamos las señales de escritura con IA y te enseñamos la evidencia, no solo un porcentaje.", "home.privacy": "El análisis se ejecuta por completo en tu navegador. Tu texto nunca sale de tu dispositivo.", @@ -149,6 +152,7 @@ "home.saved": "Guardado en este navegador", "home.signals.one": "{0} señal encontrada", "home.signals.other": "{0} señales encontradas", + "home.unmeasured.length": "Este texto tiene {0} palabras. La frontera se midió solo sobre textos de {1} palabras o más, así que no se da veredicto — la puntuación no es prueba de que lo escribiera una máquina ni de que lo escribiera una persona. Todo lo de abajo no cambia.", "home.analyzedas": "analizado como", "home.humanize": "Humanizar con IA", "home.humanizing": "Humanizando…", diff --git a/tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs b/tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs index d0e6f50..c1f72cd 100644 --- a/tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs +++ b/tests/SignsOfAI.Core.Tests/EvidenceReportTests.cs @@ -333,7 +333,10 @@ public void Withholds_the_verdict_below_the_threshold_it_can_support() // A verdict printed above "treat the score as saying nothing" is a page arguing with itself, // and the reader keeps whichever half suits them. Below the boundary the report prints the // score and the reason, and no verdict line of either kind. - var report = Report(); + // Long enough that the reason for the silence is the threshold and not the length — the two + // are different sentences since #59, and this test is about the first one. + var report = EvidenceReport.ToMarkdown( + new AiWritingAnalyzer().Analyze(Fixtures.LongEnough(Essay), "en")); Assert.DoesNotContain("No signs above the measured boundary", report); Assert.DoesNotContain("Signs of AI writing", report); diff --git a/tests/SignsOfAI.Core.Tests/Fixtures.cs b/tests/SignsOfAI.Core.Tests/Fixtures.cs new file mode 100644 index 0000000..18591f3 --- /dev/null +++ b/tests/SignsOfAI.Core.Tests/Fixtures.cs @@ -0,0 +1,41 @@ +using System; +using System.Linq; +using SignsOfAI.Core.Model; + +namespace SignsOfAI.Core.Tests; + +/// +/// Shared fixture plumbing, and one piece of it is a fact about the engine rather than a convenience. +/// +/// Since #59 a document shorter than the shortest text the boundary was measured on gets no verdict +/// at all — 662 words on the corpus this build ships. Almost every fixture in this suite is a +/// paragraph, so without a test written to check *what the verdict says* +/// silently becomes a test of the length gate, passes for the wrong reason or fails for a reason that +/// has nothing to do with what it was guarding. +/// +/// Which is itself the finding: the fixtures were short because the way people use this tool is +/// short, and that is exactly the population the boundary was never measured on. +/// +internal static class Fixtures +{ + /// + /// The passage repeated until it clears , so a test about + /// the wording of a verdict gets one. + /// + /// Repetition rather than filler on purpose: padding with unrelated prose would change what the + /// rules find and move the score the test is asserting about, while repeating the same sentences + /// keeps the sentence-length distribution — and therefore the burstiness — roughly where it was. + /// + public static string LongEnough(string text) + { + if (VerdictBands.MinimumWords is not { } floor) return text; + + var words = text.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries).Length; + if (words == 0) return text; + + // A margin over the floor, because the analyzer's word count and this split disagree by a few + // per cent and a fixture that lands one word short fails somewhere far from here. + var copies = (int)Math.Ceiling((floor * 1.25) / words); + return string.Join(" ", Enumerable.Repeat(text, Math.Max(1, copies))); + } +} diff --git a/tests/SignsOfAI.Core.Tests/LocaleFileTests.cs b/tests/SignsOfAI.Core.Tests/LocaleFileTests.cs index a83a3dd..b51404a 100644 --- a/tests/SignsOfAI.Core.Tests/LocaleFileTests.cs +++ b/tests/SignsOfAI.Core.Tests/LocaleFileTests.cs @@ -29,6 +29,8 @@ public class LocaleFileTests .. Enum.GetValues().Select(c => "cat." + c.ToString().ToLowerInvariant()), .. Enum.GetValues().Select(s => "sev." + s.ToString().ToLowerInvariant()), "verdict.signs", "verdict.none", + // Loc.Verdict picks one of four by asking VerdictBands, so a text search cannot see them. + "verdict.unmeasured.length", "verdict.unmeasured.language", "lang.english", "lang.spanish", // TaskEntry builds these from its own list, so a plain text search cannot see them and a // deleted one would render as the raw key on the front door. diff --git a/tests/SignsOfAI.Core.Tests/VerdictAgreementTests.cs b/tests/SignsOfAI.Core.Tests/VerdictAgreementTests.cs index 943fb4c..f3022ff 100644 --- a/tests/SignsOfAI.Core.Tests/VerdictAgreementTests.cs +++ b/tests/SignsOfAI.Core.Tests/VerdictAgreementTests.cs @@ -80,7 +80,9 @@ public void The_verdict_reaches_its_reader_in_their_own_language() // translation whose pin no longer matches is treated as stale — the whole report silently // falls back to English. That is the correct behaviour and a silent way to undo #36, so the // wording change and the pins have to travel together. This is the test that says they did. - var result = new AiWritingAnalyzer().Analyze(ObviouslyMachine, "en"); + // Long enough to earn a verdict at all: since #59 a paragraph gets none, and this test is + // about which language the verdict arrives in, not about whether one is given. + var result = new AiWritingAnalyzer().Analyze(Fixtures.LongEnough(ObviouslyMachine), "en"); var spanish = EvidenceReport.ToMarkdown(result, new ReportOptions { InterfaceLanguage = "es" }); Assert.Contains("Señales de escritura con IA", spanish); diff --git a/tests/SignsOfAI.Core.Tests/VerdictLengthTests.cs b/tests/SignsOfAI.Core.Tests/VerdictLengthTests.cs new file mode 100644 index 0000000..0ca2451 --- /dev/null +++ b/tests/SignsOfAI.Core.Tests/VerdictLengthTests.cs @@ -0,0 +1,171 @@ +using System.Linq; +using SignsOfAI.Core; +using SignsOfAI.Core.Calibration; +using SignsOfAI.Core.Model; +using SignsOfAI.Core.Reporting; +using Xunit; + +namespace SignsOfAI.Core.Tests; + +/// +/// The length condition on the verdict — issue #59. +/// +/// The boundary this build ships was fitted on 90 texts whose shortest is 662 words, and it was being +/// applied to a pasted paragraph with nothing on the page to say so. The same documents flag 0 of 32 +/// whole and 6 of 32 as 400-word excerpts of themselves, so the error does not merely get noisier as +/// text gets shorter: it moves one way, toward the machine. +/// +/// What these guard is a **coverage** claim, not a reliability one. Nothing here asserts that the +/// tool is wrong below 662 words. It asserts that the tool stops claiming, which is the only thing +/// the corpus supports — and that it goes on showing the evidence, because the evidence was never +/// what the boundary was about. +/// +public class VerdictLengthTests +{ + private readonly AiWritingAnalyzer _analyzer = new(); + + /// A paragraph with enough tells to score well over the boundary, and far too short. + private const string ShortAndMachineLike = + "In today's rapidly evolving digital landscape, we must delve into the rich tapestry of " + + "innovation. It is worth noting that this multifaceted approach serves as a testament to " + + "human ingenuity. It's not just a tool, it's a pivotal, transformative solution that " + + "fosters growth, unlocks potential, and empowers teams. Moreover, the seamless integration " + + "underscores a paradigm shift, highlighting the profound implications for the realm of " + + "modern work."; + + [Fact] + public void The_floor_is_the_shortest_text_the_boundary_was_measured_on() + { + var published = PublishedCalibration.Current; + Assert.NotNull(published); + + // Published rather than chosen. If this is ever null while a threshold is published, the + // build is claiming a boundary without recording the population it was fitted on. + Assert.NotNull(published!.ShortestWords); + Assert.Equal(published.ShortestWords, VerdictBands.MinimumWords); + Assert.True(published.ShortestWords < published.LongestWords, + "A range whose ends are equal is not a range; the snapshot is malformed."); + } + + [Fact] + public void A_paragraph_gets_no_verdict_however_high_it_scores() + { + var result = _analyzer.Analyze(ShortAndMachineLike, "en"); + + Assert.True(result.OverallScore >= VerdictBands.Threshold, + "The fixture stopped scoring above the boundary, so this test no longer proves anything."); + Assert.True(result.Statistics.WordCount < VerdictBands.MinimumWords); + + Assert.False(result.HasVerdict); + Assert.Equal("No verdict: below the measured length", result.Verdict); + } + + [Fact] + public void The_same_writing_at_length_does_get_one() + { + var result = _analyzer.Analyze(Fixtures.LongEnough(ShortAndMachineLike), "en"); + + Assert.True(result.Statistics.WordCount >= VerdictBands.MinimumWords); + Assert.True(result.HasVerdict); + Assert.Equal("Signs of AI writing", result.Verdict); + } + + /// + /// The verdict is withheld; the evidence is not. Suppressing the findings too would leave a + /// teacher with a shorter answer than they had before, and the findings never depended on the + /// boundary — "delve" is in the text or it is not. + /// + [Fact] + public void Withholding_the_verdict_does_not_withhold_the_evidence() + { + var result = _analyzer.Analyze(ShortAndMachineLike, "en"); + + Assert.False(result.HasVerdict); + Assert.NotEmpty(result.Signals); + Assert.Contains(result.Findings, f => f.RuleId == "lex.delve"); + Assert.True(result.OverallScore > 0, "The score is still computed and still shown."); + } + + /// + /// Two different silences, two different sentences. "Below the threshold" is a reading — the tool + /// looked and found little. "Shorter than anything measured" is a refusal to read. A reader told + /// the first when the second is true takes away a reassurance nobody offered. + /// + [Fact] + public void The_report_says_which_silence_it_is() + { + var shortReport = EvidenceReport.ToMarkdown(_analyzer.Analyze(ShortAndMachineLike, "en")); + + Assert.Contains("measured only on texts of", shortReport); + Assert.DoesNotContain("A low score is not evidence that a person wrote this", shortReport); + Assert.DoesNotContain("Signs of AI writing", shortReport); + + // And it names both numbers, so the reader can check the claim rather than take it. + Assert.Contains(VerdictBands.MinimumWords!.Value.ToString("N0"), shortReport); + } + + [Fact] + public void A_spanish_reader_is_told_the_same_thing_in_Spanish() + { + var report = EvidenceReport.ToMarkdown( + _analyzer.Analyze(ShortAndMachineLike, "en"), + new ReportOptions { InterfaceLanguage = "es" }); + + Assert.Contains("se midió solo sobre textos de", report); + Assert.DoesNotContain("measured only on texts of", report); + } + + /// + /// Colour is part of the verdict whatever the design system pretends. Before this, a withheld + /// verdict fell through to , which every surface paints green — + /// so a 72/100 passage would have been withheld in words and certified in colour. + /// + [Fact] + public void An_unmeasured_document_is_not_painted_as_a_clean_one() + { + var result = _analyzer.Analyze(ShortAndMachineLike, "en"); + + Assert.Equal( + VerdictEmphasis.Unmeasured, + VerdictBands.Emphasis(result.OverallScore, result.Language, result.Statistics.WordCount)); + + // And the plain overload, which knows nothing about length, still says what it always said — + // it is not the one to ask, and callers that ask it are the ones this test cannot catch. + Assert.NotEqual(VerdictEmphasis.Unmeasured, VerdictBands.Emphasis(result.OverallScore)); + } + + [Fact] + public void A_document_longer_than_the_corpus_is_still_judged() + { + // No ceiling, and the asymmetry is measured rather than assumed: shortening a text moves its + // score toward the machine, and nothing suggests a long thesis is at risk. Silencing the long + // end for symmetry would withhold a verdict for a reason nobody has evidence for. + Assert.True(VerdictBands.Measured(int.MaxValue)); + Assert.True(VerdictBands.Measured(PublishedCalibration.Current!.LongestWords!.Value * 10)); + } + + /// + /// One place decides. Before existed the answer was written out in + /// eight, and one engine gave three answers about the same text; length must not reintroduce a + /// second opinion. + /// + [Theory] + [InlineData("en")] + [InlineData("es")] + public void Every_way_of_asking_agrees(string language) + { + var text = language == "es" + ? "En el panorama actual, cabe destacar que este enfoque integral no solo optimiza los " + + "procesos sino que también facilita una comprensión robusta del fenómeno." + : ShortAndMachineLike; + + var result = _analyzer.Analyze(text, language); + var speaks = result.HasVerdict; + + Assert.Equal(speaks, VerdictBands.Holds(result.OverallScore, result.Language, result.Statistics.WordCount)); + Assert.Equal(speaks, result.Verdict == "Signs of AI writing"); + Assert.Equal( + speaks, + EvidenceReport.ToMarkdown(result).Contains("**" + System.Math.Round(result.OverallScore) + "/100 —")); + } +} diff --git a/tools/SignsOfAI.Calibration/Program.cs b/tools/SignsOfAI.Calibration/Program.cs index accaeb0..09c6d20 100644 --- a/tools/SignsOfAI.Calibration/Program.cs +++ b/tools/SignsOfAI.Calibration/Program.cs @@ -553,6 +553,12 @@ static double Median(List values) FlaggedAtThreshold = atThreshold?.Flagged ?? 0, RateLow = atThreshold?.RateLow ?? 0, RateHigh = atThreshold?.RateHigh ?? 1, + + // The lengths the boundary was actually fitted on. The engine withholds its verdict below the + // shorter of the two rather than extrapolating a bound onto a population nobody measured — the + // same rule the language condition follows, applied to the other dimension. See issue #59. + ShortestWords = overall.ShortestWords, + LongestWords = overall.LongestWords, NoisiestRules = [.. calibration.RuleFalsePositives.Take(8) .Select(r => new PublishedRuleRate { RuleId = r.RuleId, TextShare = r.TextShare })], @@ -586,6 +592,8 @@ static double Median(List values) Console.WriteLine($" corpus fingerprint {calibration.CorpusHash}"); Console.WriteLine($" median score {calibration.Overall.MedianScore:0.0}"); Console.WriteLine($" 90th percentile {calibration.Overall.NinetiethScore:0.0}"); +Console.WriteLine($" lengths measured {calibration.Overall.ShortestWords:N0}–{calibration.Overall.LongestWords:N0} words" + + $" (median {calibration.Overall.MedianWords:N0}) — below the shortest, no verdict"); Console.WriteLine(calibration.Overall.ThresholdForTarget is { } t ? $" threshold for {calibration.TargetFalsePositiveRate:P0} {t:0}/100" : $" threshold for {calibration.TargetFalsePositiveRate:P0} not supported by this corpus yet"); diff --git a/tools/SignsOfAI.Calibration/Report.cs b/tools/SignsOfAI.Calibration/Report.cs index 8d81a0c..f45634c 100644 --- a/tools/SignsOfAI.Calibration/Report.cs +++ b/tools/SignsOfAI.Calibration/Report.cs @@ -43,6 +43,8 @@ public static string Render(CalibrationResult r, CorpusManifest manifest, string sb.AppendLine(); sb.AppendLine($"- **Corpus** `{manifest.Id}`, fingerprint `{r.CorpusHash}`"); sb.AppendLine($"- **Texts** {r.Overall.Count:N0} ({r.Overall.TotalWords:N0} words)"); + sb.AppendLine($"- **Lengths measured** {r.Overall.ShortestWords:N0} – {r.Overall.LongestWords:N0} words " + + $"(median {r.Overall.MedianWords:N0})"); sb.AppendLine($"- **Engine** {version}"); sb.AppendLine($"- **Run** {generatedOn}"); sb.AppendLine($"- **Target false-positive rate** {Pct(r.TargetFalsePositiveRate)}"); @@ -66,6 +68,16 @@ public static string Render(CalibrationResult r, CorpusManifest manifest, string sb.AppendLine(); sb.AppendLine(Headline(r.Overall, r.TargetFalsePositiveRate)); sb.AppendLine(); + sb.AppendLine( + $"**It covers documents of {r.Overall.ShortestWords:N0} words and up, because that is what was " + + "measured.** Nothing shorter was: the corpus has no text below that length, so the boundary " + + "below is not supported there and the tool withholds its verdict rather than extrapolating. " + + "That is a statement about coverage, not about where the tool breaks — though the direction " + + "of the length effect *has* been measured, and it goes the wrong way: the same documents " + + "flagged 0 of 32 whole and 6 of 32 as 400-word excerpts of themselves " + + "(`Docs/PARAPHRASE.md`, section *Length*). Lowering this floor means measuring short writing " + + "people actually composed at that length, not slicing long documents into pieces."); + sb.AppendLine(); sb.AppendLine( "Read the interval, not the percentage. On a small corpus an observed rate is compatible " + "with a much wider range, and the recommendation below is made from the **upper** end of " +