Skip to content

Commit 7c112a5

Browse files
authored
Merge pull request #45 from peopleworks/adding-a-language-is-adding-a-file
Make adding a language mean adding a file
2 parents 2e0f9c1 + ad66333 commit 7c112a5

10 files changed

Lines changed: 199 additions & 10 deletions

File tree

CONTRIBUTING.md

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ dotnet run --project src/SignsOfAI.Cli -- check some-file.md
3737

3838
## Adding or changing a rule
3939

40-
Rules live in two JSON files: `src/SignsOfAI.Core/Rules/Packs/rules.en.json` and `rules.es.json`.
40+
Rules live in JSON files under `src/SignsOfAI.Core/Rules/Packs/` — today `rules.en.json` and
41+
`rules.es.json`, and any `rules.<code>.json` you add (see *Adding a whole language* below).
4142
There are two kinds.
4243

4344
**Lexical** — single overused words. All inflections go in `terms`:
@@ -120,6 +121,29 @@ into it will be asked for rework. Spanish AI writing has its own tells (*sumérg
120121
de*, *cabe destacar que*, *un rico tapiz de*). If you propose a Spanish rule, ground it in Spanish
121122
text you have actually seen a model produce.
122123

124+
### Adding a whole language
125+
126+
**Drop `rules.<code>.json` into `src/SignsOfAI.Core/Rules/Packs/` and it is picked up.** No project
127+
file to edit, no C# to touch, no list to register in — the build embeds the pack by wildcard and
128+
`RulePackLoader` finds it by name. That is deliberate: a contributor who cannot get their language
129+
heard without editing a build script will not contribute a language.
130+
131+
Derive it, do not translate it, for the reason above.
132+
133+
Two things the tool will then say on your behalf, and you should expect both:
134+
135+
- **Until your pack exists, text in your language is examined with the English catalog**, and the
136+
report says so and tells the reader to treat the score as meaningless — a low number would mean
137+
nothing was looked for, not that nothing was found.
138+
- **A new language has no measured error rate.** The calibration corpus contains no texts in it, so
139+
no threshold is supported and no verdict is printed. That is honest rather than broken, and it is
140+
fixed by contributing texts published before 2022 — see `Docs/Calibration/README.md`. **Roughly
141+
seventy-five such texts are worth more to your language's users than the rule pack is**, because
142+
they are what lets the tool say anything at all about how often it is wrong there.
143+
144+
The interface (`wwwroot/i18n/<code>.json`) and the report prose (`Reporting/report.<code>.json`) are
145+
separate and can land in separate pull requests: see [`Docs/TRANSLATING.md`](Docs/TRANSLATING.md).
146+
123147
## Tests
124148

125149
Every rule change needs a test in `tests/SignsOfAI.Core.Tests`. The pattern is short:

src/SignsOfAI.Core/AiWritingAnalyzer.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ public AnalysisResult Analyze(string text, string? language = null, IReadOnlyLis
8989
return new AnalysisResult
9090
{
9191
Language = lang,
92+
RulePackLanguage = RulePackLoader.Resolve(lang).Language,
9293
Findings = findings,
9394
CategoryScores = byCategory,
9495
OverallScore = overall,

src/SignsOfAI.Core/Model/AnalysisResult.cs

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,20 @@ public sealed record CategoryScore(SignCategory Category, double Score, int Find
99
/// <summary>The complete result of analyzing a document.</summary>
1010
public sealed record AnalysisResult
1111
{
12-
/// <summary>Language code actually used for analysis ("en" or "es").</summary>
12+
/// <summary>The language of the text: given by the caller, or detected.</summary>
1313
public required string Language { get; init; }
1414

15+
/// <summary>
16+
/// The language whose rule pack actually supplied the tells. Equal to <see cref="Language"/>
17+
/// except when that language has no pack yet, in which case the English catalog was used.
18+
///
19+
/// The two must not be conflated. Running English rules over French prose finds few tells, and
20+
/// reporting that as a French analysis would present "nothing fired" as a result when nothing
21+
/// French was ever looked for. Rule packs are files anyone can contribute, so a language without
22+
/// one is an ordinary state that hosts should describe rather than an error.
23+
/// </summary>
24+
public string RulePackLanguage { get; init; } = "";
25+
1526
/// <summary>
1627
/// Everything that matched, ordered by position in the text — both the findings that count as
1728
/// evidence and the ones the writer is using at a rate people write at. Highlighting works from

src/SignsOfAI.Core/Reporting/EvidenceReport.cs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,16 @@ public static string ToMarkdown(AnalysisResult result, ReportOptions? options =
119119
LanguageName(text, result.Language), result.Statistics.WordCount,
120120
result.Statistics.SentenceCount, Num(result.Statistics.Burstiness, 2));
121121
sb.AppendLine();
122+
123+
// Said before the error rate, because it outranks it: a rate measured on English writing
124+
// describes nothing about what these rules do to French prose they were never written for.
125+
if (result.RulePackLanguage is { Length: > 0 } packLanguage
126+
&& !packLanguage.Equals(result.Language, StringComparison.OrdinalIgnoreCase))
127+
{
128+
AppendBlock(sb, text, ReportMessages.NoRulePack, LanguageName(text, result.Language));
129+
sb.AppendLine();
130+
}
131+
122132
AppendLocalized(sb, text, Caveat(text, result.Language));
123133
sb.AppendLine();
124134

src/SignsOfAI.Core/Reporting/ReportMessages.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ public static class ReportMessages
1616
public const string FallbackMarker = "fallback.marker";
1717
public const string FallbackSummary = "fallback.summary";
1818
public const string FallbackLanguage = "fallback.language";
19+
public const string NoRulePack = "analysis.no-rule-pack";
1920
public const string DefaultTitle = "default.title";
2021
public const string MetaDocument = "meta.document";
2122
public const string MetaGenerated = "meta.generated";
@@ -93,6 +94,7 @@ public static class ReportMessages
9394
[FallbackMarker] = 0,
9495
[FallbackSummary] = 1,
9596
[FallbackLanguage] = 1,
97+
[NoRulePack] = 1, // {0} the language of the text
9698
[DefaultTitle] = 0,
9799
[MetaDocument] = 1,
98100
[MetaGenerated] = 2,
@@ -176,6 +178,11 @@ public static class ReportMessages
176178
[FallbackLanguage] = "This report is not available in {0}, so the whole of it is shown in English. " +
177179
"Nothing has been withheld or shortened, but a reader who cannot read English " +
178180
"cannot read the part that limits the score, and that part is the point of the page.",
181+
[NoRulePack] = "> **There is no rule pack for {0} yet, so this text was examined with the English one.** " +
182+
"Treat the score as saying nothing at all: the tells this tool knows are English " +
183+
"ones, and few of them can fire on writing in another language — so a low number " +
184+
"here means nothing was looked for, not that nothing was found. Rule packs are " +
185+
"JSON files anyone can contribute.",
179186
[DefaultTitle] = "Writing analysis report",
180187
[MetaDocument] = "**Document:** {0}",
181188
[MetaGenerated] = "**Generated:** {0} · **Engine:** SignsOfAI {1}",

src/SignsOfAI.Core/Reporting/report.en.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
"fallback.marker": { "text": "This block has not been translated yet; it is shown in English." },
66
"fallback.summary": { "text": "This report contains {0} block(s) not yet translated. Each is marked and shown in English." },
77
"fallback.language": { "text": "This report is not available in {0}, so the whole of it is shown in English. Nothing has been withheld or shortened, but a reader who cannot read English cannot read the part that limits the score, and that part is the point of the page." },
8+
"analysis.no-rule-pack": { "text": "> **There is no rule pack for {0} yet, so this text was examined with the English one.** Treat the score as saying nothing at all: the tells this tool knows are English ones, and few of them can fire on writing in another language — so a low number here means nothing was looked for, not that nothing was found. Rule packs are JSON files anyone can contribute." },
89
"default.title": { "text": "Writing analysis report" },
910
"meta.document": { "text": "**Document:** {0}" },
1011
"meta.generated": { "text": "**Generated:** {0} · **Engine:** SignsOfAI {1}" },

src/SignsOfAI.Core/Reporting/report.es.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@
1414
"text": "Este informe no está disponible en {0}, así que se muestra completo en inglés. No se ha ocultado ni acortado nada, pero quien no lea inglés no puede leer la parte que limita la puntuación, y esa parte es la razón de ser de esta página.",
1515
"sourceHash": "c43be4647f39ade1600db2c60ed02ecd38d3539b195ec9afaae737316234a6d4"
1616
},
17+
"analysis.no-rule-pack": {
18+
"text": "> **Todavía no existe un catálogo de reglas para {0}, así que este texto se examinó con el de inglés.** Trate la puntuación como si no dijera absolutamente nada: las señales que conoce esta herramienta son inglesas, y pocas pueden activarse sobre escritura en otro idioma — de modo que un número bajo aquí significa que no se buscó nada, no que no se encontró nada. Los catálogos de reglas son archivos JSON que cualquiera puede aportar.",
19+
"sourceHash": "148085719a308a40ec90895e2bd4621ed8367db5573f5d9d31f94d4c99e5e344"
20+
},
1721
"default.title": {
1822
"text": "Informe del análisis de escritura",
1923
"sourceHash": "90b8ccc0903d87a2f8ba07531f1e76736d5fce4adab65154f05ac147b919b291"

src/SignsOfAI.Core/Rules/RulePackLoader.cs

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,21 +9,54 @@ public static class RulePackLoader
99
{
1010
private static readonly ConcurrentDictionary<string, RulePack> Cache = new();
1111

12-
/// <summary>Loads the rule-pack for a language code ("en"/"es"), falling back to English.</summary>
13-
public static RulePack Load(string language)
12+
/// <summary>Loads the rule-pack for a language code, falling back to English.</summary>
13+
public static RulePack Load(string language) => Resolve(language).Pack;
14+
15+
/// <summary>
16+
/// The pack, and the language it was actually built for.
17+
///
18+
/// These differ whenever a language has no pack yet, and the difference has to travel: a text
19+
/// analysed with the English catalog is not a French analysis, and a result that claimed to be
20+
/// one would be saying nothing fired in French when nothing French was ever looked for. Rule
21+
/// packs are files anyone can add, so this is the ordinary case for a new language rather than
22+
/// an error.
23+
/// </summary>
24+
public static (RulePack Pack, string Language) Resolve(string? language)
1425
{
1526
var lang = string.IsNullOrWhiteSpace(language) ? "en" : language.ToLowerInvariant();
16-
return Cache.GetOrAdd(lang, LoadFromResource);
27+
return (Cache.GetOrAdd(lang, LoadFromResource), Available(lang) ? lang : "en");
1728
}
1829

30+
/// <summary>Whether a built-in pack exists for this language. Adding one is adding a file.</summary>
31+
public static bool Available(string? language) =>
32+
!string.IsNullOrWhiteSpace(language)
33+
&& typeof(RulePackLoader).Assembly.GetManifestResourceInfo(ResourceName(language)) is not null;
34+
35+
/// <summary>
36+
/// Every language with a built-in pack, so hosts can offer what exists rather than a hardcoded
37+
/// pair that a contributor cannot extend.
38+
/// </summary>
39+
public static IReadOnlyList<string> Languages { get; } =
40+
[.. typeof(RulePackLoader).Assembly.GetManifestResourceNames()
41+
.Where(n => n.StartsWith(Prefix, StringComparison.Ordinal)
42+
&& n.EndsWith(".json", StringComparison.Ordinal))
43+
.Select(n => n[Prefix.Length..^".json".Length])
44+
.Where(n => n.Length is > 0 and <= 12 && n.All(char.IsAsciiLetterLower))
45+
.Order(StringComparer.Ordinal)];
46+
47+
private const string Prefix = "SignsOfAI.Core.Rules.Packs.rules.";
48+
49+
private static string ResourceName(string language) =>
50+
$"SignsOfAI.Core.Rules.Packs.rules.{language.ToLowerInvariant()}.json";
51+
1952
private static RulePack LoadFromResource(string language)
2053
{
2154
var asm = typeof(RulePackLoader).Assembly;
22-
var resourceName = $"SignsOfAI.Core.Rules.Packs.rules.{language}.json";
55+
var resourceName = ResourceName(language);
2356

2457
using var stream = asm.GetManifestResourceStream(resourceName)
2558
?? (language != "en"
26-
? asm.GetManifestResourceStream("SignsOfAI.Core.Rules.Packs.rules.en.json")
59+
? asm.GetManifestResourceStream(ResourceName("en"))
2760
: null)
2861
?? throw new InvalidOperationException(
2962
$"Rule-pack resource '{resourceName}' not found. Available: " +

src/SignsOfAI.Core/SignsOfAI.Core.csproj

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,16 @@
2626
</ItemGroup>
2727

2828
<ItemGroup>
29-
<!-- WithCulture=false is essential: filenames like rules.en.json / rules.es.json would
29+
<!-- A wildcard, so adding a language is adding a file. Listing packs one by one made every
30+
translator edit the build to be heard, which contradicts this project's own rule that
31+
extension points are JSON anyone can send by pull request and never compiled code.
32+
33+
WithCulture=false is essential: filenames like rules.en.json / rules.es.json would
3034
otherwise be mistaken for culture-specific satellite resources and dropped from the
3135
main assembly manifest. LogicalName pins the reflection lookup name. -->
32-
<EmbeddedResource Include="Rules\Packs\rules.en.json" WithCulture="false" LogicalName="SignsOfAI.Core.Rules.Packs.rules.en.json" />
33-
<EmbeddedResource Include="Rules\Packs\rules.es.json" WithCulture="false" LogicalName="SignsOfAI.Core.Rules.Packs.rules.es.json" />
36+
<EmbeddedResource Include="Rules\Packs\rules.*.json" WithCulture="false">
37+
<LogicalName>SignsOfAI.Core.Rules.Packs.%(Filename)%(Extension)</LogicalName>
38+
</EmbeddedResource>
3439

3540
<!-- Reader-facing report prose is selected by interface language, independently from the rule
3641
pack selected for the analysed text. LogicalName keeps dotted locale filenames in the main
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
using SignsOfAI.Core;
2+
using SignsOfAI.Core.Reporting;
3+
using SignsOfAI.Core.Rules;
4+
5+
namespace SignsOfAI.Core.Tests;
6+
7+
/// <summary>
8+
/// The project tells contributors that adding a language is adding a file. These are the tests that
9+
/// make that true rather than aspirational — and that stop a language without a pack being reported
10+
/// as though it had one.
11+
/// </summary>
12+
public class RulePackLoaderTests
13+
{
14+
[Fact]
15+
public void Every_pack_file_in_the_project_is_discoverable_without_a_build_edit()
16+
{
17+
// The packs were listed one by one in the .csproj, so a translator had to edit the build to
18+
// be heard at all. A wildcard replaced that; this test is what keeps it a wildcard.
19+
Assert.Contains("en", RulePackLoader.Languages);
20+
Assert.Contains("es", RulePackLoader.Languages);
21+
22+
var onDisk = Directory
23+
.GetFiles(ProjectPacksDirectory(), "rules.*.json")
24+
.Select(f => Path.GetFileNameWithoutExtension(f)!.Split('.')[1])
25+
.Order(StringComparer.Ordinal);
26+
27+
Assert.Equal(onDisk, RulePackLoader.Languages.Order(StringComparer.Ordinal));
28+
}
29+
30+
[Fact]
31+
public void A_language_with_no_pack_says_which_pack_it_actually_used()
32+
{
33+
// Silently loading English while the result claimed the requested language turned "nothing
34+
// fired" into a finding, when nothing had been looked for.
35+
//
36+
// "zz" throughout, never a real code: this suite must keep passing on the day somebody
37+
// contributes the language it uses as its example, and picking "fr" would make a welcome
38+
// pull request look like a regression.
39+
var (_, language) = RulePackLoader.Resolve("zz");
40+
41+
Assert.Equal("en", language);
42+
Assert.False(RulePackLoader.Available("zz"));
43+
}
44+
45+
[Fact]
46+
public void A_language_with_a_pack_reports_itself()
47+
{
48+
Assert.Equal("es", RulePackLoader.Resolve("es").Language);
49+
Assert.True(RulePackLoader.Available("es"));
50+
}
51+
52+
[Fact]
53+
public void The_result_keeps_the_two_languages_apart()
54+
{
55+
var result = new AiWritingAnalyzer().Analyze("Le texte est court mais suffisant.", "zz");
56+
57+
// The text is in the language asked for. The rules that read it were not.
58+
Assert.Equal("zz", result.Language);
59+
Assert.Equal("en", result.RulePackLanguage);
60+
}
61+
62+
[Fact]
63+
public void The_report_refuses_to_present_an_English_reading_as_a_result_in_that_language()
64+
{
65+
var result = new AiWritingAnalyzer().Analyze(
66+
"La rédaction académique exige de la précision et une structure claire.", "zz");
67+
68+
var report = EvidenceReport.ToMarkdown(result);
69+
70+
Assert.Contains("no rule pack for", report);
71+
Assert.Contains("nothing was looked for", report);
72+
}
73+
74+
[Fact]
75+
public void A_language_that_has_a_pack_carries_no_such_warning()
76+
{
77+
var result = new AiWritingAnalyzer().Analyze(
78+
"La redacción académica exige precisión y una estructura clara.", "es");
79+
80+
Assert.DoesNotContain("no rule pack for", EvidenceReport.ToMarkdown(result));
81+
}
82+
83+
/// <summary>The packs as they sit in the repository, not as the build happened to embed them.</summary>
84+
private static string ProjectPacksDirectory()
85+
{
86+
var dir = new DirectoryInfo(AppContext.BaseDirectory);
87+
while (dir is not null && !File.Exists(Path.Combine(dir.FullName, "SignsOfAI.slnx")))
88+
dir = dir.Parent;
89+
90+
Assert.NotNull(dir);
91+
return Path.Combine(dir!.FullName, "src", "SignsOfAI.Core", "Rules", "Packs");
92+
}
93+
}

0 commit comments

Comments
 (0)