Merge PR #154: Add MDS037 duplicated-content rule for cross-file paragraph detection

Author: jeduden

PLAN.md

@@ -16,7 +16,7 @@ footer: |
 
 | ID  | Status | Title                                                                                                           |
 |-----|--------|-----------------------------------------------------------------------------------------------------------------|
-| 50  | 🔲     | [Redundancy / Duplication Detection](plan/50_redundancy-duplication-detection.md)                               |
+| 50  | ✅     | [Redundancy / Duplication Detection](plan/50_redundancy-duplication-detection.md)                               |
 | 51  | ✅     | [Section-Level Size Limits](plan/51_section-level-size-limits.md)                                               |
 | 52  | 🔲     | [Archetype / Template Library for Agentic Patterns](plan/52_archetype-template-library.md)                      |
 | 53  | ⛔     | [Conciseness Scoring](plan/53_conciseness-scoring.md)                                                           |
@@ -47,4 +47,6 @@ footer: |
 | 87  | ✅     | [Flavor validation for GitHub Alerts](plan/87_markdown-flavor-github-alerts.md)                                 |
 | 88  | ✅     | [TOC directive migration aid](plan/88_toc-directive-migration.md)                                               |
 | 89  | 🔲     | [TOC generator directive and MDS035 auto-fix](plan/89_toc-generator-directive.md)                               |
+| 90  | 🔲     | [Isolate corpus test git config from host signing](plan/90_corpus-test-git-config-isolation.md)                 |
+| 91  | 🔲     | [MDS037 skips paragraphs inside generated sections](plan/91_mds037-skip-generated-sections.md)                  |
 <?/catalog?>

cmd/mdsmith/main.go

@@ -30,6 +30,7 @@ import (
 	_ "github.com/jeduden/mdsmith/internal/rules/concisenessscoring"
 	_ "github.com/jeduden/mdsmith/internal/rules/crossfilereferenceintegrity"
 	_ "github.com/jeduden/mdsmith/internal/rules/directorystructure"
+	_ "github.com/jeduden/mdsmith/internal/rules/duplicatedcontent"
 	_ "github.com/jeduden/mdsmith/internal/rules/emptysectionbody"
 	_ "github.com/jeduden/mdsmith/internal/rules/fencedcodelanguage"
 	_ "github.com/jeduden/mdsmith/internal/rules/fencedcodestyle"

internal/config/config_test.go

@@ -18,6 +18,7 @@ import (
 	_ "github.com/jeduden/mdsmith/internal/rules/concisenessscoring"
 	_ "github.com/jeduden/mdsmith/internal/rules/crossfilereferenceintegrity"
 	_ "github.com/jeduden/mdsmith/internal/rules/directorystructure"
+	_ "github.com/jeduden/mdsmith/internal/rules/duplicatedcontent"
 	_ "github.com/jeduden/mdsmith/internal/rules/emptysectionbody"
 	_ "github.com/jeduden/mdsmith/internal/rules/fencedcodelanguage"
 	_ "github.com/jeduden/mdsmith/internal/rules/fencedcodestyle"

internal/engine/categories_test.go

@@ -18,6 +18,7 @@ import (
 	_ "github.com/jeduden/mdsmith/internal/rules/catalog"
 	_ "github.com/jeduden/mdsmith/internal/rules/crossfilereferenceintegrity"
 	_ "github.com/jeduden/mdsmith/internal/rules/directorystructure"
+	_ "github.com/jeduden/mdsmith/internal/rules/duplicatedcontent"
 	_ "github.com/jeduden/mdsmith/internal/rules/emptysectionbody"
 	_ "github.com/jeduden/mdsmith/internal/rules/fencedcodelanguage"
 	_ "github.com/jeduden/mdsmith/internal/rules/fencedcodestyle"

internal/integration/rules_test.go

@@ -23,6 +23,7 @@ import (
 	_ "github.com/jeduden/mdsmith/internal/rules/concisenessscoring"
 	_ "github.com/jeduden/mdsmith/internal/rules/crossfilereferenceintegrity"
 	"github.com/jeduden/mdsmith/internal/rules/directorystructure"
+	_ "github.com/jeduden/mdsmith/internal/rules/duplicatedcontent"
 	_ "github.com/jeduden/mdsmith/internal/rules/emptysectionbody"
 	_ "github.com/jeduden/mdsmith/internal/rules/fencedcodelanguage"
 	_ "github.com/jeduden/mdsmith/internal/rules/fencedcodestyle"

internal/lint/file.go

@@ -22,6 +22,13 @@ type File struct {
 	FrontMatter []byte
 	LineOffset  int
 
+	// StripFrontMatter records whether this file was parsed in
+	// front-matter-stripping mode. Rules that read other files
+	// from the corpus should mirror the same mode so that line
+	// numbers in cross-file diagnostics are computed against the
+	// same coordinate system as the current file.
+	StripFrontMatter bool
+
 	// MaxInputBytes is the maximum file size in bytes that rules
 	// should enforce when reading secondary files (includes, schemas,
 	// cross-references). Zero or negative means unlimited.
@@ -110,6 +117,7 @@ func NewFileFromSource(path string, source []byte, stripFrontMatter bool) (*File
 	}
 	f.FrontMatter = fm
 	f.LineOffset = offset
+	f.StripFrontMatter = stripFrontMatter
 	return f, nil
 }
 

internal/rules/MDS037-duplicated-content/README.md

@@ -0,0 +1,131 @@
+---
+id: MDS037
+name: duplicated-content
+status: ready
+description: Paragraphs should not repeat verbatim across Markdown files.
+---
+# MDS037: duplicated-content
+
+Paragraphs should not repeat verbatim across Markdown files.
+
+- **ID**: MDS037
+- **Name**: `duplicated-content`
+- **Status**: ready
+- **Default**: disabled (opt-in via `.mdsmith.yml`);
+  include: [], exclude: [], min-chars: 200
+- **Fixable**: no
+- **Implementation**:
+  [source](./)
+- **Category**: meta
+
+## Settings
+
+| Setting     | Type | Default | Description                                    |
+|-------------|------|---------|------------------------------------------------|
+| `include`   | list | `[]`    | glob patterns limiting which siblings to scan  |
+| `exclude`   | list | `[]`    | glob patterns of siblings to skip              |
+| `min-chars` | int  | `200`   | minimum normalized paragraph length to compare |
+
+Before comparing, the rule normalizes each paragraph. Whitespace
+collapses to single spaces. Letters become lowercase. Leading and
+trailing space is trimmed. A paragraph shorter than `min-chars` runes
+is skipped; short stubs would otherwise produce noise.
+
+The rule walks `RootFS` when the project root is known. Otherwise it
+falls back to the file's own directory. An `include` list narrows the
+scan to matching paths. An `exclude` entry takes precedence.
+
+## Performance
+
+Each checked file reads every other Markdown file in scope
+(`.md` and `.markdown`). A project
+with *N* Markdown files performs *O(N²)* reads. Small and medium
+corpora stay fast. For large corpora add an `exclude` entry for
+generated or vendored directories.
+
+## Config
+
+```yaml
+rules:
+  duplicated-content:
+    include:
+      - "docs/**"
+    exclude:
+      - "docs/generated/**"
+    min-chars: 200
+```
+
+Disable:
+
+```yaml
+rules:
+  duplicated-content: false
+```
+
+## Examples
+
+### Good
+
+<?include
+file: good/simple.md
+wrap: markdown
+?>
+
+```markdown
+# Simple Fixture
+
+One short fixture sits alone in its folder and exists to exercise
+the duplicate detector. Every other rule stays quiet because the
+text is simple and brief. The paragraph holds enough characters to
+pass two hundred runes after normalization. Each sentence is plain
+and ends early. No other file here repeats this wording.
+```
+
+<?/include?>
+
+### Bad -- duplicated paragraph
+
+<?include
+file: bad/duplicate.md
+wrap: markdown
+?>
+
+```markdown
+# Duplicate Fixture
+
+A distinctive paragraph appears in this file and in a sibling
+fixture, so MDS037 must flag the match and point at the other
+location. The wording stays above the default two-hundred character
+threshold after normalization. It stays unique relative to the
+other rule fixtures so nothing matches by accident across the test
+suite.
+```
+
+<?/include?>
+
+### Bad -- duplicated source
+
+<?include
+file: bad/ref/source.md
+wrap: markdown
+?>
+
+```markdown
+# Source Fixture
+
+A distinctive paragraph appears in this file and in a sibling
+fixture, so MDS037 must flag the match and point at the other
+location. The wording stays above the default two-hundred character
+threshold after normalization. It stays unique relative to the
+other rule fixtures so nothing matches by accident across the test
+suite.
+```
+
+<?/include?>
+
+## Diagnostics
+
+| Condition         | Message                                                                  |
+|-------------------|--------------------------------------------------------------------------|
+| paragraph repeats | paragraph duplicated in {other}:{line}                                   |
+| invalid glob      | duplicated-content: {include,exclude}: invalid glob pattern "{pat}": ... |

internal/rules/MDS037-duplicated-content/bad/duplicate.md

@@ -0,0 +1,14 @@
+---
+diagnostics:
+  - line: 3
+    column: 1
+    message: 'paragraph duplicated in ref/source.md:3'
+---
+# Duplicate Fixture
+
+A distinctive paragraph appears in this file and in a sibling
+fixture, so MDS037 must flag the match and point at the other
+location. The wording stays above the default two-hundred character
+threshold after normalization. It stays unique relative to the
+other rule fixtures so nothing matches by accident across the test
+suite.

internal/rules/MDS037-duplicated-content/bad/ref/source.md

@@ -0,0 +1,8 @@
+# Source Fixture
+
+A distinctive paragraph appears in this file and in a sibling
+fixture, so MDS037 must flag the match and point at the other
+location. The wording stays above the default two-hundred character
+threshold after normalization. It stays unique relative to the
+other rule fixtures so nothing matches by accident across the test
+suite.

internal/rules/MDS037-duplicated-content/good/simple.md

@@ -0,0 +1,7 @@
+# Simple Fixture
+
+One short fixture sits alone in its folder and exists to exercise
+the duplicate detector. Every other rule stays quiet because the
+text is simple and brief. The paragraph holds enough characters to
+pass two hundred runes after normalization. Each sentence is plain
+and ends early. No other file here repeats this wording.