MDS037: Skip paragraphs inside generated sections

PLAN.md

@@ -26,5 +26,5 @@ footer: |
 | 86  | 🔳     | [Markdown flavor validation](plan/86_markdown-flavor-validation.md)                                  |
 | 89  | 🔲     | [TOC generator directive and MDS035 auto-fix](plan/89_toc-generator-directive.md)                    |
 | 90  | ✅     | [Isolate corpus test git config from host signing](plan/90_corpus-test-git-config-isolation.md)      |
-| 91  | 🔲     | [MDS037 skips paragraphs inside generated sections](plan/91_mds037-skip-generated-sections.md)       |
+| 91  | ✅     | [MDS037 skips paragraphs inside generated sections](plan/91_mds037-skip-generated-sections.md)       |
 <?/catalog?>

internal/rules/MDS037-duplicated-content/README.md

@@ -35,6 +35,18 @@ The rule walks `RootFS` when the project root is known. Otherwise it
 falls back to the file's own directory. An `include` list narrows the
 scan to matching paths. An `exclude` entry takes precedence.
 
+## Generated sections
+
+Paragraphs inside `<?include?>` and `<?catalog?>` directive bodies are
+skipped automatically. This applies to the file being checked and to
+every corpus file scanned for matches.
+
+These paragraphs are copies of content owned by another file. Flagging
+them would produce false positives on any project that uses generated
+sections. The same skip applies during corpus indexing: a host file's
+generated body is not added to the index. That prevents the original
+source file from matching its own text in a host's generated copy.
+
 ## Performance
 
 Each checked file reads every other Markdown file in scope

internal/rules/duplicatedcontent/rule.go

@@ -131,19 +131,82 @@ type externalMatch struct {
 	line int
 }
 
+// generatedRanges returns the [start, stop) byte ranges that cover the
+// body of generated sections (<?include?> and <?catalog?>). Only
+// top-level, well-formed open/close pairs produce a range; malformed or
+// unmatched markers are silently skipped, which is safe because the
+// generated-section rule (MDS031/MDS032) handles those errors separately.
+//
+// Nested same-name pairs (an inner <?include?> inside an outer
+// <?include?> body) are handled with a depth counter so the outer range
+// does not close prematurely on the inner end marker.
+func generatedRanges(f *lint.File) [][2]int {
+	if f.AST == nil {
+		return nil
+	}
+	var ranges [][2]int
+	var openPI *lint.ProcessingInstruction
+	depth := 0
+	for n := f.AST.FirstChild(); n != nil; n = n.NextSibling() {
+		pi, ok := n.(*lint.ProcessingInstruction)
+		if !ok {
+			continue
+		}
+		if openPI == nil {
+			if (pi.Name == "include" || pi.Name == "catalog") && pi.HasClosure() {
+				openPI = pi
+				depth = 0
+			}
+		} else if pi.Name == openPI.Name && pi.HasClosure() {
+			depth++
+		} else if pi.Name == "/"+openPI.Name && pi.HasClosure() && pi.Lines().Len() > 0 {
+			if depth > 0 {
+				depth--
+			} else {
+				start := openPI.ClosureLine.Stop
+				stop := pi.Lines().At(0).Start
+				if stop > start {
+					ranges = append(ranges, [2]int{start, stop})
+				}
+				openPI = nil
+			}
+		}
+	}
+	return ranges
+}
+
+// inGeneratedRange reports whether offset falls within any of the given
+// [start, stop) byte ranges.
+func inGeneratedRange(offset int, ranges [][2]int) bool {
+	for _, r := range ranges {
+		if offset >= r[0] && offset < r[1] {
+			return true
+		}
+	}
+	return false
+}
+
 // extractParagraphs walks f.AST and returns fingerprints for every
 // paragraph whose normalized text is at least minChars runes long.
 // Paragraphs are read via Node.Lines so raw markdown text — not rendered
 // inline output — feeds the fingerprint. Paragraphs with no source lines
-// (a shape goldmark never produces today, but cheap to guard) and ones
-// shorter than the threshold are skipped via the same min-chars gate.
+// (a shape goldmark never produces today, but cheap to guard), ones
+// shorter than the threshold, and paragraphs inside generated sections
+// (<?include?> or <?catalog?> bodies) are skipped.
 func extractParagraphs(f *lint.File, minChars int) []paragraph {
+	genRanges := generatedRanges(f)
 	var out []paragraph
 	_ = ast.Walk(f.AST, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
 		if !entering || n.Kind() != ast.KindParagraph {
 			return ast.WalkContinue, nil
 		}
 		lines := n.Lines()
+		if lines.Len() == 0 {
+			return ast.WalkSkipChildren, nil
+		}
+		if inGeneratedRange(lines.At(0).Start, genRanges) {
+			return ast.WalkSkipChildren, nil
+		}
 		var b strings.Builder
 		for i := 0; i < lines.Len(); i++ {
 			seg := lines.At(i)

internal/rules/duplicatedcontent/rule_test.go

@@ -675,6 +675,182 @@ func TestRootRelative_RelErrorOnRelativeRoot(t *testing.T) {
 	assert.Empty(t, got)
 }
 
+func TestCheck_SkipsIncludeGeneratedSection(t *testing.T) {
+	// A paragraph inside an <?include?> generated section must not be
+	// flagged as a duplicate, even when the same text appears in
+	// another corpus file (it's the source of the inclusion).
+	dir := t.TempDir()
+	p := longParagraph("the quick brown fox jumps over the lazy dog")
+
+	// source.md contains the paragraph that gets included.
+	writeFile(t, filepath.Join(dir, "source.md"), "# Source\n\n"+p+"\n")
+
+	// host.md includes source.md; the generated body holds the same paragraph.
+	host := "# Host\n\n" +
+		"<?include\nfile: source.md\n?>\n" +
+		p + "\n" +
+		"<?/include?>\n"
+	writeFile(t, filepath.Join(dir, "host.md"), host)
+
+	f := newLintFileWithRoot(t, filepath.Join(dir, "host.md"), dir)
+	diags := (&Rule{MinChars: 10}).Check(f)
+	assert.Empty(t, diags,
+		"paragraph inside <?include?> body must not be flagged as a duplicate")
+}
+
+func TestCheck_SkipsCatalogGeneratedSection(t *testing.T) {
+	// A paragraph inside a <?catalog?> generated section must not be
+	// flagged even when the same text appears in another corpus file.
+	dir := t.TempDir()
+	p := longParagraph("the quick brown fox jumps over the lazy dog")
+
+	writeFile(t, filepath.Join(dir, "source.md"), "# Source\n\n"+p+"\n")
+
+	host := "# Host\n\n" +
+		"<?catalog\nglob: \"*.md\"\n?>\n" +
+		p + "\n" +
+		"<?/catalog?>\n"
+	writeFile(t, filepath.Join(dir, "host.md"), host)
+
+	f := newLintFileWithRoot(t, filepath.Join(dir, "host.md"), dir)
+	diags := (&Rule{MinChars: 10}).Check(f)
+	assert.Empty(t, diags,
+		"paragraph inside <?catalog?> body must not be flagged as a duplicate")
+}
+
+func TestCheck_DuplicateOutsideGeneratedSectionStillFires(t *testing.T) {
+	// A paragraph outside any generated section must still be flagged
+	// when it appears verbatim in another corpus file.
+	dir := t.TempDir()
+	p := longParagraph("the quick brown fox jumps over the lazy dog")
+
+	// both.md has the paragraph before the generated section.
+	both := "# Both\n\n" +
+		p + "\n\n" +
+		"<?include\nfile: source.md\n?>\n" +
+		"generated content goes here and is definitely not the same\n" +
+		"<?/include?>\n"
+	writeFile(t, filepath.Join(dir, "both.md"), both)
+	writeFile(t, filepath.Join(dir, "other.md"), "# Other\n\n"+p+"\n")
+
+	f := newLintFileWithRoot(t, filepath.Join(dir, "both.md"), dir)
+	diags := (&Rule{MinChars: 10}).Check(f)
+	require.Len(t, diags, 1,
+		"real duplicate outside generated section must still fire")
+	assert.Contains(t, diags[0].Message, "other.md")
+}
+
+func TestCheck_CorpusSkipsIncludeGeneratedSection(t *testing.T) {
+	// When a corpus file contains an <?include?> generated section,
+	// the paragraphs inside it must not be indexed. Otherwise a host
+	// file checking its own (non-generated) paragraph against the
+	// corpus would find a false match in the corpus file's generated body.
+	dir := t.TempDir()
+	p := longParagraph("the quick brown fox jumps over the lazy dog")
+
+	// corpus-host.md has the same paragraph inside a generated section.
+	corpusHost := "# CorpusHost\n\n" +
+		"<?include\nfile: source.md\n?>\n" +
+		p + "\n" +
+		"<?/include?>\n"
+	writeFile(t, filepath.Join(dir, "corpus-host.md"), corpusHost)
+
+	// current.md has the paragraph as real content.
+	writeFile(t, filepath.Join(dir, "current.md"), "# Current\n\n"+p+"\n")
+
+	f := newLintFileWithRoot(t, filepath.Join(dir, "current.md"), dir)
+	diags := (&Rule{MinChars: 10}).Check(f)
+	assert.Empty(t, diags,
+		"paragraph inside corpus file's generated section must not be indexed")
+}
+
+func TestGeneratedRanges_EmptyFile(t *testing.T) {
+	f, err := lint.NewFile("test.md", []byte("# Hello\n"))
+	require.NoError(t, err)
+	assert.Empty(t, generatedRanges(f))
+}
+
+func TestGeneratedRanges_SingleIncludePair(t *testing.T) {
+	src := "<?include\nfile: x.md\n?>\nsome content\n<?/include?>\n"
+	f, err := lint.NewFile("test.md", []byte(src))
+	require.NoError(t, err)
+	ranges := generatedRanges(f)
+	require.Len(t, ranges, 1)
+	// Range must cover "some content\n" but not the PI markers.
+	contentStart := strings.Index(src, "some content")
+	contentEnd := strings.Index(src, "<?/include?>")
+	assert.Equal(t, contentStart, ranges[0][0])
+	assert.Equal(t, contentEnd, ranges[0][1])
+}
+
+func TestGeneratedRanges_MultiplePairs(t *testing.T) {
+	src := "<?include\nfile: a.md\n?>\ncontent a\n<?/include?>\n" +
+		"<?catalog\nglob: \"*.md\"\n?>\ncontent b\n<?/catalog?>\n"
+	f, err := lint.NewFile("test.md", []byte(src))
+	require.NoError(t, err)
+	ranges := generatedRanges(f)
+	assert.Len(t, ranges, 2)
+}
+
+func TestGeneratedRanges_NestedSameNamePair(t *testing.T) {
+	// An inner <?include?> inside an outer <?include?> body must not
+	// prematurely close the outer range. The outer range must span all
+	// content between the outer open and outer close markers.
+	src := "<?include\nfile: outer.md\n?>\n" +
+		"before inner\n" +
+		"<?include\nfile: inner.md\n?>\n" +
+		"inner content\n" +
+		"<?/include?>\n" +
+		"after inner\n" +
+		"<?/include?>\n"
+	f, err := lint.NewFile("test.md", []byte(src))
+	require.NoError(t, err)
+	ranges := generatedRanges(f)
+	require.Len(t, ranges, 1, "nested pair must produce exactly one outer range")
+	// The range must cover everything from after the outer open to before
+	// the outer close, so both "before inner" and "after inner" are inside it.
+	beforeInnerOffset := strings.Index(src, "before inner")
+	afterInnerOffset := strings.Index(src, "after inner")
+	// Find the *last* <?/include?> — that's the outer close.
+	lastCloseOffset := strings.LastIndex(src, "<?/include?>")
+	assert.LessOrEqual(t, ranges[0][0], beforeInnerOffset,
+		"range start must be before 'before inner'")
+	assert.Greater(t, ranges[0][1], afterInnerOffset,
+		"range end must be after 'after inner'")
+	assert.Equal(t, lastCloseOffset, ranges[0][1],
+		"range end must point to the outer <?/include?> marker")
+}
+
+func TestCheck_SkipsNestedIncludeGeneratedSection(t *testing.T) {
+	// When the outer generated body contains a nested <?include?> pair,
+	// paragraphs appearing AFTER the inner pair (but still inside the
+	// outer body) must also be skipped.
+	dir := t.TempDir()
+	p := longParagraph("the quick brown fox jumps over the lazy dog")
+
+	writeFile(t, filepath.Join(dir, "source.md"), "# Source\n\n"+p+"\n")
+
+	host := "# Host\n\n" +
+		"<?include\nfile: outer.md\n?>\n" +
+		"before inner paragraph is filler content not a real paragraph\n\n" +
+		"<?include\nfile: inner.md\n?>\n" +
+		p + "\n" +
+		"<?/include?>\n" +
+		p + "\n" +
+		"<?/include?>\n"
+	writeFile(t, filepath.Join(dir, "host.md"), host)
+
+	f := newLintFileWithRoot(t, filepath.Join(dir, "host.md"), dir)
+	diags := (&Rule{MinChars: 10}).Check(f)
+	assert.Empty(t, diags,
+		"paragraph after inner nested <?include?> but inside outer must not be flagged")
+}
+
+func TestGeneratedRanges_NilAST(t *testing.T) {
+	f := &lint.File{}
+	assert.Empty(t, generatedRanges(f))
+}
+
 func TestRootRelative_AbsErrorWhenCWDIsRemoved(t *testing.T) {
 	// filepath.Abs errors when os.Getwd fails, which happens when
 	// the process's current directory was removed underneath it.

plan/91_mds037-skip-generated-sections.md

@@ -1,7 +1,7 @@
 ---
 id: 91
 title: MDS037 skips paragraphs inside generated sections
-status: "🔲"
+status: "✅"
 summary: >-
   Paragraphs inside `<?include?>` and `<?catalog?>`
   directive bodies are copies of content owned by
@@ -49,35 +49,35 @@ without re-parsing the file.
 
 ## Tasks
 
-1. Walk top-level AST once to build a list of
+1. [x] Walk top-level AST once to build a list of
    generated-section byte ranges. A range starts at
    the opening PI's segment end and closes at the
    closing PI's segment start.
-2. In `extractParagraphs`, skip any paragraph whose
+2. [x] In `extractParagraphs`, skip any paragraph whose
    first-line byte offset sits inside one of those
    ranges.
-3. Likewise skip the paragraph when the rule walks
+3. [x] Likewise skip the paragraph when the rule walks
    *other* files during corpus indexing — otherwise
    a host file would still match against source text
    the other file only carries because of its own
    generated section.
-4. Update the README to document this behavior and
+4. [x] Update the README to document this behavior and
    drop the recommendation to hand-exclude the host
    files.
-5. Add tests: include-expanded paragraph in host is
+5. [x] Add tests: include-expanded paragraph in host is
    not flagged; catalog-rendered paragraph in host
    is not flagged; duplicates outside generated
    sections still fire.
 
 ## Acceptance Criteria
 
-- [ ] A paragraph in a file's body whose bytes sit
+- [x] A paragraph in a file's body whose bytes sit
       between `<?include?>` / `<?/include?>` markers
       does not produce an MDS037 diagnostic.
-- [ ] Same for `<?catalog?>` / `<?/catalog?>`.
-- [ ] A real duplicate outside any generated
+- [x] Same for `<?catalog?>` / `<?/catalog?>`.
+- [x] A real duplicate outside any generated
       section still fires.
-- [ ] `mdsmith check .` stays clean on this repo
+- [x] `mdsmith check .` stays clean on this repo
       with MDS037 enabled in `.mdsmith.yml`.
-- [ ] All tests pass: `go test ./...`
-- [ ] `go tool golangci-lint run` reports no issues
+- [x] All tests pass: `go test ./...`
+- [x] `go tool golangci-lint run` reports no issues