From f4837dc7396aea17344080e8a3ed012c508b267f Mon Sep 17 00:00:00 2001 From: Christopher LaPointe Date: Wed, 28 Jan 2026 22:06:35 -0500 Subject: [PATCH 1/4] Add frontmatter to docs, support clearer doc lists and mkdocs description --- cmd/docs.go | 51 +++++++++++++++++++++++++++-- docs/usage/aggregators.md | 4 +++ docs/usage/dissect.md | 5 +++ docs/usage/examples.md | 3 ++ docs/usage/expressions.md | 4 +++ docs/usage/extractor.md | 4 +++ docs/usage/funcsfile.md | 5 +++ docs/usage/input.md | 4 +++ docs/usage/json.md | 5 +++ docs/usage/math.md | 5 +++ docs/usage/overview.md | 4 +++ docs/usage/regexp.md | 5 +++ pkg/markdowncli/frontmatter.go | 50 ++++++++++++++++++++++++++++ pkg/markdowncli/frontmatter_test.go | 33 +++++++++++++++++++ pkg/markdowncli/mardowncli.go | 7 +++- pkg/markdowncli/markdowncli_test.go | 8 +++++ 16 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 pkg/markdowncli/frontmatter.go create mode 100644 pkg/markdowncli/frontmatter_test.go diff --git a/cmd/docs.go b/cmd/docs.go index 4ccc0471..e9df7f8b 100644 --- a/cmd/docs.go +++ b/cmd/docs.go @@ -7,10 +7,12 @@ import ( "io" "io/fs" "os" + "sort" "strings" "github.com/zix99/rare/cmd/helpers" "github.com/zix99/rare/docs" + "github.com/zix99/rare/pkg/color" "github.com/zix99/rare/pkg/markdowncli" "github.com/urfave/cli/v2" @@ -22,6 +24,8 @@ func docsFunction(c *cli.Context) error { if docname == "" || docname == "list" { listDocFiles() } else if file, err := openDocFileByPartialName(docname); err == nil { + defer file.Close() + var buf bytes.Buffer markdowncli.WriteMarkdownToBuf(&buf, file) if c.Bool("no-pager") || helpers.TryWritePager(&buf) != nil { @@ -35,10 +39,53 @@ func docsFunction(c *cli.Context) error { } func listDocFiles() { - fmt.Println("Available Docs:") + fmt.Println(color.Wrap(color.Bold, "Available Docs:")) + + type docInfo struct { + name string + summary string + order, depth int + } + entries, _ := docs.DocFS.ReadDir(docs.BasePath) + docList := make([]docInfo, 0, len(entries)) + maxNameLen := 1 for _, entry := range entries { - fmt.Printf(" %s\n", strings.TrimSuffix(entry.Name(), ".md")) + info := docInfo{ + name: strings.TrimSuffix(entry.Name(), ".md"), + } + maxNameLen = max(maxNameLen, len(info.name)) + + r, err := docs.DocFS.Open(docs.BasePath + "/" + entry.Name()) + if err == nil { + defer r.Close() + frontmatter := markdowncli.ExtractFrontmatter(r) + info.summary = frontmatter.Description() + info.order = frontmatter.Order() + info.depth = frontmatter.Depth() + } + + docList = append(docList, info) + } + + sort.Slice(docList, func(i, j int) bool { + di, dj := docList[i], docList[j] + if di.order != dj.order { + return di.order < dj.order + } + if di.depth != dj.depth { + return di.depth < dj.depth + } + return di.name < dj.name + }) + + for _, d := range docList { + fmt.Print(strings.Repeat(" ", d.depth+1)) + fmt.Printf("%s%s", color.Wrap(color.BrightWhite, d.name), strings.Repeat(" ", maxNameLen-len(d.name))) + if d.summary != "" { + fmt.Print(" ", d.summary) + } + fmt.Println() } } diff --git a/docs/usage/aggregators.md b/docs/usage/aggregators.md index 0a506018..d1594cb7 100644 --- a/docs/usage/aggregators.md +++ b/docs/usage/aggregators.md @@ -1,3 +1,7 @@ +--- +description: Available display aggregators, arguments and examples +order: -1 +--- # Aggregators *Aggregators* represent different ways to count and output data as it is processed diff --git a/docs/usage/dissect.md b/docs/usage/dissect.md index 919c647d..eed6e808 100644 --- a/docs/usage/dissect.md +++ b/docs/usage/dissect.md @@ -1,3 +1,8 @@ +--- +description: Dissect expression syntax +order: 6 +depth: 1 +--- # Dissect Syntax *Dissect* is a simple token-based search algorithm, and can diff --git a/docs/usage/examples.md b/docs/usage/examples.md index e6612419..932ea865 100644 --- a/docs/usage/examples.md +++ b/docs/usage/examples.md @@ -1,3 +1,6 @@ +--- +description: Simple examples of using rare +--- # Examples !!! note diff --git a/docs/usage/expressions.md b/docs/usage/expressions.md index 68cc4f61..d50926c7 100644 --- a/docs/usage/expressions.md +++ b/docs/usage/expressions.md @@ -1,3 +1,7 @@ +--- +description: Expression syntax and functions +order: 1 +--- # Expressions *rare* expressions are handlebars-like in their ability to process data with diff --git a/docs/usage/extractor.md b/docs/usage/extractor.md index 0f4cc453..b4537f6a 100644 --- a/docs/usage/extractor.md +++ b/docs/usage/extractor.md @@ -1,3 +1,7 @@ +--- +description: Data extraction overview +order: 5 +--- # Extractor (Matcher) The main component of *rare* is the extractor (or matcher). There are diff --git a/docs/usage/funcsfile.md b/docs/usage/funcsfile.md index d273d348..e8a986a6 100644 --- a/docs/usage/funcsfile.md +++ b/docs/usage/funcsfile.md @@ -1,3 +1,8 @@ +--- +description: Adding custom functions for expressions +order: 1 +depth: 1 +--- # Expression Functions File A *functions file* allows you to specify additional expression diff --git a/docs/usage/input.md b/docs/usage/input.md index 4b0f33f7..7a6d10c9 100644 --- a/docs/usage/input.md +++ b/docs/usage/input.md @@ -1,3 +1,7 @@ +--- +description: Input methods and arguments +order: 0 +--- # Input *rare* reads the supplied inputs in massive parallelization, rather diff --git a/docs/usage/json.md b/docs/usage/json.md index 1214e2f6..56ef2243 100644 --- a/docs/usage/json.md +++ b/docs/usage/json.md @@ -1,3 +1,8 @@ +--- +description: JSON querying syntax +order: 1 +depth: 1 +--- # Json Syntax: `{json field expression}` diff --git a/docs/usage/math.md b/docs/usage/math.md index fb4bc102..dcf4b220 100644 --- a/docs/usage/math.md +++ b/docs/usage/math.md @@ -1,3 +1,8 @@ +--- +description: Expressing mathematical expressions +order: 1 +depth: 1 +--- # Math Math expressions are evaluated using the `{! expr}` helper. They diff --git a/docs/usage/overview.md b/docs/usage/overview.md index 578126d1..84ef91e5 100644 --- a/docs/usage/overview.md +++ b/docs/usage/overview.md @@ -1,3 +1,7 @@ +--- +description: Top-level concepts, and data pipeline +order: -99 +--- # rare Rare is a fast, realtime regex-extraction, and aggregation into common formats diff --git a/docs/usage/regexp.md b/docs/usage/regexp.md index 9735fab2..26040a6e 100644 --- a/docs/usage/regexp.md +++ b/docs/usage/regexp.md @@ -1,3 +1,8 @@ +--- +description: Regular expression syntax +order: 5 +depth: 1 +--- # Regexp Syntax Source: https://golang.org/pkg/regexp/syntax/ diff --git a/pkg/markdowncli/frontmatter.go b/pkg/markdowncli/frontmatter.go new file mode 100644 index 00000000..e82cc2b8 --- /dev/null +++ b/pkg/markdowncli/frontmatter.go @@ -0,0 +1,50 @@ +package markdowncli + +import ( + "bufio" + "io" + "strconv" + "strings" +) + +type Frontmatter map[string]string + +func (s Frontmatter) Description() string { + return s["description"] +} + +func (s Frontmatter) Order() int { + v, _ := strconv.Atoi(s["order"]) + return v +} + +func (s Frontmatter) Depth() int { + v, _ := strconv.Atoi(s["depth"]) + return v +} + +func ExtractFrontmatter(r io.Reader) Frontmatter { + ret := make(Frontmatter) + + scanner := bufio.NewScanner(r) + + scanner.Scan() + if scanner.Text() != "---" { + return ret + } + + for scanner.Scan() { + line := scanner.Text() + if line == "---" { + break + } + parts := strings.SplitN(line, ":", 2) + if len(parts) == 2 { + key := strings.TrimSpace(parts[0]) + value := strings.TrimSpace(parts[1]) + ret[key] = value + } + } + + return ret +} diff --git a/pkg/markdowncli/frontmatter_test.go b/pkg/markdowncli/frontmatter_test.go new file mode 100644 index 00000000..7bffe0e6 --- /dev/null +++ b/pkg/markdowncli/frontmatter_test.go @@ -0,0 +1,33 @@ +package markdowncli + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFrontmatterParse(t *testing.T) { + r := strings.NewReader(`--- +summary: hi +order: 1 +depth: 2 +--- +real data +and more real data`) + + fm := ExtractFrontmatter(r) + assert.Equal(t, "hi", fm.Description()) + assert.Equal(t, 1, fm.Order()) + assert.Equal(t, 2, fm.Depth()) +} + +func TestEmptyFrontmatter(t *testing.T) { + r := strings.NewReader(`real data + and new line`) + + fm := ExtractFrontmatter(r) + assert.Equal(t, "", fm.Description()) + assert.Equal(t, 0, fm.Order()) + assert.Equal(t, 0, fm.Depth()) +} diff --git a/pkg/markdowncli/mardowncli.go b/pkg/markdowncli/mardowncli.go index 94d2c4ca..c24e9932 100644 --- a/pkg/markdowncli/mardowncli.go +++ b/pkg/markdowncli/mardowncli.go @@ -26,10 +26,15 @@ func WriteMarkdownToBuf(out io.Writer, reader io.Reader) { headerDepth := 0 isCodeBlock := false inNoteBlock := false + isFrontmatter := false for scanner.Scan() { line := scanner.Text() - if strings.HasPrefix(line, tokenHeader) && !isCodeBlock && !inNoteBlock { // header + if line == "---" && headerDepth == 0 { // skip frontmatter + isFrontmatter = !isFrontmatter + } else if isFrontmatter { + continue + } else if strings.HasPrefix(line, tokenHeader) && !isCodeBlock && !inNoteBlock { // header headerDepth = strings.Count(line, tokenHeader) - 1 headerColor := headerColors[headerDepth%len(headerColors)] fmt.Fprintf(out, "%s%s\n", strings.Repeat(" ", headerDepth), color.Wrap(color.Bold, color.Wrap(headerColor, line))) diff --git a/pkg/markdowncli/markdowncli_test.go b/pkg/markdowncli/markdowncli_test.go index 8738e9a9..eb5620cf 100644 --- a/pkg/markdowncli/markdowncli_test.go +++ b/pkg/markdowncli/markdowncli_test.go @@ -25,3 +25,11 @@ func TestNoteBlock(t *testing.T) { assert.Equal(t, "# Title\n !!! note\n this is a note block\n\n", w.String()) } + +func TestSkipFrontmatter(t *testing.T) { + r := strings.NewReader("---\nsummary: hello\n---\n# Title\n!!! note\n this is a note block\n\n") + w := &bytes.Buffer{} + WriteMarkdownToBuf(w, r) + + assert.Equal(t, "# Title\n !!! note\n this is a note block\n\n", w.String()) +} From 4cfe102b004d054fdba57ff219e824753599b9d3 Mon Sep 17 00:00:00 2001 From: Christopher LaPointe Date: Wed, 28 Jan 2026 22:09:42 -0500 Subject: [PATCH 2/4] Clarifying descriptions --- docs/usage/extractor.md | 2 +- docs/usage/math.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/usage/extractor.md b/docs/usage/extractor.md index b4537f6a..4cff3fdb 100644 --- a/docs/usage/extractor.md +++ b/docs/usage/extractor.md @@ -1,5 +1,5 @@ --- -description: Data extraction overview +description: Input data parsing overview (matcher) order: 5 --- # Extractor (Matcher) diff --git a/docs/usage/math.md b/docs/usage/math.md index dcf4b220..a62ab2cd 100644 --- a/docs/usage/math.md +++ b/docs/usage/math.md @@ -1,5 +1,5 @@ --- -description: Expressing mathematical expressions +description: Mathematical formulas in expressions order: 1 depth: 1 --- From 1db9c26e143386b07e2df42fa3f4a1ae52cc3e7c Mon Sep 17 00:00:00 2001 From: Christopher LaPointe Date: Wed, 28 Jan 2026 22:44:33 -0500 Subject: [PATCH 3/4] Fix test --- pkg/markdowncli/frontmatter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/markdowncli/frontmatter_test.go b/pkg/markdowncli/frontmatter_test.go index 7bffe0e6..6f33ce80 100644 --- a/pkg/markdowncli/frontmatter_test.go +++ b/pkg/markdowncli/frontmatter_test.go @@ -9,7 +9,7 @@ import ( func TestFrontmatterParse(t *testing.T) { r := strings.NewReader(`--- -summary: hi +description: hi order: 1 depth: 2 --- From 1c9f5cb40617f5fcd1e17f4e4a9a68b03d0493bd Mon Sep 17 00:00:00 2001 From: Christopher LaPointe Date: Wed, 28 Jan 2026 23:28:42 -0500 Subject: [PATCH 4/4] Close docs as read --- cmd/docs.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/docs.go b/cmd/docs.go index e9df7f8b..610ff653 100644 --- a/cmd/docs.go +++ b/cmd/docs.go @@ -58,8 +58,9 @@ func listDocFiles() { r, err := docs.DocFS.Open(docs.BasePath + "/" + entry.Name()) if err == nil { - defer r.Close() frontmatter := markdowncli.ExtractFrontmatter(r) + r.Close() + info.summary = frontmatter.Description() info.order = frontmatter.Order() info.depth = frontmatter.Depth()