Add frontmatter to docs, support clearer doc lists and mkdocs description

cmd/docs.go

@@ -7,10 +7,12 @@ import (
 	"io"
 	"io/fs"
 	"os"
+	"sort"
 	"strings"
 
 	"github.com/zix99/rare/cmd/helpers"
 	"github.com/zix99/rare/docs"
+	"github.com/zix99/rare/pkg/color"
 	"github.com/zix99/rare/pkg/markdowncli"
 
 	"github.com/urfave/cli/v2"
@@ -22,6 +24,8 @@ func docsFunction(c *cli.Context) error {
 	if docname == "" || docname == "list" {
 		listDocFiles()
 	} else if file, err := openDocFileByPartialName(docname); err == nil {
+		defer file.Close()
+
 		var buf bytes.Buffer
 		markdowncli.WriteMarkdownToBuf(&buf, file)
 		if c.Bool("no-pager") || helpers.TryWritePager(&buf) != nil {
@@ -35,10 +39,54 @@ func docsFunction(c *cli.Context) error {
 }
 
 func listDocFiles() {
-	fmt.Println("Available Docs:")
+	fmt.Println(color.Wrap(color.Bold, "Available Docs:"))
+
+	type docInfo struct {
+		name         string
+		summary      string
+		order, depth int
+	}
+
 	entries, _ := docs.DocFS.ReadDir(docs.BasePath)
+	docList := make([]docInfo, 0, len(entries))
+	maxNameLen := 1
 	for _, entry := range entries {
-		fmt.Printf("  %s\n", strings.TrimSuffix(entry.Name(), ".md"))
+		info := docInfo{
+			name: strings.TrimSuffix(entry.Name(), ".md"),
+		}
+		maxNameLen = max(maxNameLen, len(info.name))
+
+		r, err := docs.DocFS.Open(docs.BasePath + "/" + entry.Name())
+		if err == nil {
+			frontmatter := markdowncli.ExtractFrontmatter(r)
+			r.Close()
+
+			info.summary = frontmatter.Description()
+			info.order = frontmatter.Order()
+			info.depth = frontmatter.Depth()
+		}
+
+		docList = append(docList, info)
+	}
+
+	sort.Slice(docList, func(i, j int) bool {
+		di, dj := docList[i], docList[j]
+		if di.order != dj.order {
+			return di.order < dj.order
+		}
+		if di.depth != dj.depth {
+			return di.depth < dj.depth
+		}
+		return di.name < dj.name
+	})
+
+	for _, d := range docList {
+		fmt.Print(strings.Repeat("  ", d.depth+1))
+		fmt.Printf("%s%s", color.Wrap(color.BrightWhite, d.name), strings.Repeat(" ", maxNameLen-len(d.name)))
+		if d.summary != "" {
+			fmt.Print("  ", d.summary)
+		}
+		fmt.Println()
 	}
 }
 

docs/usage/aggregators.md

@@ -1,3 +1,7 @@
+---
+description: Available display aggregators, arguments and examples
+order: -1
+---
 # Aggregators
 
 *Aggregators* represent different ways to count and output data as it is processed

docs/usage/dissect.md

@@ -1,3 +1,8 @@
+---
+description: Dissect expression syntax
+order: 6
+depth: 1
+---
 # Dissect Syntax
 
 *Dissect* is a simple token-based search algorithm, and can

docs/usage/examples.md

@@ -1,3 +1,6 @@
+---
+description: Simple examples of using rare
+---
 # Examples
 
 !!! note

docs/usage/expressions.md

@@ -1,3 +1,7 @@
+---
+description: Expression syntax and functions
+order: 1
+---
 # Expressions
 
 *rare* expressions are handlebars-like in their ability to process data with 

docs/usage/extractor.md

@@ -1,3 +1,7 @@
+---
+description: Input data parsing overview (matcher)
+order: 5
+---
 # Extractor (Matcher)
 
 The main component of *rare* is the extractor (or matcher).  There are

docs/usage/funcsfile.md

@@ -1,3 +1,8 @@
+---
+description: Adding custom functions for expressions
+order: 1
+depth: 1
+---
 # Expression Functions File
 
 A *functions file* allows you to specify additional expression

docs/usage/input.md

@@ -1,3 +1,7 @@
+---
+description: Input methods and arguments
+order: 0
+---
 # Input
 
 *rare* reads the supplied inputs in massive parallelization, rather

docs/usage/json.md

@@ -1,3 +1,8 @@
+---
+description: JSON querying syntax
+order: 1
+depth: 1
+---
 # Json
 
 Syntax: `{json field expression}`

docs/usage/math.md

@@ -1,3 +1,8 @@
+---
+description: Mathematical formulas in expressions
+order: 1
+depth: 1
+---
 # Math
 
 Math expressions are evaluated using the `{! expr}` helper. They

docs/usage/overview.md

@@ -1,3 +1,7 @@
+---
+description: Top-level concepts, and data pipeline
+order: -99
+---
 # rare
 
 Rare is a fast, realtime regex-extraction, and aggregation into common formats

docs/usage/regexp.md

@@ -1,3 +1,8 @@
+---
+description: Regular expression syntax
+order: 5
+depth: 1
+---
 # Regexp Syntax
 
 Source: https://golang.org/pkg/regexp/syntax/

pkg/markdowncli/frontmatter.go

@@ -0,0 +1,50 @@
+package markdowncli
+
+import (
+	"bufio"
+	"io"
+	"strconv"
+	"strings"
+)
+
+type Frontmatter map[string]string
+
+func (s Frontmatter) Description() string {
+	return s["description"]
+}
+
+func (s Frontmatter) Order() int {
+	v, _ := strconv.Atoi(s["order"])
+	return v
+}
+
+func (s Frontmatter) Depth() int {
+	v, _ := strconv.Atoi(s["depth"])
+	return v
+}
+
+func ExtractFrontmatter(r io.Reader) Frontmatter {
+	ret := make(Frontmatter)
+
+	scanner := bufio.NewScanner(r)
+
+	scanner.Scan()
+	if scanner.Text() != "---" {
+		return ret
+	}
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line == "---" {
+			break
+		}
+		parts := strings.SplitN(line, ":", 2)
+		if len(parts) == 2 {
+			key := strings.TrimSpace(parts[0])
+			value := strings.TrimSpace(parts[1])
+			ret[key] = value
+		}
+	}
+
+	return ret
+}

pkg/markdowncli/frontmatter_test.go

@@ -0,0 +1,33 @@
+package markdowncli
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFrontmatterParse(t *testing.T) {
+	r := strings.NewReader(`---
+description: hi
+order: 1
+depth: 2
+---
+real data
+and more real data`)
+
+	fm := ExtractFrontmatter(r)
+	assert.Equal(t, "hi", fm.Description())
+	assert.Equal(t, 1, fm.Order())
+	assert.Equal(t, 2, fm.Depth())
+}
+
+func TestEmptyFrontmatter(t *testing.T) {
+	r := strings.NewReader(`real data
+	and new line`)
+
+	fm := ExtractFrontmatter(r)
+	assert.Equal(t, "", fm.Description())
+	assert.Equal(t, 0, fm.Order())
+	assert.Equal(t, 0, fm.Depth())
+}

pkg/markdowncli/mardowncli.go

@@ -26,10 +26,15 @@ func WriteMarkdownToBuf(out io.Writer, reader io.Reader) {
 	headerDepth := 0
 	isCodeBlock := false
 	inNoteBlock := false
+	isFrontmatter := false
 
 	for scanner.Scan() {
 		line := scanner.Text()
-		if strings.HasPrefix(line, tokenHeader) && !isCodeBlock && !inNoteBlock { // header
+		if line == "---" && headerDepth == 0 { // skip frontmatter
+			isFrontmatter = !isFrontmatter
+		} else if isFrontmatter {
+			continue
+		} else if strings.HasPrefix(line, tokenHeader) && !isCodeBlock && !inNoteBlock { // header
 			headerDepth = strings.Count(line, tokenHeader) - 1
 			headerColor := headerColors[headerDepth%len(headerColors)]
 			fmt.Fprintf(out, "%s%s\n", strings.Repeat(" ", headerDepth), color.Wrap(color.Bold, color.Wrap(headerColor, line)))

pkg/markdowncli/markdowncli_test.go

@@ -25,3 +25,11 @@ func TestNoteBlock(t *testing.T) {
 
 	assert.Equal(t, "# Title\n !!! note\n     this is a note block\n\n", w.String())
 }
+
+func TestSkipFrontmatter(t *testing.T) {
+	r := strings.NewReader("---\nsummary: hello\n---\n# Title\n!!! note\n    this is a note block\n\n")
+	w := &bytes.Buffer{}
+	WriteMarkdownToBuf(w, r)
+
+	assert.Equal(t, "# Title\n !!! note\n     this is a note block\n\n", w.String())
+}