conf: support optional include? directive

Add parsing support for `include? <path>` so missing include files are ignored.

Keep existing behavior for invalid include files and add lexer/parser tests.

Implements: #5297
Signed-off-by: Ollie Hensman-Crook <olliehensmancrook@gmail.com>

Author: olliehcrook

conf/lex.go

@@ -54,6 +54,7 @@ const (
 	itemCommentStart
 	itemVariable
 	itemInclude
+	itemIncludeOptional
 )
 
 const (
@@ -486,6 +487,12 @@ func (lx *lexer) keyCheckKeyword(fallThrough, push stateFn) stateFn {
 			lx.push(push)
 		}
 		return lexIncludeStart
+	case "include?":
+		lx.ignore()
+		if push != nil {
+			lx.push(push)
+		}
+		return lexIncludeOptionalStart
 	}
 	lx.emit(itemKey)
 	return fallThrough
@@ -501,6 +508,16 @@ func lexIncludeStart(lx *lexer) stateFn {
 	return lexInclude
 }
 
+// lexIncludeOptionalStart will consume the whitespace til the start of the value.
+func lexIncludeOptionalStart(lx *lexer) stateFn {
+	r := lx.next()
+	if isWhitespace(r) {
+		return lexSkip(lx, lexIncludeOptionalStart)
+	}
+	lx.backup()
+	return lexIncludeOptional
+}
+
 // lexIncludeQuotedString consumes the inner contents of a string. It assumes that the
 // beginning '"' has already been consumed and ignored. It will not interpret any
 // internal contents.
@@ -582,6 +599,83 @@ func lexInclude(lx *lexer) stateFn {
 	return lexIncludeString
 }
 
+// lexIncludeOptional will consume the optional include value.
+func lexIncludeOptional(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case r == sqStringStart:
+		lx.ignore() // ignore the " or '
+		return lexIncludeOptionalQuotedString
+	case r == dqStringStart:
+		lx.ignore() // ignore the " or '
+		return lexIncludeOptionalDubQuotedString
+	case r == arrayStart:
+		return lx.errorf("Expected include value but found start of an array")
+	case r == mapStart:
+		return lx.errorf("Expected include value but found start of a map")
+	case r == blockStart:
+		return lx.errorf("Expected include value but found start of a block")
+	case unicode.IsDigit(r), r == '-':
+		return lx.errorf("Expected include value but found start of a number")
+	case r == '\\':
+		return lx.errorf("Expected include value but found escape sequence")
+	case isNL(r):
+		return lx.errorf("Expected include value but found new line")
+	}
+	lx.backup()
+	return lexIncludeOptionalString
+}
+
+// lexIncludeOptionalQuotedString consumes the inner contents of a string.
+func lexIncludeOptionalQuotedString(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case r == sqStringEnd:
+		lx.backup()
+		lx.emit(itemIncludeOptional)
+		lx.next()
+		lx.ignore()
+		return lx.pop()
+	case r == eof:
+		return lx.errorf("Unexpected EOF in quoted include")
+	}
+	return lexIncludeOptionalQuotedString
+}
+
+// lexIncludeOptionalDubQuotedString consumes the inner contents of a string.
+func lexIncludeOptionalDubQuotedString(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case r == dqStringEnd:
+		lx.backup()
+		lx.emit(itemIncludeOptional)
+		lx.next()
+		lx.ignore()
+		return lx.pop()
+	case r == eof:
+		return lx.errorf("Unexpected EOF in double quoted include")
+	}
+	return lexIncludeOptionalDubQuotedString
+}
+
+// lexIncludeOptionalString consumes the inner contents of a raw string.
+func lexIncludeOptionalString(lx *lexer) stateFn {
+	r := lx.next()
+	switch {
+	case isNL(r) || r == eof || r == optValTerm || r == mapEnd || isWhitespace(r):
+		lx.backup()
+		lx.emit(itemIncludeOptional)
+		return lx.pop()
+	case r == sqStringEnd:
+		lx.backup()
+		lx.emit(itemIncludeOptional)
+		lx.next()
+		lx.ignore()
+		return lx.pop()
+	}
+	return lexIncludeOptionalString
+}
+
 // lexKey consumes the text of a key. Assumes that the first character (which
 // is not whitespace) has already been consumed.
 func lexKey(lx *lexer) stateFn {
@@ -1302,6 +1396,8 @@ func (itype itemType) String() string {
 		return "Variable"
 	case itemInclude:
 		return "Include"
+	case itemIncludeOptional:
+		return "IncludeOptional"
 	}
 	panic(fmt.Sprintf("BUG: Unknown type '%s'.", itype.String()))
 }

conf/lex_test.go

@@ -1412,6 +1412,25 @@ func TestInclude(t *testing.T) {
 	expect(t, lx, expectedItems)
 }
 
+func TestOptionalInclude(t *testing.T) {
+	expectedItems := []item{
+		{itemIncludeOptional, "users.conf", 1, 10},
+		{itemEOF, "", 1, 0},
+	}
+	lx := lex("include? \"users.conf\"")
+	expect(t, lx, expectedItems)
+
+	lx = lex("include? 'users.conf'")
+	expect(t, lx, expectedItems)
+
+	expectedItems = []item{
+		{itemIncludeOptional, "users.conf", 1, 9},
+		{itemEOF, "", 1, 0},
+	}
+	lx = lex("include? users.conf")
+	expect(t, lx, expectedItems)
+}
+
 func TestMapInclude(t *testing.T) {
 	expectedItems := []item{
 		{itemKey, "foo", 1, 0},
@@ -1455,6 +1474,18 @@ func TestMapInclude(t *testing.T) {
 	expect(t, lx, expectedItems)
 }
 
+func TestMapOptionalInclude(t *testing.T) {
+	expectedItems := []item{
+		{itemKey, "foo", 1, 0},
+		{itemMapStart, "", 1, 5},
+		{itemIncludeOptional, "users.conf", 1, 15},
+		{itemMapEnd, "", 1, 27},
+		{itemEOF, "", 1, 0},
+	}
+	lx := lex("foo { include? users.conf }")
+	expect(t, lx, expectedItems)
+}
+
 func TestJSONCompat(t *testing.T) {
 	for _, test := range []struct {
 		name     string

conf/parse.go

@@ -89,7 +89,7 @@ func ParseWithChecks(data string) (map[string]any, error) {
 func ParseFile(fp string) (map[string]any, error) {
 	data, err := os.ReadFile(fp)
 	if err != nil {
-		return nil, fmt.Errorf("error opening config file: %v", err)
+		return nil, fmt.Errorf("error opening config file: %w", err)
 	}
 
 	p, err := parse(string(data), fp, false)
@@ -435,6 +435,33 @@ func (p *parser) processItem(it item, fp string) error {
 		for k, v := range m {
 			p.pushKey(k)
 
+			if p.pedantic {
+				switch tk := v.(type) {
+				case *token:
+					p.pushItemKey(tk.item)
+				}
+			}
+			p.setValue(v)
+		}
+	case itemIncludeOptional:
+		var (
+			m   map[string]any
+			err error
+		)
+		if p.pedantic {
+			m, err = ParseFileWithChecks(filepath.Join(p.fp, it.val))
+		} else {
+			m, err = ParseFile(filepath.Join(p.fp, it.val))
+		}
+		if err != nil {
+			if os.IsNotExist(err) || strings.Contains(strings.ToLower(err.Error()), "no such file or directory") {
+				break
+			}
+			return fmt.Errorf("error parsing include file '%s', %v", it.val, err)
+		}
+		for k, v := range m {
+			p.pushKey(k)
+
 			if p.pedantic {
 				switch tk := v.(type) {
 				case *token:

conf/parse_test.go

@@ -364,6 +364,56 @@ func TestIncludes(t *testing.T) {
 	}
 }
 
+func TestOptionalIncludeMissingFile(t *testing.T) {
+	sdir := t.TempDir()
+	cfg := `
+listen: 127.0.0.1:4222
+include? ./nats-cluster.conf
+`
+	fp := filepath.Join(sdir, "nats.conf")
+	if err := os.WriteFile(fp, []byte(cfg), 0666); err != nil {
+		t.Fatal(err)
+	}
+
+	m, err := ParseFile(fp)
+	if err != nil {
+		t.Fatalf("Received err: %v\n", err)
+	}
+	if got := m["listen"]; got != "127.0.0.1:4222" {
+		t.Fatalf("Expected listen to be set, got: %v", got)
+	}
+
+	m, err = ParseFileWithChecks(fp)
+	if err != nil {
+		t.Fatalf("Received err: %v\n", err)
+	}
+	if got := m["listen"]; got.(*token).Value() != "127.0.0.1:4222" {
+		t.Fatalf("Expected listen to be set, got: %v", got)
+	}
+}
+
+func TestOptionalIncludeInvalidFile(t *testing.T) {
+	sdir := t.TempDir()
+	cfg := `
+listen: 127.0.0.1:4222
+include? ./nats-cluster.conf
+`
+	fp := filepath.Join(sdir, "nats.conf")
+	if err := os.WriteFile(fp, []byte(cfg), 0666); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(sdir, "nats-cluster.conf"), []byte("?????????????"), 0666); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := ParseFile(fp); err == nil {
+		t.Fatal("expected an error")
+	}
+	if _, err := ParseFileWithChecks(fp); err == nil {
+		t.Fatal("expected an error")
+	}
+}
+
 var varIncludedVariablesSample = `
 authorization {