Skip to content

Commit 7b9e977

Browse files
committed
Add cmd.exe line continuation, grouping parens, comma delimiter, and stream redirects
Handle edge cases discovered through deep research of cmd.exe's actual lexing behavior: caret-newline line continuation, ( ) command grouping operators, comma as word delimiter, and numeric stream redirects (2>, 2>&1) via PostProcess. Fix caret to be literal inside double quotes (matching cmd.exe's Phase 2 parser where only " and <LF> are special inside quotes) using a new EscapeNotInEscapingQuote format flag. Fix cmdQuoteWord to use close-quote/^"/reopen-quote for embedding literal double quotes. Assisted-by: Crush:glm-5.2
1 parent 60a494e commit 7b9e977

17 files changed

Lines changed: 333 additions & 39 deletions

format.go

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,15 @@ type Format interface {
4444
// Only elvish needs this (\ is a bareword char in elvish).
4545
EscapeNotBareword() bool
4646

47+
// EscapeNotInEscapingQuote returns true if the escape character is
48+
// literal inside the escaping quote (double quotes) rather than acting
49+
// as an escape. When true, the QUOTING_ESCAPING_STATE handler treats
50+
// the escape rune as a regular word character instead of entering
51+
// ESCAPING_QUOTED_STATE. Only cmd needs this: cmd's caret (^) is
52+
// completely literal inside double quotes — it does not escape the
53+
// next character when quoted.
54+
EscapeNotInEscapingQuote() bool
55+
4756
// EscapingQuoteEscapeChars returns the set of characters that backslash
4857
// can escape inside the escaping quote (double quotes). If nil, backslash
4958
// escapes any character. POSIX shells (bash, zsh, tcsh) return the
@@ -113,8 +122,8 @@ type LineContinuationEscaper interface {
113122
// it enters a dedicated BLOCK_COMMENT_STATE that scans until the
114123
// blockCommentCloser runes are found, spanning multiple lines.
115124
type BlockCommenter interface {
116-
BlockCommentOpener() string // e.g. "<#" for PowerShell
117-
BlockCommentCloser() string // e.g. "#>" for PowerShell
125+
BlockCommentOpener() string // e.g. "<#" for PowerShell
126+
BlockCommentCloser() string // e.g. "#>" for PowerShell
118127
}
119128

120129
// StopParsingToken is an optional interface for formats that support a

format_bash.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ func (bashFormat) NonEscapingQuoteEscapes() bool { return false }
3232

3333
func (bashFormat) NonEscapingQuoteBackslashEscapes() bool { return false }
3434
func (bashFormat) EscapeNotBareword() bool { return true }
35+
func (bashFormat) EscapeNotInEscapingQuote() bool { return false }
3536
func (bashFormat) EscapingQuoteEscapeChars() map[rune]bool {
3637
return map[rune]bool{
3738
'\\': true,

format_cmd.go

Lines changed: 100 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,17 @@ package shlex
22

33
// cmdFormat implements Format for cmd.exe (with clink) lexing.
44
// Key differences from POSIX:
5-
// - Caret (^) is the escape character, not backslash (\)
6-
// - Double quotes (") are the only quote — simple toggle, no \ escapes inside
7-
// - No single quotes — ' is a literal word character
8-
// - & is a command separator (like ; in POSIX), not background
9-
// - ; is NOT a separator (literal character)
10-
// - REM and :: are comments (keyword/two-rune, not rune-based)
11-
// - % is a word character (variable expansion sigil)
12-
// - \ is a literal word character (Windows paths)
5+
// - Caret (^) is the escape character outside quotes, but literal inside "..."
6+
// - Double quotes (") are the only quote — simple toggle, no escapes inside
7+
// - No single quotes — ' is a literal word character
8+
// - & is a command separator (like ; in POSIX), not background
9+
// - ; is NOT a separator (literal character)
10+
// - ( ) are grouping operators (command blocks)
11+
// - REM and :: are comments (keyword/two-rune, not rune-based)
12+
// - % is a word character (variable expansion sigil)
13+
// - \ is a literal word character (Windows paths)
14+
// - Caret + newline is line continuation (consumed, not part of word)
15+
// - Numeric stream redirects: 2>, 2>&1, 1>&2 (merged in PostProcess)
1316
type cmdFormat struct{}
1417

1518
// CmdFormat returns the cmd.exe lexical format.
@@ -19,17 +22,23 @@ func CmdFormat() Format { return cmdFormat{} }
1922
func (cmdFormat) Classifier() tokenClassifier {
2023
t := tokenClassifier{}
2124
t.addRuneClass(spaceRunes, spaceRuneClass)
25+
// Cmd: comma is a word delimiter (like space), but not a command separator.
26+
// Semicolons and equals are also delimiters in cmd, but ; is kept as a
27+
// literal word char because it's safer for completion (e.g. set VAR=value
28+
// would break if = were a delimiter). Comma is always safe to split.
29+
t.addRuneClass(",", spaceRuneClass)
2230
// Cmd: only " is a quote. ' is a regular word char.
2331
t.addRuneClass(escapingQuoteRunes, escapingQuoteRuneClass) // " is the escaping quote
2432
// Cmd: ^ is the escape character, not \
2533
t.addRuneClass("^", escapeRuneClass)
2634
// Cmd: # is not a comment (that's REM/::). Don't classify it as comment.
2735
// REM/:: comments need keyword detection (deferred).
2836

29-
// Cmd operators: |, &, <, >
37+
// Cmd operators: |, &, <, >, (, )
3038
// Note: & is a command separator (like ; in POSIX), not background
31-
// ; is NOT a separator in cmd
32-
wordbreakRunes := "|&<>"
39+
// ; is NOT a separator in cmd (it is a literal character)
40+
// ( and ) are grouping operators for command blocks
41+
wordbreakRunes := "|&<>()"
3342
filtered := make([]rune, 0)
3443
for _, r := range wordbreakRunes {
3544
if t.ClassifyRune(r) == unknownRuneClass {
@@ -55,6 +64,9 @@ func (cmdFormat) ClassifyOperator(raw string) WordbreakType {
5564
return WORDBREAK_REDIRECT_OUTPUT
5665
case "<":
5766
return WORDBREAK_REDIRECT_INPUT
67+
case "(", ")":
68+
// Cmd: parentheses are grouping operators for command blocks
69+
return WORDBREAK_UNKNOWN
5870
default:
5971
return WORDBREAK_UNKNOWN
6072
}
@@ -65,7 +77,84 @@ func (cmdFormat) KeywordOperators() map[string]WordbreakType { return nil }
6577
func (cmdFormat) NonEscapingQuoteEscapes() bool { return false }
6678
func (cmdFormat) NonEscapingQuoteBackslashEscapes() bool { return false }
6779
func (cmdFormat) EscapeNotBareword() bool { return true }
80+
func (cmdFormat) EscapeNotInEscapingQuote() bool { return true }
6881
func (cmdFormat) EscapingQuoteEscapeChars() map[rune]bool { return nil }
6982
func (cmdFormat) QuoteWord(s string) string { return cmdQuoteWord(s) }
7083
func (cmdFormat) TripleQuoteSupport() bool { return false }
7184
func (cmdFormat) RawPrefixSupport() bool { return false }
85+
86+
// IsLineContinuation implements LineContinuationEscaper. cmd.exe's caret
87+
// followed by \n or \r is a line continuation — the sequence is consumed
88+
// and the word continues on the next line.
89+
func (cmdFormat) IsLineContinuation(r rune) bool {
90+
return r == '\n' || r == '\r'
91+
}
92+
93+
// PostProcess merges cmd.exe numeric stream-redirect operators. The
94+
// tokenizer produces e.g. `2` as a WORD_TOKEN and `>` (or `>>`) as a
95+
// WORDBREAK_TOKEN. This step detects adjacent word+wordbreak sequences
96+
// like `2>`, `2>>`, `2>&1`, `1>&2` and reclassifies them as single
97+
// WORDBREAK_TOKENs with the appropriate WordbreakType.
98+
func (cmdFormat) PostProcess(tokens TokenSlice) TokenSlice {
99+
result := make(TokenSlice, 0, len(tokens))
100+
for i := 0; i < len(tokens); i++ {
101+
t := tokens[i]
102+
103+
// Look for bare WORD_TOKEN (digit 1-2) immediately followed by
104+
// WORDBREAK_TOKEN starting with '>' (redirect operator)
105+
if t.Type == WORD_TOKEN && t.Value == t.RawValue && i+1 < len(tokens) {
106+
next := tokens[i+1]
107+
if next.Type == WORDBREAK_TOKEN && next.adjoins(t) &&
108+
next.WordbreakType.IsRedirect() && len(next.RawValue) > 0 && next.RawValue[0] == '>' {
109+
if len(t.Value) == 1 && (t.Value[0] == '1' || t.Value[0] == '2') {
110+
wbType := next.WordbreakType
111+
mergedRaw := t.RawValue + next.RawValue
112+
mergedVal := t.Value + next.Value
113+
mergedSpan := Span{Start: t.Span.Start, End: next.Span.End}
114+
115+
// Check for &N pattern (stream merge) in the token after next
116+
if i+2 < len(tokens) && tokens[i+2].Type == WORDBREAK_TOKEN &&
117+
tokens[i+2].Value == "&" && tokens[i+2].adjoins(next) {
118+
if i+3 < len(tokens) && tokens[i+3].Type == WORD_TOKEN &&
119+
tokens[i+3].Value == tokens[i+3].RawValue &&
120+
tokens[i+3].adjoins(tokens[i+2]) &&
121+
len(tokens[i+3].Value) == 1 &&
122+
(tokens[i+3].Value[0] == '1' || tokens[i+3].Value[0] == '2') {
123+
// 2>&1 pattern — merge all four tokens
124+
mergedRaw += tokens[i+2].RawValue + tokens[i+3].RawValue
125+
mergedVal += tokens[i+2].Value + tokens[i+3].Value
126+
mergedSpan.End = tokens[i+3].Span.End
127+
wbType = WORDBREAK_REDIRECT_OUTPUT_BOTH
128+
merged := Token{
129+
Type: WORDBREAK_TOKEN,
130+
Value: mergedVal,
131+
RawValue: mergedRaw,
132+
Span: mergedSpan,
133+
State: tokens[i+3].State,
134+
WordbreakType: wbType,
135+
}
136+
result = append(result, merged)
137+
i += 3
138+
continue
139+
}
140+
}
141+
142+
merged := Token{
143+
Type: WORDBREAK_TOKEN,
144+
Value: mergedVal,
145+
RawValue: mergedRaw,
146+
Span: mergedSpan,
147+
State: next.State,
148+
WordbreakType: wbType,
149+
}
150+
result = append(result, merged)
151+
i += 1
152+
continue
153+
}
154+
}
155+
}
156+
157+
result = append(result, t)
158+
}
159+
return result
160+
}

format_cmd_test.go

Lines changed: 166 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,15 +108,22 @@ func TestCmdFormat_DoubleAnd(t *testing.T) {
108108
}
109109

110110
func TestCmdFormat_CaretInQuotes(t *testing.T) {
111-
// Cmd: ^ escapes inside double quotes (^" → literal ")
111+
// Cmd: ^ is LITERAL inside double quotes — it does not escape.
112+
// "say ^" → ^ is literal, " closes the quote.
113+
// Outside quotes, ^" → literal " (caret escapes).
112114
tokens, err := SplitWith(`echo "say ^"hello^""`, CmdFormat())
113115
if err != nil {
114116
t.Fatal(err)
115117
}
116118
words := tokens.Words()
117119
last := words[len(words)-1]
118-
if last.Value != `say "hello"` {
119-
t.Errorf("cmd caret in quotes: Value = %q, want %q", last.Value, `say "hello"`)
120+
// "say ^" → quote contains "say ^", then " closes quote
121+
// hello → bareword outside quotes
122+
// ^" → caret escapes the quote → literal "
123+
// " → this final quote opens a new quoted region (unterminated)
124+
// Words() merges adjacent tokens, so the whole thing is one word.
125+
if last.Value != `say ^hello"` {
126+
t.Errorf("cmd caret in quotes: Value = %q, want %q", last.Value, `say ^hello"`)
120127
}
121128
}
122129

@@ -176,3 +183,159 @@ func TestCmdFormat_CaretAtEOF(t *testing.T) {
176183
t.Errorf("cmd caret EOF: Value = %q, want %q", last.Value, "foo")
177184
}
178185
}
186+
187+
func TestCmdFormat_CaretLiteralInQuotes(t *testing.T) {
188+
// Cmd: ^ is literal inside double quotes — does not escape the next char.
189+
// "hello^world" should produce hello^world, not helloworld.
190+
tokens, err := SplitWith(`echo "hello^world"`, CmdFormat())
191+
if err != nil {
192+
t.Fatal(err)
193+
}
194+
words := tokens.Words().Strings()
195+
if len(words) != 2 || words[0] != "echo" || words[1] != "hello^world" {
196+
t.Errorf("cmd caret literal in quotes: Words = %v, want [echo hello^world]", words)
197+
}
198+
}
199+
200+
func TestCmdFormat_DoubleCaretLiteralInQuotes(t *testing.T) {
201+
// Cmd: ^^ inside quotes is literal ^^ (both carets), not a single ^.
202+
tokens, err := SplitWith(`echo "hello^^world"`, CmdFormat())
203+
if err != nil {
204+
t.Fatal(err)
205+
}
206+
words := tokens.Words().Strings()
207+
if len(words) != 2 || words[0] != "echo" || words[1] != "hello^^world" {
208+
t.Errorf("cmd double caret in quotes: Words = %v, want [echo hello^^world]", words)
209+
}
210+
}
211+
212+
func TestCmdFormat_LineContinuation(t *testing.T) {
213+
// Cmd: ^ at end of line is a line continuation — ^\n is consumed
214+
tokens, err := SplitWith("echo foo^\nbar", CmdFormat())
215+
if err != nil {
216+
t.Fatal(err)
217+
}
218+
words := tokens.Words().Strings()
219+
if len(words) != 2 || words[0] != "echo" || words[1] != "foobar" {
220+
t.Errorf("cmd line continuation: Words = %v, want [echo foobar]", words)
221+
}
222+
}
223+
224+
func TestCmdFormat_LineContinuationCRLF(t *testing.T) {
225+
// Cmd: ^ at end of line with CRLF is a line continuation
226+
tokens, err := SplitWith("echo foo^\r\nbar", CmdFormat())
227+
if err != nil {
228+
t.Fatal(err)
229+
}
230+
words := tokens.Words().Strings()
231+
if len(words) != 2 || words[0] != "echo" || words[1] != "foobar" {
232+
t.Errorf("cmd line continuation CRLF: Words = %v, want [echo foobar]", words)
233+
}
234+
}
235+
236+
func TestCmdFormat_ParenGrouping(t *testing.T) {
237+
// Cmd: ( and ) are grouping operators
238+
tokens, err := SplitWith("(echo foo) & echo bar", CmdFormat())
239+
if err != nil {
240+
t.Fatal(err)
241+
}
242+
pipelines := tokens.Pipelines()
243+
if len(pipelines) != 2 {
244+
t.Errorf("cmd parens: %d pipelines, want 2", len(pipelines))
245+
}
246+
}
247+
248+
func TestCmdFormat_ParenBeforeCommand(t *testing.T) {
249+
// Cmd: ( and ) are wordbreak operators; with spaces they separate from words
250+
// They are not redirect operators, so FilterRedirects keeps them.
251+
// Words() does not merge non-adjacent tokens.
252+
tokens, err := SplitWith("( echo hello )", CmdFormat())
253+
if err != nil {
254+
t.Fatal(err)
255+
}
256+
pipeline := tokens.CurrentPipeline()
257+
words := pipeline.Words().Strings()
258+
if len(words) != 4 || words[0] != "(" || words[1] != "echo" || words[2] != "hello" || words[3] != ")" {
259+
t.Errorf("cmd paren before cmd: Words = %v, want [( echo hello )]", words)
260+
}
261+
}
262+
263+
func TestCmdFormat_CommaDelimiter(t *testing.T) {
264+
// Cmd: comma is a word delimiter (like space)
265+
tokens, err := SplitWith("echo hello,world", CmdFormat())
266+
if err != nil {
267+
t.Fatal(err)
268+
}
269+
words := tokens.Words().Strings()
270+
if len(words) != 3 || words[0] != "echo" || words[1] != "hello" || words[2] != "world" {
271+
t.Errorf("cmd comma: Words = %v, want [echo hello world]", words)
272+
}
273+
}
274+
275+
func TestCmdFormat_CommaInQuotes(t *testing.T) {
276+
// Cmd: comma inside double quotes is literal
277+
tokens, err := SplitWith(`echo "hello,world"`, CmdFormat())
278+
if err != nil {
279+
t.Fatal(err)
280+
}
281+
words := tokens.Words().Strings()
282+
if len(words) != 2 || words[0] != "echo" || words[1] != "hello,world" {
283+
t.Errorf("cmd comma in quotes: Words = %v, want [echo hello,world]", words)
284+
}
285+
}
286+
287+
func TestCmdFormat_StreamRedirect2(t *testing.T) {
288+
// Cmd: 2> should be recognized as a stream redirect (stderr)
289+
tokens, err := SplitWith("echo foo 2> bar", CmdFormat())
290+
if err != nil {
291+
t.Fatal(err)
292+
}
293+
pipelines := tokens.Pipelines()
294+
if len(pipelines) != 1 {
295+
t.Errorf("cmd 2>: %d pipelines, want 1", len(pipelines))
296+
}
297+
// The 2> should be a redirect, so filtered words should not include "2"
298+
filtered := pipelines[0].FilterRedirects().Words().Strings()
299+
if len(filtered) != 2 || filtered[0] != "echo" || filtered[1] != "foo" {
300+
t.Errorf("cmd 2> filtered: Words = %v, want [echo foo]", filtered)
301+
}
302+
}
303+
304+
func TestCmdFormat_StreamRedirectMerge(t *testing.T) {
305+
// Cmd: 2>&1 should be recognized as a stream merge redirect
306+
tokens, err := SplitWith("echo foo 2>&1 bar", CmdFormat())
307+
if err != nil {
308+
t.Fatal(err)
309+
}
310+
pipelines := tokens.Pipelines()
311+
if len(pipelines) != 1 {
312+
t.Errorf("cmd 2>&1: %d pipelines, want 1", len(pipelines))
313+
}
314+
filtered := pipelines[0].FilterRedirects().Words().Strings()
315+
if len(filtered) != 2 || filtered[0] != "echo" || filtered[1] != "foo" {
316+
t.Errorf("cmd 2>&1 filtered: Words = %v, want [echo foo]", filtered)
317+
}
318+
}
319+
320+
func TestCmdFormat_StreamRedirectCompletion(t *testing.T) {
321+
// Cmd: completing after 2> should detect redirect
322+
ctx := SplitForCompletion("echo foo 2> bar", CmdFormat())
323+
if !ctx.IsRedirect {
324+
t.Errorf("cmd 2> completion: IsRedirect = false, want true")
325+
}
326+
if ctx.CurrentWord != "bar" {
327+
t.Errorf("cmd 2> completion: CurrentWord = %q, want %q", ctx.CurrentWord, "bar")
328+
}
329+
}
330+
331+
func TestCmdFormat_CaretLineContinuationAtEOF(t *testing.T) {
332+
// Cmd: ^ at EOF (no newline) should enter ESCAPING_STATE, not line continuation
333+
tokens, err := SplitWith("echo foo^", CmdFormat())
334+
if err != nil {
335+
t.Fatal(err)
336+
}
337+
last := tokens.Words().CurrentToken()
338+
if last.State != ESCAPING_STATE {
339+
t.Errorf("cmd caret EOF: State = %v, want ESCAPING_STATE", last.State)
340+
}
341+
}

format_elvish.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ func (elvishFormat) KeywordOperators() map[string]WordbreakType { return nil }
4343
func (elvishFormat) NonEscapingQuoteEscapes() bool { return true } // '' → '
4444
func (elvishFormat) NonEscapingQuoteBackslashEscapes() bool { return false }
4545
func (elvishFormat) EscapeNotBareword() bool { return false }
46+
func (elvishFormat) EscapeNotInEscapingQuote() bool { return false }
4647
func (elvishFormat) EscapingQuoteEscapeChars() map[rune]bool { return nil }
4748
func (elvishFormat) QuoteWord(s string) string { return elvishQuoteWord(s) }
4849
func (elvishFormat) TripleQuoteSupport() bool { return false }

format_fish.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ func (fishFormat) KeywordOperators() map[string]WordbreakType {
6565
func (fishFormat) NonEscapingQuoteEscapes() bool { return true } // ' and \\ inside single quotes
6666
func (fishFormat) NonEscapingQuoteBackslashEscapes() bool { return true }
6767
func (fishFormat) EscapeNotBareword() bool { return true }
68+
func (fishFormat) EscapeNotInEscapingQuote() bool { return false }
6869

6970
// EscapingQuoteEscapeChars returns the limited set of characters that
7071
// backslash can escape inside fish double quotes: ", $, \, and newline.

format_nushell.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ func (nushellFormat) KeywordOperators() map[string]WordbreakType { return nil }
6464
func (nushellFormat) NonEscapingQuoteEscapes() bool { return false }
6565
func (nushellFormat) NonEscapingQuoteBackslashEscapes() bool { return false }
6666
func (nushellFormat) EscapeNotBareword() bool { return true }
67+
func (nushellFormat) EscapeNotInEscapingQuote() bool { return false }
6768
func (nushellFormat) EscapingQuoteEscapeChars() map[rune]bool { return nil }
6869
func (nushellFormat) QuoteWord(s string) string { return nushellQuoteWord(s) }
6970
func (nushellFormat) TripleQuoteSupport() bool { return false }

0 commit comments

Comments
 (0)