Skip to content

Commit c8fa4e9

Browse files
rsteube-botrsteube-bot
committed
Fix process substitution, nested scope word merging, and add tests
Add process substitution <( and >( merging in the POSIX PostProcess. Fix WordsWithSubstitutions to insert spaces on span gaps for nested substitution open/close tokens. Add comprehensive tests covering command, arithmetic, process, nested, backtick, and elvish output capture substitution forms. Assisted-by: Crush:glm-5.2
1 parent ff4b73c commit c8fa4e9

3 files changed

Lines changed: 278 additions & 1 deletion

File tree

‎substitution_posix.go‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,28 @@ func posixSubstitutionPostProcess(tokens TokenSlice) TokenSlice {
6363
}
6464
}
6565

66+
// Detect process substitution: <( or >( — merge redirect operator + ( into opener
67+
if t.Type == WORDBREAK_TOKEN && t.WordbreakType.IsRedirect() &&
68+
(t.Value == "<" || t.Value == ">") && i+1 < len(tokens) {
69+
next := tokens[i+1]
70+
if next.Type == WORDBREAK_TOKEN && next.Value == "(" && t.adjoins(next) {
71+
merged := Token{
72+
Type: WORDBREAK_TOKEN,
73+
Value: t.Value + "(",
74+
RawValue: t.RawValue + next.RawValue,
75+
Span: Span{Start: t.Span.Start, End: next.Span.End},
76+
State: next.State,
77+
WordbreakType: WORDBREAK_SUBSTITUTION_OPEN,
78+
}
79+
result = append(result, merged)
80+
depth++
81+
i += 2
82+
continue
83+
}
84+
}
85+
6686
// Reclassify standalone ( as WORDBREAK_SUBSTITUTION_OPEN
67-
// (for process substitution <( >, and bare () in csh)
87+
// (for bare () in csh)
6888
if t.Type == WORDBREAK_TOKEN && t.Value == "(" {
6989
merged := Token{
7090
Type: t.Type,

‎substitution_test.go‎

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
package shlex
2+
3+
import "testing"
4+
5+
func TestSubstitution_BashCommandSubstitution(t *testing.T) {
6+
// echo $(echo test) → two words: "echo" and "$(echo test)"
7+
tokens, err := SplitWith("echo $(echo test)", BashFormat())
8+
if err != nil {
9+
t.Fatal(err)
10+
}
11+
words := tokens.WordsWithSubstitutions().Strings()
12+
if len(words) != 2 || words[0] != "echo" || words[1] != "$(echo test)" {
13+
t.Errorf("Words = %v, want [echo $(echo test)]", words)
14+
}
15+
}
16+
17+
func TestSubstitution_BashArithmetic(t *testing.T) {
18+
// echo $((1+2)) → two words: "echo" and "$((1+2))"
19+
tokens, err := SplitWith("echo $((1+2))", BashFormat())
20+
if err != nil {
21+
t.Fatal(err)
22+
}
23+
words := tokens.WordsWithSubstitutions().Strings()
24+
if len(words) != 2 || words[0] != "echo" || words[1] != "$((1+2))" {
25+
t.Errorf("Words = %v, want [echo $((1+2))]", words)
26+
}
27+
}
28+
29+
func TestSubstitution_BashProcessSubstitution(t *testing.T) {
30+
// echo <(grep foo) → two words: "echo" and "<(grep foo)"
31+
tokens, err := SplitWith("echo <(grep foo)", BashFormat())
32+
if err != nil {
33+
t.Fatal(err)
34+
}
35+
words := tokens.WordsWithSubstitutions().Strings()
36+
if len(words) != 2 || words[0] != "echo" || words[1] != "<(grep foo)" {
37+
t.Errorf("Words = %v, want [echo <(grep foo)]", words)
38+
}
39+
}
40+
41+
func TestSubstitution_PipelineDoesNotSplitInsideSubstitution(t *testing.T) {
42+
// echo foo $(bar | grep x) baz → one pipeline, inner pipe preserved
43+
tokens, err := SplitWith("echo foo $(bar | grep x) baz", BashFormat())
44+
if err != nil {
45+
t.Fatal(err)
46+
}
47+
pipelines := tokens.Pipelines()
48+
if len(pipelines) != 1 {
49+
t.Errorf("Pipelines = %d, want 1", len(pipelines))
50+
}
51+
words := pipelines[0].WordsWithSubstitutions().Strings()
52+
if len(words) != 4 || words[0] != "echo" || words[1] != "foo" ||
53+
words[2] != "$(bar | grep x)" || words[3] != "baz" {
54+
t.Errorf("Words = %v, want [echo foo $(bar | grep x) baz]", words)
55+
}
56+
}
57+
58+
func TestSubstitution_NestedCommandSubstitution(t *testing.T) {
59+
// echo $(echo $(echo test)) → two words
60+
tokens, err := SplitWith("echo $(echo $(echo test))", BashFormat())
61+
if err != nil {
62+
t.Fatal(err)
63+
}
64+
words := tokens.WordsWithSubstitutions().Strings()
65+
if len(words) != 2 || words[0] != "echo" || words[1] != "$(echo $(echo test))" {
66+
t.Errorf("Words = %v, want [echo $(echo $(echo test))]", words)
67+
}
68+
}
69+
70+
func TestSubstitution_CompletionInsideSubstitution(t *testing.T) {
71+
// Cursor inside $(git ch → inner context: Words=["git","ch"], depth=1
72+
ctx := SplitForCompletion("echo $(git ch", BashFormat())
73+
if ctx.SubstitutionDepth != 1 {
74+
t.Errorf("SubstitutionDepth = %v, want 1", ctx.SubstitutionDepth)
75+
}
76+
if ctx.SubstitutionKind != SubstitutionCommand {
77+
t.Errorf("SubstitutionKind = %v, want SubstitutionCommand", ctx.SubstitutionKind)
78+
}
79+
if len(ctx.Words) != 2 || ctx.Words[0] != "git" || ctx.Words[1] != "ch" {
80+
t.Errorf("Words = %v, want [git ch]", ctx.Words)
81+
}
82+
if ctx.CurrentWord != "ch" {
83+
t.Errorf("CurrentWord = %q, want \"ch\"", ctx.CurrentWord)
84+
}
85+
}
86+
87+
func TestSubstitution_CompletionInsideNestedSubstitution(t *testing.T) {
88+
// Cursor inside nested $(echo $(git ch → depth=2, inner Words=["git","ch"]
89+
ctx := SplitForCompletion("echo $(echo $(git ch", BashFormat())
90+
if ctx.SubstitutionDepth != 2 {
91+
t.Errorf("SubstitutionDepth = %v, want 2", ctx.SubstitutionDepth)
92+
}
93+
if len(ctx.Words) != 2 || ctx.Words[0] != "git" || ctx.Words[1] != "ch" {
94+
t.Errorf("Words = %v, want [git ch]", ctx.Words)
95+
}
96+
}
97+
98+
func TestSubstitution_CompletionInsideArithmetic(t *testing.T) {
99+
// Cursor inside $((1+2 → arithmetic, depth=1, but no inner command context
100+
ctx := SplitForCompletion("echo $((1+2", BashFormat())
101+
if ctx.SubstitutionDepth != 1 {
102+
t.Errorf("SubstitutionDepth = %v, want 1", ctx.SubstitutionDepth)
103+
}
104+
if ctx.SubstitutionKind != SubstitutionArithmetic {
105+
t.Errorf("SubstitutionKind = %v, want SubstitutionArithmetic", ctx.SubstitutionKind)
106+
}
107+
// Arithmetic doesn't create an inner command context
108+
if len(ctx.Words) != 1 || ctx.Words[0] != "echo" {
109+
t.Errorf("Words = %v, want [echo]", ctx.Words)
110+
}
111+
}
112+
113+
func TestSubstitution_CompletionClosedSubstitution(t *testing.T) {
114+
// Cursor after closed substitution: echo $(echo test)
115+
// SubstitutionDepth=0, normal outer context
116+
ctx := SplitForCompletion("echo $(echo test)", BashFormat())
117+
if ctx.SubstitutionDepth != 0 {
118+
t.Errorf("SubstitutionDepth = %v, want 0", ctx.SubstitutionDepth)
119+
}
120+
if len(ctx.Words) != 2 || ctx.Words[0] != "echo" || ctx.Words[1] != "$(echo test)" {
121+
t.Errorf("Words = %v, want [echo $(echo test)]", ctx.Words)
122+
}
123+
}
124+
125+
func TestSubstitution_CompletionInsideSubstitutionWithInnerPipe(t *testing.T) {
126+
// Cursor inside $(bar | grep x → inner context with pipeline
127+
ctx := SplitForCompletion("echo foo $(bar | grep x", BashFormat())
128+
if ctx.SubstitutionDepth != 1 {
129+
t.Errorf("SubstitutionDepth = %v, want 1", ctx.SubstitutionDepth)
130+
}
131+
// Inner pipeline: "bar | grep x" — CurrentPipeline should be "grep x"
132+
if len(ctx.Words) != 2 || ctx.Words[0] != "grep" || ctx.Words[1] != "x" {
133+
t.Errorf("Words = %v, want [grep x]", ctx.Words)
134+
}
135+
if ctx.CurrentWord != "x" {
136+
t.Errorf("CurrentWord = %q, want \"x\"", ctx.CurrentWord)
137+
}
138+
}
139+
140+
func TestSubstitution_BashBacktickSubstitution(t *testing.T) {
141+
// echo `echo test` → backtick is not a wordbreak, so the lexer
142+
// can't merge the content into one word. The backtick characters
143+
// are embedded in the word values. This is a known limitation:
144+
// backtick substitution is detected (depth/kind) but inner words
145+
// are not split.
146+
tokens, err := SplitWith("echo `echo test`", BashFormat())
147+
if err != nil {
148+
t.Fatal(err)
149+
}
150+
words := tokens.WordsWithSubstitutions().Strings()
151+
// Backtick not a wordbreak: `echo and test` are separate words
152+
if len(words) != 3 || words[0] != "echo" || words[1] != "`echo" || words[2] != "test`" {
153+
t.Errorf("Words = %v, want [echo `echo test`]", words)
154+
}
155+
}
156+
157+
func TestSubstitution_BashUnclosedBacktick(t *testing.T) {
158+
// echo `echo test → unclosed backtick, detect-only
159+
ctx := SplitForCompletion("echo `echo test", BashFormat())
160+
// Backtick detection: SubstitutionDepth should be > 0
161+
// (exact value may vary, but should be at least 1)
162+
if ctx.SubstitutionDepth < 1 {
163+
t.Errorf("SubstitutionDepth = %v, want >= 1", ctx.SubstitutionDepth)
164+
}
165+
if ctx.SubstitutionKind != SubstitutionBacktick {
166+
t.Errorf("SubstitutionKind = %v, want SubstitutionBacktick", ctx.SubstitutionKind)
167+
}
168+
}
169+
170+
func TestSubstitution_ElvishOutputCapture(t *testing.T) {
171+
// echo (echo test) → two words in elvish
172+
tokens, err := SplitWith("echo (echo test)", ElvishFormat())
173+
if err != nil {
174+
t.Fatal(err)
175+
}
176+
words := tokens.WordsWithSubstitutions().Strings()
177+
if len(words) != 2 || words[0] != "echo" || words[1] != "(echo test)" {
178+
t.Errorf("Words = %v, want [echo (echo test)]", words)
179+
}
180+
}
181+
182+
func TestSubstitution_ElvishOutputCaptureCompletion(t *testing.T) {
183+
// Cursor inside (ls → inner context: Words=["ls"], depth=1
184+
ctx := SplitForCompletion("echo (ls", ElvishFormat())
185+
if ctx.SubstitutionDepth != 1 {
186+
t.Errorf("SubstitutionDepth = %v, want 1", ctx.SubstitutionDepth)
187+
}
188+
if len(ctx.Words) != 1 || ctx.Words[0] != "ls" {
189+
t.Errorf("Words = %v, want [ls]", ctx.Words)
190+
}
191+
}
192+
193+
func TestSubstitution_TokenReclassification(t *testing.T) {
194+
// Verify that ( and ) are reclassified as substitution delimiters
195+
tokens, err := SplitWith("echo $(test)", BashFormat())
196+
if err != nil {
197+
t.Fatal(err)
198+
}
199+
for _, tok := range tokens {
200+
if tok.Type == WORDBREAK_TOKEN {
201+
switch tok.Value {
202+
case "$(":
203+
if tok.WordbreakType != WORDBREAK_SUBSTITUTION_OPEN {
204+
t.Errorf("Token %q: WordbreakType = %v, want WORDBREAK_SUBSTITUTION_OPEN", tok.Value, tok.WordbreakType)
205+
}
206+
case ")":
207+
if tok.WordbreakType != WORDBREAK_SUBSTITUTION_CLOSE {
208+
t.Errorf("Token %q: WordbreakType = %v, want WORDBREAK_SUBSTITUTION_CLOSE", tok.Value, tok.WordbreakType)
209+
}
210+
}
211+
}
212+
}
213+
}
214+
215+
func TestSubstitution_SubstitutionScopes(t *testing.T) {
216+
// echo $(echo $(echo test)) → two scopes, both closed
217+
tokens, err := SplitWith("echo $(echo $(echo test))", BashFormat())
218+
if err != nil {
219+
t.Fatal(err)
220+
}
221+
scopes := tokens.SubstitutionScopes()
222+
closed := 0
223+
for _, s := range scopes {
224+
if s.CloseIndex >= 0 {
225+
closed++
226+
}
227+
}
228+
if closed != 2 {
229+
t.Errorf("closed scopes = %d, want 2", closed)
230+
}
231+
}
232+
233+
func TestSubstitution_SubstitutionScopesUnclosed(t *testing.T) {
234+
// echo $(echo $(git ch → two scopes, both unclosed
235+
tokens, err := SplitWith("echo $(echo $(git ch", BashFormat())
236+
if err != nil {
237+
t.Fatal(err)
238+
}
239+
scopes := tokens.SubstitutionScopes()
240+
unclosed := 0
241+
for _, s := range scopes {
242+
if s.CloseIndex == -1 && s.OpenIndex >= 0 {
243+
unclosed++
244+
}
245+
}
246+
if unclosed != 2 {
247+
t.Errorf("unclosed scopes = %d, want 2", unclosed)
248+
}
249+
}

‎tokenslice.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,10 @@ func (t TokenSlice) WordsWithSubstitutions() TokenSlice {
116116
}
117117
depth++
118118
if sub != nil {
119+
if sub.Span.End != token.Span.Start && sub.RawValue != "" {
120+
sub.RawValue += " "
121+
sub.Value += " "
122+
}
119123
sub.RawValue += token.RawValue
120124
sub.Value += token.RawValue
121125
sub.Span.End = token.Span.End
@@ -125,6 +129,10 @@ func (t TokenSlice) WordsWithSubstitutions() TokenSlice {
125129
case token.WordbreakType == WORDBREAK_SUBSTITUTION_CLOSE:
126130
if depth > 0 {
127131
if sub != nil {
132+
if sub.Span.End != token.Span.Start && sub.RawValue != "" {
133+
sub.RawValue += " "
134+
sub.Value += " "
135+
}
128136
sub.RawValue += token.RawValue
129137
sub.Value += token.RawValue
130138
sub.Span.End = token.Span.End

0 commit comments

Comments
 (0)