Add zsh-specific operators and missing bash-shared operators

rsteube · rsteube · commit ae1bac1beb67 · 2026-07-01T20:02:12.000+02:00
Add operator classification for &gt;|, &gt;&gt;|, ;;, ;&amp;, ;|, &amp;| found in zsh
lex.c but previously returning WORDBREAK_UNKNOWN. This fixes redirect
detection for &gt;| and &gt;&gt;| and pipeline splitting for all new list
operators. Also documents remaining zsh edge cases (RC_QUOTES inside
$'...', line continuation, INTERACTIVECOMMENTS, CSHJUNKIEQUOTES).

Assisted-by: Crush:glm-5.2
diff --git a/format_bash_test.go b/format_bash_test.go
@@ -117,3 +117,34 @@ func TestBashFormat_Comment(t *testing.T) {
 		t.Errorf("bash comment: Words = %v, want 2 words", words)
 	}
 }
+
+func TestBashFormat_ForceOutputRedirect(t *testing.T) {
+	ctx := SplitForCompletion("echo foo >| bar", BashFormat())
+	if !ctx.IsRedirect {
+		t.Errorf("bash >|: IsRedirect = false, want true")
+	}
+}
+
+func TestBashFormat_CaseTerminator(t *testing.T) {
+	tokens, err := SplitWith("echo foo ;; bar", BashFormat())
+	if err != nil {
+		t.Fatal(err)
+	}
+	pipelines := tokens.Pipelines()
+	if len(pipelines) != 2 {
+		t.Errorf("bash ;;: Pipelines = %d, want 2", len(pipelines))
+	}
+	var found *Token
+	for i := range tokens {
+		if tokens[i].Type == WORDBREAK_TOKEN {
+			found = &tokens[i]
+			break
+		}
+	}
+	if found == nil {
+		t.Fatal("bash ;;: no wordbreak token found")
+	}
+	if found.WordbreakType != WORDBREAK_LIST_SEQUENTIAL_DOUBLE {
+		t.Errorf("bash ;;: WordbreakType = %v, want WORDBREAK_LIST_SEQUENTIAL_DOUBLE", found.WordbreakType)
+	}
+}
diff --git a/format_zsh.go b/format_zsh.go
@@ -1,7 +1,8 @@
 package shlex
 
 // zshFormat implements Format for zsh lexing.
-// Extends bash with RC_QUOTES (” → ' inside single quotes).
+// Extends bash with RC_QUOTES, zsh-specific operators (>>|, ;&, ;|, &|),
+// and WORDCHARS/FIGNORE for word breaks.
 type zshFormat struct{}
 
 // ZshFormat returns the zsh lexical format.
@@ -13,7 +14,18 @@ func (zshFormat) Classifier() tokenClassifier {
 }
 
 func (zshFormat) ClassifyOperator(raw string) WordbreakType {
-	return bashWordbreakType(raw) // zsh uses the same operator grammar as bash
+	switch raw {
+	case ">>|":
+		return WORDBREAK_REDIRECT_OUTPUT_APPEND_FORCE
+	case ";&":
+		return WORDBREAK_LIST_FALLTHROUGH
+	case ";|":
+		return WORDBREAK_LIST_FALLTHROUGH_RETRY
+	case "&|":
+		return WORDBREAK_LIST_ASYNC_ERRCHECK
+	default:
+		return bashWordbreakType(raw)
+	}
 }
 
 func (zshFormat) KeywordOperators() map[string]WordbreakType { return nil }
diff --git a/format_zsh_test.go b/format_zsh_test.go
@@ -64,3 +64,62 @@ func TestZshFormat_RCQuotesLonger(t *testing.T) {
 		t.Errorf("zsh RC_QUOTES longer: Words = %v, want [echo it's a test]", words)
 	}
 }
+
+func TestZshFormat_Operators(t *testing.T) {
+	tests := []struct {
+		input    string
+		wantType WordbreakType
+		wantRaw  string
+	}{
+		{"echo foo >| bar", WORDBREAK_REDIRECT_OUTPUT_FORCE, ">|"},
+		{"echo foo >>| bar", WORDBREAK_REDIRECT_OUTPUT_APPEND_FORCE, ">>|"},
+		{"echo foo ;& bar", WORDBREAK_LIST_FALLTHROUGH, ";&"},
+		{"echo foo ;| bar", WORDBREAK_LIST_FALLTHROUGH_RETRY, ";|"},
+		{"echo foo &| bar", WORDBREAK_LIST_ASYNC_ERRCHECK, "&|"},
+		{"echo foo ;; bar", WORDBREAK_LIST_SEQUENTIAL_DOUBLE, ";;"},
+	}
+	for _, tt := range tests {
+		tokens, err := SplitWith(tt.input, ZshFormat())
+		if err != nil {
+			t.Fatalf("zsh operator %q: %v", tt.wantRaw, err)
+		}
+		var found *Token
+		for i := range tokens {
+			if tokens[i].Type == WORDBREAK_TOKEN && tokens[i].RawValue == tt.wantRaw {
+				found = &tokens[i]
+				break
+			}
+		}
+		if found == nil {
+			t.Fatalf("zsh operator %q: not found in tokens %v", tt.wantRaw, tokens)
+		}
+		if found.WordbreakType != tt.wantType {
+			t.Errorf("zsh operator %q: WordbreakType = %v, want %v", tt.wantRaw, found.WordbreakType, tt.wantType)
+		}
+	}
+}
+
+func TestZshFormat_ForceRedirectIsRedirect(t *testing.T) {
+	ctx := SplitForCompletion("echo foo >| bar", ZshFormat())
+	if !ctx.IsRedirect {
+		t.Errorf("zsh >|: IsRedirect = false, want true")
+	}
+}
+
+func TestZshFormat_ForceAppendRedirectIsRedirect(t *testing.T) {
+	ctx := SplitForCompletion("echo foo >>| bar", ZshFormat())
+	if !ctx.IsRedirect {
+		t.Errorf("zsh >>|: IsRedirect = false, want true")
+	}
+}
+
+func TestZshFormat_FallthroughIsPipelineDelimiter(t *testing.T) {
+	tokens, err := SplitWith("echo foo ;& bar", ZshFormat())
+	if err != nil {
+		t.Fatal(err)
+	}
+	pipelines := tokens.Pipelines()
+	if len(pipelines) != 2 {
+		t.Errorf("zsh ;&: Pipelines = %d, want 2", len(pipelines))
+	}
+}
diff --git a/skills/shlex/references/comparison.md b/skills/shlex/references/comparison.md
@@ -64,8 +64,8 @@ These are the characters that break a word and are classified as `WORDBREAK_TOKE
 
 | Shell | Pipe | Redirect | Command sep | List operators | Other wordbreaks |
 |-------|------|----------|-------------|----------------|------------------|
-| **bash** | `\|` | `< > >> <<< <> <& &> &>>` | `;` `&` `&&` `\|\|` `\|&` | `&&` `\|\|` `&` `;` | `@ = ( :` + `COMP_WORDBREAKS` |
-| **zsh** | `\|` | `< > >> <<< <> <& &> &>>` `\|&` `=(...)` | `;` `&` `&&` `\|\|` | `&&` `\|\|` `&` `;` | `@ = ( :` + `WORDCHARS`/`FIGNORE` |
+| **bash** | `\|` | `< > >> >| <<< <> <& &> &>>` | `;` `&` `&&` `\|\|` `\|&` `;;` | `&&` `\|\|` `&` `;` `;;` | `@ = ( :` + `COMP_WORDBREAKS` |
+| **zsh** | `\|` | `< > >> >| <<< <> <& &> &>>` `\|&` `=(...)` | `;` `&` `&&` `\|\|` `;&` `;\|` `&\|` `;;` | `&&` `\|\|` `&` `;` `;&` `;\|` `&\|` `;;` | `@ = ( :` + `WORDCHARS`/`FIGNORE` |
 | **oil** | `\|` | `< > >> <<< <> <& &> &>>` | `;` `&` `&&` `\|\|` `\|&` | same | `@ = ( :` |
 | **tcsh** | `\|` | `< > >> << >& <&` | `;` `&` | `&&` `\|\|` `&` `;` | `( )` |
 | **fish** | `\|` | `< > >> >>? >? <>&` | `;` | `and` `or` `not` (keywords) | `( )` |
diff --git a/skills/shlex/references/format-bash.md b/skills/shlex/references/format-bash.md
@@ -99,6 +99,7 @@ This is critical for completion: bash's `COMP_WORDBREAKS` determines where the w
 | `<` | `<` | `WORDBREAK_REDIRECT_INPUT` | redirect |
 | `>` | `>` | `WORDBREAK_REDIRECT_OUTPUT` | redirect |
 | `>>` | `>>` | `WORDBREAK_REDIRECT_OUTPUT_APPEND` | redirect |
+| `>\|` | `>\|` | `WORDBREAK_REDIRECT_OUTPUT_FORCE` | redirect (noclobber override) |
 | `&>` / `>&` | `&>` / `>&` | `WORDBREAK_REDIRECT_OUTPUT_BOTH` | redirect |
 | `&>>` | `&>>` | `WORDBREAK_REDIRECT_OUTPUT_BOTH_APPEND` | redirect |
 | `<<<` | `<<<` | `WORDBREAK_REDIRECT_INPUT_STRING` | redirect |
@@ -110,11 +111,12 @@ This is critical for completion: bash's `COMP_WORDBREAKS` determines where the w
 | `;` | `;` | `WORDBREAK_LIST_SEQUENTIAL` | list |
 | `&&` | `&&` | `WORDBREAK_LIST_AND` | list |
 | `\|\|` | `\|\|` | `WORDBREAK_LIST_OR` | list |
+| `;;` | `;;` | `WORDBREAK_LIST_SEQUENTIAL_DOUBLE` | list (case terminator) |
 
 Multi-char operators are matched greedily in the `WORDBREAK_STATE` (consecutive wordbreak runes accumulate into one `WORDBREAK_TOKEN`).
 
-`IsPipelineDelimiter()` → `|`, `|&`, `&`, `;`, `&&`, `||` — these split pipelines in `CurrentPipeline()`.
-`IsRedirect()` → the eight redirect operators — these are stripped in `FilterRedirects()`.
+`IsPipelineDelimiter()` → `|`, `|&`, `&`, `;`, `&&`, `||`, `;;` — these split pipelines in `CurrentPipeline()`.
+`IsRedirect()` → the redirect operators including `>|` — these are stripped in `FilterRedirects()`.
 
 ### File-descriptor prefixes
 
diff --git a/skills/shlex/references/format-zsh.md b/skills/shlex/references/format-zsh.md
@@ -82,11 +82,20 @@ Zsh has a 5-state model (vs bash's 2) because of the "full quoting" states where
 
 ## Operators
 
-Same operator grammar as bash (`|`, `||`, `|&`, `&`, `;`, `&&`, `<`, `>`, `>>`, `<<<`, `<>`, `<&`, `&>`, `&>>`). See [format-bash.md](format-bash.md#operators-wordbreaktype) for the full table.
+Same operator grammar as bash (`|`, `||`, `|&`, `&`, `;`, `&&`, `<`, `>`, `>>`, `>|`, `<<<`, `<>`, `<&`, `&>`, `&>>`, `;;`). See [format-bash.md](format-bash.md#operators-wordbreaktype) for the full table.
 
-Zsh additions (not in bash):
-- **`=(...)` process substitution** — `=` as a wordbreak followed by `(...)`.
-- **Glob qualifiers** `(...)` after a path — these are word breaks but not operators in the pipeline sense.
+Zsh-specific operators (not in bash):
+
+| Operator | RawValue | WordbreakType | Category |
+|----------|----------|---------------|----------|
+| `>>\|` | `>>\|` | `WORDBREAK_REDIRECT_OUTPUT_APPEND_FORCE` | redirect (noclobber override) |
+| `;&` | `;&` | `WORDBREAK_LIST_FALLTHROUGH` | list (case fall-through) |
+| `;\|` | `;\|` | `WORDBREAK_LIST_FALLTHROUGH_RETRY` | list (case fall-through with retry) |
+| `&\|` | `&\|` | `WORDBREAK_LIST_ASYNC_ERRCHECK` | list (background with error check) |
+
+Three of the four are pipeline delimiters (`IsPipelineDelimiter() == true`): `;&`, `;|`, `&|`. The exception is `>>|`, which is a redirect (`IsRedirect() == true`) and therefore not a pipeline delimiter.
+
+Zsh also shares `>|` (force output redirect) and `;;` (case terminator) with bash — both are classified in `bashWordbreakType`.
 
 ## Comments
 
@@ -95,8 +104,14 @@ Zsh additions (not in bash):
 ## Edge Cases
 
 - **`RC_QUOTES` off (default)**: `''` closes then reopens single quotes (same as bash).
+- **`RC_QUOTES` inside `$'...'`**: zsh disables RC_QUOTES inside `$'...'` (ANSI-C quoting). The lexer cannot distinguish `$'...'` from `'...'`, so `$'it''s'` with `RC_QUOTES` would incorrectly produce `it's` instead of two segments. Unlikely in completion input.
 - **Named directories**: zsh's `hash -d` creates `~name` expansions. Carapace handles this via `NamedDirectories.Matches` in `quoteValue`, not in the lexer.
 - **`FULL_QUOTING_*_STATE` quirk**: when a word both starts and ends with the same quote, zsh places the trailing space *inside* the quote. Carapace forces nospace in these states.
+- **`=(...)` process substitution**: `=` is a wordbreak (in `BASH_WORDBREAKS`) but classified as `WORDBREAK_UNKNOWN`, not a redirect.
+- **Glob qualifiers** `(...)` after a path: these are word breaks but not operators in the pipeline sense.
+- **Backslash-newline line continuation**: zsh consumes `\` + newline entirely (no character added). The lexer's `ESCAPING_STATE` adds the newline as a literal. Rare in single-line completion input.
+- **`INTERACTIVECOMMENTS`**: `#` is only a comment in interactive mode when this option is set (on by default in modern zsh). The lexer always treats `#` as a comment.
+- **`CSHJUNKIEQUOTES`**: with this tcsh-compatibility option, newlines inside single quotes close the quote. Very niche, not handled.
 
 ## References
 
diff --git a/wordbreak.go b/wordbreak.go
@@ -12,6 +12,8 @@ const (
 	WORDBREAK_REDIRECT_INPUT
 	WORDBREAK_REDIRECT_OUTPUT
 	WORDBREAK_REDIRECT_OUTPUT_APPEND
+	WORDBREAK_REDIRECT_OUTPUT_FORCE        // >| (noclobber override)
+	WORDBREAK_REDIRECT_OUTPUT_APPEND_FORCE // >>| (noclobber override for append)
 	WORDBREAK_REDIRECT_OUTPUT_BOTH
 	WORDBREAK_REDIRECT_OUTPUT_BOTH_APPEND
 	WORDBREAK_REDIRECT_INPUT_STRING
@@ -27,6 +29,10 @@ const (
 	WORDBREAK_LIST_SEQUENTIAL
 	WORDBREAK_LIST_AND
 	WORDBREAK_LIST_OR
+	WORDBREAK_LIST_SEQUENTIAL_DOUBLE // ;; (case terminator)
+	WORDBREAK_LIST_FALLTHROUGH       // ;& (zsh case fall-through)
+	WORDBREAK_LIST_FALLTHROUGH_RETRY // ;| (zsh case fall-through with retry)
+	WORDBREAK_LIST_ASYNC_ERRCHECK    // &| (zsh background with error check)
 	// COMP_WORDBREAKS
 	WORDBREAK_CUSTOM
 	// Elvish-specific: output capture delimiters ( and )
@@ -36,25 +42,31 @@ const (
 )
 
 var wordbreakTypes = map[WordbreakType]string{
-	WORDBREAK_UNKNOWN:                     "WORDBREAK_UNKNOWN",
-	WORDBREAK_REDIRECT_INPUT:              "WORDBREAK_REDIRECT_INPUT",
-	WORDBREAK_REDIRECT_OUTPUT:             "WORDBREAK_REDIRECT_OUTPUT",
-	WORDBREAK_REDIRECT_OUTPUT_APPEND:      "WORDBREAK_REDIRECT_OUTPUT_APPEND",
-	WORDBREAK_REDIRECT_OUTPUT_BOTH:        "WORDBREAK_REDIRECT_OUTPUT_BOTH",
-	WORDBREAK_REDIRECT_OUTPUT_BOTH_APPEND: "WORDBREAK_REDIRECT_OUTPUT_BOTH_APPEND",
-	WORDBREAK_REDIRECT_INPUT_STRING:       "WORDBREAK_REDIRECT_INPUT_STRING",
-	WORDBREAK_REDIRECT_INPUT_DUPLICATE:    "WORDBREAK_REDIRECT_INPUT_DUPLICATE",
-	WORDBREAK_REDIRECT_INPUT_OUTPUT:       "WORDBREAK_REDIRECT_INPUT_OUTPUT",
-	WORDBREAK_PIPE:                        "WORDBREAK_PIPE",
-	WORDBREAK_PIPE_WITH_STDERR:            "WORDBREAK_PIPE_WITH_STDERR",
-	WORDBREAK_LAMBDA_PIPE:                 "WORDBREAK_LAMBDA_PIPE",
-	WORDBREAK_LIST_ASYNC:                  "WORDBREAK_LIST_ASYNC",
-	WORDBREAK_LIST_SEQUENTIAL:             "WORDBREAK_LIST_SEQUENTIAL",
-	WORDBREAK_LIST_AND:                    "WORDBREAK_LIST_AND",
-	WORDBREAK_LIST_OR:                     "WORDBREAK_LIST_OR",
-	WORDBREAK_CUSTOM:                      "WORDBREAK_CUSTOM",
-	WORDBREAK_OUTPUT_CAPTURE:              "WORDBREAK_OUTPUT_CAPTURE",
-	WORDBREAK_BRACKET:                     "WORDBREAK_BRACKET",
+	WORDBREAK_UNKNOWN:                      "WORDBREAK_UNKNOWN",
+	WORDBREAK_REDIRECT_INPUT:               "WORDBREAK_REDIRECT_INPUT",
+	WORDBREAK_REDIRECT_OUTPUT:              "WORDBREAK_REDIRECT_OUTPUT",
+	WORDBREAK_REDIRECT_OUTPUT_APPEND:       "WORDBREAK_REDIRECT_OUTPUT_APPEND",
+	WORDBREAK_REDIRECT_OUTPUT_FORCE:        "WORDBREAK_REDIRECT_OUTPUT_FORCE",
+	WORDBREAK_REDIRECT_OUTPUT_APPEND_FORCE: "WORDBREAK_REDIRECT_OUTPUT_APPEND_FORCE",
+	WORDBREAK_REDIRECT_OUTPUT_BOTH:         "WORDBREAK_REDIRECT_OUTPUT_BOTH",
+	WORDBREAK_REDIRECT_OUTPUT_BOTH_APPEND:  "WORDBREAK_REDIRECT_OUTPUT_BOTH_APPEND",
+	WORDBREAK_REDIRECT_INPUT_STRING:        "WORDBREAK_REDIRECT_INPUT_STRING",
+	WORDBREAK_REDIRECT_INPUT_DUPLICATE:     "WORDBREAK_REDIRECT_INPUT_DUPLICATE",
+	WORDBREAK_REDIRECT_INPUT_OUTPUT:        "WORDBREAK_REDIRECT_INPUT_OUTPUT",
+	WORDBREAK_PIPE:                         "WORDBREAK_PIPE",
+	WORDBREAK_PIPE_WITH_STDERR:             "WORDBREAK_PIPE_WITH_STDERR",
+	WORDBREAK_LAMBDA_PIPE:                  "WORDBREAK_LAMBDA_PIPE",
+	WORDBREAK_LIST_ASYNC:                   "WORDBREAK_LIST_ASYNC",
+	WORDBREAK_LIST_SEQUENTIAL:              "WORDBREAK_LIST_SEQUENTIAL",
+	WORDBREAK_LIST_AND:                     "WORDBREAK_LIST_AND",
+	WORDBREAK_LIST_OR:                      "WORDBREAK_LIST_OR",
+	WORDBREAK_LIST_SEQUENTIAL_DOUBLE:       "WORDBREAK_LIST_SEQUENTIAL_DOUBLE",
+	WORDBREAK_LIST_FALLTHROUGH:             "WORDBREAK_LIST_FALLTHROUGH",
+	WORDBREAK_LIST_FALLTHROUGH_RETRY:       "WORDBREAK_LIST_FALLTHROUGH_RETRY",
+	WORDBREAK_LIST_ASYNC_ERRCHECK:          "WORDBREAK_LIST_ASYNC_ERRCHECK",
+	WORDBREAK_CUSTOM:                       "WORDBREAK_CUSTOM",
+	WORDBREAK_OUTPUT_CAPTURE:               "WORDBREAK_OUTPUT_CAPTURE",
+	WORDBREAK_BRACKET:                      "WORDBREAK_BRACKET",
 }
 
 func (w WordbreakType) MarshalJSON() ([]byte, error) {
@@ -69,7 +81,11 @@ func (w WordbreakType) IsPipelineDelimiter() bool {
 		WORDBREAK_LIST_ASYNC,
 		WORDBREAK_LIST_SEQUENTIAL,
 		WORDBREAK_LIST_AND,
-		WORDBREAK_LIST_OR:
+		WORDBREAK_LIST_OR,
+		WORDBREAK_LIST_SEQUENTIAL_DOUBLE,
+		WORDBREAK_LIST_FALLTHROUGH,
+		WORDBREAK_LIST_FALLTHROUGH_RETRY,
+		WORDBREAK_LIST_ASYNC_ERRCHECK:
 		return true
 	default:
 		return false
@@ -82,6 +98,8 @@ func (w WordbreakType) IsRedirect() bool {
 		WORDBREAK_REDIRECT_INPUT,
 		WORDBREAK_REDIRECT_OUTPUT,
 		WORDBREAK_REDIRECT_OUTPUT_APPEND,
+		WORDBREAK_REDIRECT_OUTPUT_FORCE,
+		WORDBREAK_REDIRECT_OUTPUT_APPEND_FORCE,
 		WORDBREAK_REDIRECT_OUTPUT_BOTH,
 		WORDBREAK_REDIRECT_OUTPUT_BOTH_APPEND,
 		WORDBREAK_REDIRECT_INPUT_STRING,
@@ -103,6 +121,8 @@ func bashWordbreakType(raw string) WordbreakType {
 		return WORDBREAK_REDIRECT_OUTPUT
 	case ">>":
 		return WORDBREAK_REDIRECT_OUTPUT_APPEND
+	case ">|":
+		return WORDBREAK_REDIRECT_OUTPUT_FORCE
 	case "&>", ">&":
 		return WORDBREAK_REDIRECT_OUTPUT_BOTH
 	case "&>>":
@@ -125,6 +145,8 @@ func bashWordbreakType(raw string) WordbreakType {
 		return WORDBREAK_LIST_AND
 	case "||":
 		return WORDBREAK_LIST_OR
+	case ";;":
+		return WORDBREAK_LIST_SEQUENTIAL_DOUBLE
 	default:
 		// TODO check COMP_WORDBREAKS -> WORDBREAK_OTHER
 		return WORDBREAK_UNKNOWN
