Add PowerShell block comments, line continuation, stop-parsing, and stream redirects

Implement four PowerShell-specific tokenizer features identified from
the PowerShell source code (tokenizer.cs, parser.cs):

- Block comments <# ... #>: multi-line comments via new BlockCommenter
  interface, with a dedicated BLOCK_COMMENT_STATE that scans until the
  #> closer
- Backtick line continuation: backtick + newline is consumed and
  discarded (not added to word value) via new LineContinuationEscaper
  interface, matching PowerShell's LineContinuation token behavior
- --% stop-parsing token: raw lexing mode for the remainder of the line
  via new StopParsingToken interface, with dedicated scanStopParsing
  that reads until newline or | (double-quote toggles literal mode)
- Stream redirects 2>, 2>>, 2>&1, 1>&2, *>, *>>: merged in PostProcess
  from adjacent WORD_TOKEN + WORDBREAK_TOKEN pairs

Assisted-by: Crush:glm-5.2

Author: rsteube

format.go

@@ -90,3 +90,41 @@ type EscapingQuoteUnescaper interface {
 type PostProcessor interface {
 	PostProcess(tokens TokenSlice) TokenSlice
 }
+
+// LineContinuationEscaper is an optional interface for formats where the
+// escape character followed by a newline (or carriage return) acts as a
+// line continuation — the escape+newline sequence is consumed and discarded,
+// NOT added to the word value. This matches PowerShell's backtick line
+// continuation behavior.
+//
+// Without this interface, the ESCAPING_STATE handler always adds the
+// post-escape rune to the word value, which is correct for POSIX shells
+// where backslash-newline is handled differently.
+type LineContinuationEscaper interface {
+	// IsLineContinuation returns true if the rune following the escape
+	// character should be treated as a line continuation. The parameter
+	// is the rune that follows the escape character (e.g. '\n' or '\r').
+	IsLineContinuation(r rune) bool
+}
+
+// BlockCommenter is an optional interface for formats that support
+// multi-line block comments (e.g. PowerShell's <# ... #>). When the
+// tokenizer encounters the blockCommentOpener runes at a word boundary,
+// it enters a dedicated BLOCK_COMMENT_STATE that scans until the
+// blockCommentCloser runes are found, spanning multiple lines.
+type BlockCommenter interface {
+	BlockCommentOpener() string  // e.g. "<#" for PowerShell
+	BlockCommentCloser() string  // e.g. "#>" for PowerShell
+}
+
+// StopParsingToken is an optional interface for formats that support a
+// stop-parsing token (e.g. PowerShell's --%). When the tokenizer encounters
+// this token as a bare word, it switches to a raw lexing mode for the
+// remainder of the line (until newline or pipeline delimiter), where
+// all characters except the pipeline delimiters are treated as literal
+// word content.
+type StopParsingToken interface {
+	// StopParsingWord returns the literal token that triggers raw mode.
+	// e.g. "--%" for PowerShell.
+	StopParsingWord() string
+}

format_powershell.go

@@ -7,7 +7,11 @@ package shlex
 //   - "" inside double quotes → literal " (doubled quote)
 //   - No single-quote-as-quote for outer quote pairs in the POSIX sense;
 //     both ' and " are quote chars
-//   - Here-strings (@'...'@, @"..."@) and --% are deferred to Phase 4
+//   - Backtick + newline is line continuation (consumed, not part of word)
+//   - Block comments <# ... #> (multi-line)
+//   - --% stop-parsing token (raw mode for remainder of line)
+//   - Stream redirects: 2>, 2>>, 2>&1, 1>&2, *>, *>> (merged in PostProcess)
+//   - Here-strings (@'...'@, @"..."@) are deferred
 type powershellFormat struct{}
 
 // PowershellFormat returns the PowerShell lexical format.
@@ -65,3 +69,93 @@ func (powershellFormat) EscapingQuoteEscapeChars() map[rune]bool { return nil }
 func (powershellFormat) QuoteWord(s string) string               { return powershellQuoteWord(s) }
 func (powershellFormat) TripleQuoteSupport() bool                { return false }
 func (powershellFormat) RawPrefixSupport() bool                  { return false }
+
+// IsLineContinuation implements LineContinuationEscaper. PowerShell's
+// backtick followed by \n or \r is a line continuation — the sequence is
+// consumed and the word continues on the next line.
+func (powershellFormat) IsLineContinuation(r rune) bool {
+	return r == '\n' || r == '\r'
+}
+
+// BlockCommentOpener implements BlockCommenter. PowerShell supports
+// multi-line block comments delimited by <# and #>.
+func (powershellFormat) BlockCommentOpener() string { return "<#" }
+
+// BlockCommentCloser implements BlockCommenter.
+func (powershellFormat) BlockCommentCloser() string { return "#>" }
+
+// StopParsingWord implements StopParsingToken. PowerShell's --% token
+// stops PowerShell from interpreting subsequent input.
+func (powershellFormat) StopParsingWord() string { return "--%" }
+
+// PostProcess merges PowerShell stream-redirect operators. The tokenizer
+// produces e.g. `2` as a WORD_TOKEN and `>` (or `>>`) as a WORDBREAK_TOKEN.
+// This step detects adjacent word+wordbreak sequences like `2>`, `2>>`,
+// `2>&1`, `1>&2`, `*>`, `*>>` and reclassifies them as single
+// WORDBREAK_TOKENs with the appropriate WordbreakType.
+func (powershellFormat) PostProcess(tokens TokenSlice) TokenSlice {
+	result := make(TokenSlice, 0, len(tokens))
+	for i := 0; i < len(tokens); i++ {
+		t := tokens[i]
+
+		// Look for bare WORD_TOKEN (digit or *) immediately followed by
+		// WORDBREAK_TOKEN starting with '>' (redirect operator)
+		if t.Type == WORD_TOKEN && t.Value == t.RawValue && i+1 < len(tokens) {
+			next := tokens[i+1]
+			if next.Type == WORDBREAK_TOKEN && next.adjoins(t) &&
+				next.WordbreakType.IsRedirect() && len(next.RawValue) > 0 && next.RawValue[0] == '>' {
+				// Check if the word is a valid stream number or *
+				if t.Value == "*" || (len(t.Value) == 1 && t.Value[0] >= '1' && t.Value[0] <= '6') {
+					// Check for merging redirect: next token after > is &N
+					// e.g. 2>&1 — the & and digit are separate wordbreak/word tokens
+					wbType := next.WordbreakType
+					mergedRaw := t.RawValue + next.RawValue
+					mergedVal := t.Value + next.Value
+					mergedSpan := Span{Start: t.Span.Start, End: next.Span.End}
+
+					// Check for &N pattern (stream merge) in the token after next
+					if i+2 < len(tokens) && tokens[i+2].Type == WORDBREAK_TOKEN &&
+						tokens[i+2].Value == "&" && tokens[i+2].adjoins(next) {
+						if i+3 < len(tokens) && tokens[i+3].Type == WORD_TOKEN &&
+							tokens[i+3].Value == tokens[i+3].RawValue &&
+							tokens[i+3].adjoins(tokens[i+2]) &&
+							len(tokens[i+3].Value) == 1 &&
+							(tokens[i+3].Value[0] == '1' || tokens[i+3].Value[0] == '2') {
+							// 2>&1 pattern — merge all four tokens
+							mergedRaw += tokens[i+2].RawValue + tokens[i+3].RawValue
+							mergedVal += tokens[i+2].Value + tokens[i+3].Value
+							mergedSpan.End = tokens[i+3].Span.End
+							wbType = WORDBREAK_REDIRECT_OUTPUT_BOTH
+							merged := Token{
+								Type:          WORDBREAK_TOKEN,
+								Value:         mergedVal,
+								RawValue:      mergedRaw,
+								Span:          mergedSpan,
+								State:         tokens[i+3].State,
+								WordbreakType: wbType,
+							}
+							result = append(result, merged)
+							i += 3
+							continue
+						}
+					}
+
+					merged := Token{
+						Type:          WORDBREAK_TOKEN,
+						Value:         mergedVal,
+						RawValue:      mergedRaw,
+						Span:          mergedSpan,
+						State:         next.State,
+						WordbreakType: wbType,
+					}
+					result = append(result, merged)
+					i += 1
+					continue
+				}
+			}
+		}
+
+		result = append(result, t)
+	}
+	return result
+}

format_powershell_test.go

@@ -111,3 +111,188 @@ func TestPowershellFormat_OpenDoubleQuote(t *testing.T) {
 		t.Errorf("powershell open double: State = %v, want QUOTING_ESCAPING_STATE", last.State)
 	}
 }
+
+func TestPowershellFormat_BacktickLineContinuation(t *testing.T) {
+	// backtick + newline should be consumed as line continuation, not part of word
+	tokens, err := SplitWith("echo foo`\nbar", PowershellFormat())
+	if err != nil {
+		t.Fatal(err)
+	}
+	words := tokens.Words().Strings()
+	if len(words) != 2 || words[1] != "foobar" {
+		t.Errorf("powershell line continuation: Words = %v, want [echo foobar]", words)
+	}
+}
+
+func TestPowershellFormat_BacktickLineContinuationCRLF(t *testing.T) {
+	tokens, err := SplitWith("echo foo`\r\nbar", PowershellFormat())
+	if err != nil {
+		t.Fatal(err)
+	}
+	words := tokens.Words().Strings()
+	if len(words) != 2 || words[1] != "foobar" {
+		t.Errorf("powershell line continuation CRLF: Words = %v, want [echo foobar]", words)
+	}
+}
+
+func TestPowershellFormat_BacktickLineContinuationStartOfWord(t *testing.T) {
+	// backtick + newline at start of word — word continues on next line
+	tokens, err := SplitWith("echo `\nbar", PowershellFormat())
+	if err != nil {
+		t.Fatal(err)
+	}
+	words := tokens.Words().Strings()
+	if len(words) != 2 || words[1] != "bar" {
+		t.Errorf("powershell line continuation start: Words = %v, want [echo bar]", words)
+	}
+}
+
+func TestPowershellFormat_BlockComment(t *testing.T) {
+	tokens, err := SplitWith("echo <# multi\nline\ncomment #> foo", PowershellFormat())
+	if err != nil {
+		t.Fatal(err)
+	}
+	words := tokens.Words().Strings()
+	if len(words) != 2 || words[0] != "echo" || words[1] != "foo" {
+		t.Errorf("powershell block comment: Words = %v, want [echo foo]", words)
+	}
+}
+
+func TestPowershellFormat_BlockCommentSingleLine(t *testing.T) {
+	tokens, err := SplitWith("echo <# inline comment #> foo", PowershellFormat())
+	if err != nil {
+		t.Fatal(err)
+	}
+	words := tokens.Words().Strings()
+	if len(words) != 2 || words[0] != "echo" || words[1] != "foo" {
+		t.Errorf("powershell block comment inline: Words = %v, want [echo foo]", words)
+	}
+}
+
+func TestPowershellFormat_StopParsingToken(t *testing.T) {
+	// After --%, everything is literal until newline or |
+	tokens, err := SplitWith("echo --% /grant Dom\\HVAdmin:(CI)(OI)F", PowershellFormat())
+	if err != nil {
+		t.Fatal(err)
+	}
+	words := tokens.Words().Strings()
+	// --% should be a word, then the rest is raw text as one word
+	if len(words) < 3 {
+		t.Errorf("powershell --%%: Words = %v, expected at least 3 words", words)
+	}
+	if words[0] != "echo" {
+		t.Errorf("powershell --%%: first word = %q, want echo", words[0])
+	}
+	if words[1] != "--%" {
+		t.Errorf("powershell --%%: second word = %q, want --%%", words[1])
+	}
+}
+
+func TestPowershellFormat_StopParsingPipeDelim(t *testing.T) {
+	// After --%, | is still a pipeline delimiter
+	tokens, err := SplitWith("echo --% foo | Select-String bar", PowershellFormat())
+	if err != nil {
+		t.Fatal(err)
+	}
+	pipelines := tokens.Pipelines()
+	if len(pipelines) != 2 {
+		t.Errorf("powershell --%% pipe: %d pipelines, want 2", len(pipelines))
+	}
+}
+
+func TestPowershellFormat_StopParsingRawContent(t *testing.T) {
+	// After --%, content like (CI) should be literal, not split
+	tokens, err := SplitWith("icacls X: --% /grant Dom\\HVAdmin:(CI)(OI)F", PowershellFormat())
+	if err != nil {
+		t.Fatal(err)
+	}
+	words := tokens.Words().Strings()
+	// The raw content after --% should be one word
+	if len(words) != 4 {
+		t.Errorf("powershell --%% raw: Words = %v, want 4 words", words)
+	}
+	if words[2] != "--%" {
+		t.Errorf("powershell --%% raw: third word = %q, want --%%", words[2])
+	}
+	rawContent := words[3]
+	if rawContent != "/grant Dom\\HVAdmin:(CI)(OI)F" {
+		t.Errorf("powershell --%% raw: content = %q, want /grant Dom\\HVAdmin:(CI)(OI)F", rawContent)
+	}
+}
+
+func TestPowershellFormat_StreamRedirect2(t *testing.T) {
+	// 2> should be recognized as a stream redirect
+	tokens, err := SplitWith("echo foo 2> error.txt", PowershellFormat())
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Check that 2> is a single WORDBREAK_TOKEN with redirect type
+	found := false
+	for _, tok := range tokens {
+		if tok.Type == WORDBREAK_TOKEN && tok.RawValue == "2>" {
+			if !tok.WordbreakType.IsRedirect() {
+				t.Errorf("powershell 2>: WordbreakType = %v, want redirect", tok.WordbreakType)
+			}
+			found = true
+		}
+	}
+	if !found {
+		t.Errorf("powershell 2>: no merged 2> token found in %v", tokens)
+	}
+}
+
+func TestPowershellFormat_StreamRedirect2Append(t *testing.T) {
+	tokens, err := SplitWith("echo foo 2>> error.txt", PowershellFormat())
+	if err != nil {
+		t.Fatal(err)
+	}
+	found := false
+	for _, tok := range tokens {
+		if tok.Type == WORDBREAK_TOKEN && tok.RawValue == "2>>" {
+			if !tok.WordbreakType.IsRedirect() {
+				t.Errorf("powershell 2>>: WordbreakType = %v, want redirect", tok.WordbreakType)
+			}
+			found = true
+		}
+	}
+	if !found {
+		t.Errorf("powershell 2>>: no merged 2>> token found in %v", tokens)
+	}
+}
+
+func TestPowershellFormat_StreamRedirectMerge(t *testing.T) {
+	// 2>&1 should be recognized as a merged stream redirect
+	tokens, err := SplitWith("echo foo 2>&1", PowershellFormat())
+	if err != nil {
+		t.Fatal(err)
+	}
+	found := false
+	for _, tok := range tokens {
+		if tok.Type == WORDBREAK_TOKEN && tok.RawValue == "2>&1" {
+			found = true
+		}
+	}
+	if !found {
+		t.Errorf("powershell 2>&1: no merged token found in %v", tokens)
+	}
+}
+
+func TestPowershellFormat_StreamRedirectStar(t *testing.T) {
+	// *> should be recognized as all-streams redirect
+	tokens, err := SplitWith("echo foo *> output.txt", PowershellFormat())
+	if err != nil {
+		t.Fatal(err)
+	}
+	found := false
+	for _, tok := range tokens {
+		if tok.Type == WORDBREAK_TOKEN && tok.RawValue == "*>" {
+			if !tok.WordbreakType.IsRedirect() {
+				t.Errorf("powershell *>: WordbreakType = %v, want redirect", tok.WordbreakType)
+			}
+			found = true
+		}
+	}
+	if !found {
+		t.Errorf("powershell *>: no merged *> token found in %v", tokens)
+	}
+}