Fix heredoc leading whitespace stripping (#52)

* Fix heredoc leading whitespace stripping

The heredoc content was losing leading whitespace due to two issues:

1. In lexer.ts, skipWhitespace() was called before checking for pending
   heredocs, which stripped leading whitespace from the first line of
   heredoc content. Fixed by moving the heredoc check before skipWhitespace().

2. In Bash.ts, the script normalization (trimStart on every line) was
   stripping whitespace from heredoc content. Fixed by making the
   normalizeScript function heredoc-aware so it preserves content inside
   heredocs while still normalizing the surrounding script.

Added tests for whitespace preservation in heredocs.

Co-Authored-By: Claude Opus 4.5 <[email protected]>

* Fix heredoc leading whitespace stripping

The heredoc content was losing leading whitespace due to two issues:

1. In lexer.ts, skipWhitespace() was called before checking for pending
   heredocs, which stripped leading whitespace from the first line of
   heredoc content. Fixed by moving the heredoc check before skipWhitespace().

2. In Bash.ts, the script normalization (trimStart on every line) was
   stripping whitespace from heredoc content. Fixed by making the
   normalizeScript function heredoc-aware so it preserves content inside
   heredocs while still normalizing the surrounding script.

Added tests for whitespace preservation in heredocs.

Co-Authored-By: Claude Opus 4.5 <[email protected]>

* Fix heredoc delimiter matching edge cases

1. Don't use trimStart() for regular << heredocs - delimiter must match
   exactly. A line like "  EOF" should be content, not end the heredoc.

2. Allow hyphens in heredoc delimiters (e.g., END-TEST) by using
   [\w-]+ instead of \w+ in the regex pattern.

Added tests for both edge cases.

Co-Authored-By: Claude Opus 4.5 <[email protected]>

* Fix heredoc delimiter matching edge cases

1. Don't use trimStart() for regular << heredocs - delimiter must match
   exactly. A line like "  EOF" should be content, not end the heredoc.

2. Allow hyphens in heredoc delimiters (e.g., END-TEST) by using
   [\w-]+ instead of \w+ in the regex pattern.

Added tests for both edge cases.

Co-Authored-By: Claude Opus 4.5 <[email protected]>

* Don't skip some read tests

---------

Co-authored-by: Claude Opus 4.5 <[email protected]>

Author: tobiaslins

src/Bash.ts

@@ -351,12 +351,11 @@ export class Bash {
     };
 
     // Normalize indented multi-line scripts (unless rawScript is true)
+    // This allows writing indented bash scripts in template literals
+    // BUT we must preserve whitespace inside heredoc content
     let normalized = commandLine;
     if (!options?.rawScript) {
-      const normalizedLines = commandLine
-        .split("\n")
-        .map((line) => line.trimStart());
-      normalized = normalizedLines.join("\n");
+      normalized = normalizeScript(commandLine);
     }
 
     try {
@@ -449,3 +448,63 @@ export class Bash {
     return { ...this.state.env };
   }
 }
+
+/**
+ * Normalize a script by stripping leading whitespace from lines,
+ * while preserving whitespace inside heredoc content.
+ *
+ * This allows writing indented bash scripts in template literals:
+ * ```
+ * await bash.exec(`
+ *   if [ -f foo ]; then
+ *     echo "yes"
+ *   fi
+ * `);
+ * ```
+ *
+ * Heredocs are detected by looking for << or <<- operators and their delimiters.
+ */
+function normalizeScript(script: string): string {
+  const lines = script.split("\n");
+  const result: string[] = [];
+
+  // Stack of pending heredoc delimiters (for nested heredocs)
+  const pendingDelimiters: { delimiter: string; stripTabs: boolean }[] = [];
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+
+    // If we're inside a heredoc, check if this line ends it
+    if (pendingDelimiters.length > 0) {
+      const current = pendingDelimiters[pendingDelimiters.length - 1];
+      // For <<-, strip leading tabs when checking delimiter
+      // For <<, require exact match (no leading whitespace allowed)
+      const lineToCheck = current.stripTabs ? line.replace(/^\t+/, "") : line;
+      if (lineToCheck === current.delimiter) {
+        // End of heredoc - this line can be normalized
+        result.push(line.trimStart());
+        pendingDelimiters.pop();
+        continue;
+      }
+      // Inside heredoc - preserve the line exactly as-is
+      result.push(line);
+      continue;
+    }
+
+    // Not inside a heredoc - normalize the line and check for heredoc starts
+    const normalizedLine = line.trimStart();
+    result.push(normalizedLine);
+
+    // Check for heredoc operators in this line
+    // Match: <<DELIM, <<-DELIM, << 'DELIM', <<- "DELIM", etc.
+    // Multiple heredocs on one line are possible: cmd <<EOF1 <<EOF2
+    const heredocPattern = /<<(-?)\s*(['"]?)([\w-]+)\2/g;
+    for (const match of normalizedLine.matchAll(heredocPattern)) {
+      const stripTabs = match[1] === "-";
+      const delimiter = match[3];
+      pendingDelimiters.push({ delimiter, stripTabs });
+    }
+  }
+
+  return result.join("\n");
+}

src/parser/lexer.ts

@@ -244,11 +244,8 @@ export class Lexer {
     const pendingHeredocs = this.pendingHeredocs;
 
     while (this.pos < len) {
-      this.skipWhitespace();
-
-      if (this.pos >= len) break;
-
-      // Check for pending here-documents after newline
+      // Check for pending here-documents after newline BEFORE skipping whitespace
+      // to preserve leading whitespace in heredoc content
       if (
         pendingHeredocs.length > 0 &&
         tokens.length > 0 &&
@@ -258,6 +255,10 @@ export class Lexer {
         continue;
       }
 
+      this.skipWhitespace();
+
+      if (this.pos >= len) break;
+
       const token = this.nextToken();
       if (token) {
         tokens.push(token);

src/spec-tests/cases/builtin-read.test.sh

@@ -416,7 +416,6 @@ b
 ## N-I dash/zsh stdout-json: ""
 
 #### read with IFS=$'\n'
-## SKIP: IFS with newline character not implemented
 # The leading spaces are stripped if they appear in IFS.
 IFS=$(echo -e '\n')
 read var <<EOF
@@ -444,7 +443,6 @@ printf "%s\n" "[$a|$b|$c|$d]"
 ## stdout: [  \a |b: c|d  e|]
 
 #### read with IFS=''
-## SKIP: Read with special IFS values not implemented
 IFS=''
 read x y <<EOF
   a b c d

src/syntax/here-document.test.ts

@@ -125,4 +125,108 @@ echo done`);
     expect(result.stdout).toBe("hello\ndone\n");
     expect(result.exitCode).toBe(0);
   });
+
+  describe("whitespace preservation", () => {
+    it("should preserve leading spaces in here document content", async () => {
+      const env = new Bash();
+      const result = await env.exec(`cat <<EOF
+    four spaces at start
+  two spaces at start
+no spaces
+EOF`);
+      expect(result.stdout).toBe(
+        "    four spaces at start\n  two spaces at start\nno spaces\n",
+      );
+      expect(result.exitCode).toBe(0);
+    });
+
+    it("should preserve leading tabs in here document content (not <<-)", async () => {
+      const env = new Bash();
+      const result = await env.exec(`cat <<EOF
+\tleading tab
+no tab
+EOF`);
+      expect(result.stdout).toBe("\tleading tab\nno tab\n");
+      expect(result.exitCode).toBe(0);
+    });
+
+    it("should preserve mixed whitespace in here document", async () => {
+      const env = new Bash();
+      const result = await env.exec(`cat <<EOF
+  spaces
+\ttab
+  \tmixed
+EOF`);
+      expect(result.stdout).toBe("  spaces\n\ttab\n  \tmixed\n");
+      expect(result.exitCode).toBe(0);
+    });
+
+    it("should preserve whitespace even when script is indented", async () => {
+      const env = new Bash();
+      // This tests that the script normalization doesn't strip heredoc content
+      const result = await env.exec(`
+        cat <<EOF
+    indented content
+        more indented
+EOF
+      `);
+      expect(result.stdout).toBe(
+        "    indented content\n        more indented\n",
+      );
+      expect(result.exitCode).toBe(0);
+    });
+
+    it("should preserve ASCII art triangle with leading spaces", async () => {
+      const env = new Bash();
+      const result = await env.exec(`cat <<'EOF'
+                    *
+                   * *
+                  *   *
+                 *     *
+                *       *
+               *         *
+              *           *
+             *             *
+            *               *
+           *                 *
+          *                   *
+         *********************
+EOF`);
+      expect(result.stdout).toBe(`                    *
+                   * *
+                  *   *
+                 *     *
+                *       *
+               *         *
+              *           *
+             *             *
+            *               *
+           *                 *
+          *                   *
+         *********************
+`);
+      expect(result.exitCode).toBe(0);
+    });
+
+    it("should not treat indented delimiter as end of heredoc", async () => {
+      const env = new Bash();
+      // A line with "  EOF" (spaces before EOF) should be content, not delimiter
+      const result = await env.exec(`cat <<EOF
+line 1
+  EOF
+line 2
+EOF`);
+      expect(result.stdout).toBe("line 1\n  EOF\nline 2\n");
+      expect(result.exitCode).toBe(0);
+    });
+
+    it("should handle delimiter with hyphen", async () => {
+      const env = new Bash();
+      const result = await env.exec(`cat <<END-TEST
+  content with spaces
+END-TEST`);
+      expect(result.stdout).toBe("  content with spaces\n");
+      expect(result.exitCode).toBe(0);
+    });
+  });
 });