This PR introduces significant performance optimizations for filesystem operations and adds new features for file processing commands. Key improvements include parallel batch processing, shared utilities for multi-file operations, and new command options. (#44)

* This PR introduces significant performance optimizations for filesystem operations and adds new features for file processing commands. Key improvements include parallel batch processing, shared utilities for multi-file operations, and new command options.

* fix

* Self-feedback and test coverage

Author: cramforce

src/cli/exec.ts

@@ -9,8 +9,10 @@
  *   cat script.sh | pnpm dev:exec
  *
  * Options:
- *   --no-ast      Disable AST output
+ *   --print-ast   Show the parsed AST
  *   --real-bash   Also run the script with real bash for comparison
+ *   --root <path> Use OverlayFS with specified root directory
+ *   --no-limit    Remove execution limits (for large scripts)
  *
  * Output:
  *   - AST: The parsed Abstract Syntax Tree as JSON (unless --no-ast)
@@ -21,11 +23,21 @@
  */
 
 import { spawnSync } from "node:child_process";
+import { resolve } from "node:path";
 import { Bash } from "../Bash.js";
+import { OverlayFs } from "../fs/overlay-fs/index.js";
 import { parse } from "../parser/parser.js";
 
 const showAst = process.argv.includes("--print-ast");
 const runRealBash = process.argv.includes("--real-bash");
+const noLimit = process.argv.includes("--no-limit");
+
+// Parse --root option
+let rootPath: string | undefined;
+const rootIndex = process.argv.indexOf("--root");
+if (rootIndex !== -1 && rootIndex + 1 < process.argv.length) {
+  rootPath = resolve(process.argv[rootIndex + 1]);
+}
 
 async function readStdin(): Promise<string> {
   const chunks: Buffer[] = [];
@@ -66,7 +78,27 @@ if (showAst) {
   console.log("AST: Request with --print-ast");
 }
 
-const env = new Bash();
+// Create Bash environment with optional OverlayFS
+// Use high limits for dev:exec (typical use is exploration of large filesystems)
+const executionLimits = noLimit
+  ? {
+      maxCommandCount: Number.MAX_SAFE_INTEGER,
+      maxLoopIterations: Number.MAX_SAFE_INTEGER,
+    }
+  : {
+      maxCommandCount: 100000, // Higher default for dev:exec
+      maxLoopIterations: 100000,
+    };
+
+let env: Bash;
+if (rootPath) {
+  const fs = new OverlayFs({ root: rootPath });
+  const mountPoint = fs.getMountPoint();
+  env = new Bash({ fs, cwd: mountPoint, executionLimits });
+  console.log(`OverlayFS: ${rootPath} mounted at ${mountPoint}`);
+} else {
+  env = new Bash({ executionLimits });
+}
 const r = await env.exec(script);
 console.log("exitCode:", r.exitCode);
 console.log("stderr:", JSON.stringify(r.stderr));

src/commands/du/du.ts

@@ -1,5 +1,6 @@
 import type { Command, CommandContext, ExecResult } from "../../types.js";
 import { parseArgs } from "../../utils/args.js";
+import { DEFAULT_BATCH_SIZE } from "../../utils/constants.js";
 import { hasHelpFlag, showHelp } from "../help.js";
 
 const duHelp = {
@@ -120,49 +121,123 @@ async function calculateSize(
       return result;
     }
 
-    // Directory
-    const entries = await ctx.fs.readdir(fullPath);
+    // Directory - use readdirWithFileTypes if available for better performance
     let dirSize = 0;
 
-    for (const entry of entries) {
-      const entryPath = fullPath === "/" ? `/${entry}` : `${fullPath}/${entry}`;
-      const entryDisplayPath =
-        displayPath === "." ? entry : `${displayPath}/${entry}`;
+    // Get entries with type info if possible
+    interface EntryInfo {
+      name: string;
+      isDirectory: boolean;
+      size?: number;
+    }
+    const entryInfos: EntryInfo[] = [];
 
-      try {
-        const entryStat = await ctx.fs.stat(entryPath);
-
-        if (entryStat.isDirectory) {
-          const subResult = await calculateSize(
-            ctx,
-            entryPath,
-            entryDisplayPath,
-            options,
-            depth + 1,
-          );
-          dirSize += subResult.totalSize;
-
-          // Only output subdirectories if not summarizing and within depth limit
-          if (!options.summarize) {
-            if (options.maxDepth === null || depth + 1 <= options.maxDepth) {
-              result.output += subResult.output;
-            } else {
-              // Still need to count the size even if not displaying
-              dirSize += 0; // Size already counted
+    if (ctx.fs.readdirWithFileTypes) {
+      const entriesWithTypes = await ctx.fs.readdirWithFileTypes(fullPath);
+      // For files, we still need stat to get size, but we know directories
+      const fileEntries = entriesWithTypes.filter((e) => e.isFile);
+      const dirEntries = entriesWithTypes.filter((e) => e.isDirectory);
+
+      // Parallel stat for files to get sizes
+      for (let i = 0; i < fileEntries.length; i += DEFAULT_BATCH_SIZE) {
+        const batch = fileEntries.slice(i, i + DEFAULT_BATCH_SIZE);
+        const stats = await Promise.all(
+          batch.map(async (e) => {
+            const entryPath =
+              fullPath === "/" ? `/${e.name}` : `${fullPath}/${e.name}`;
+            try {
+              const s = await ctx.fs.stat(entryPath);
+              return { name: e.name, isDirectory: false, size: s.size };
+            } catch {
+              return { name: e.name, isDirectory: false, size: 0 };
             }
-          }
-        } else {
-          dirSize += entryStat.size;
-          if (options.allFiles && !options.summarize) {
-            result.output +=
-              formatSize(entryStat.size, options.humanReadable) +
-              "\t" +
-              entryDisplayPath +
-              "\n";
+          }),
+        );
+        entryInfos.push(...stats);
+      }
+
+      // Add directory entries (size will be calculated recursively)
+      entryInfos.push(
+        ...dirEntries.map((e) => ({ name: e.name, isDirectory: true })),
+      );
+    } else {
+      // Fall back to readdir + parallel stat
+      const entries = await ctx.fs.readdir(fullPath);
+      for (let i = 0; i < entries.length; i += DEFAULT_BATCH_SIZE) {
+        const batch = entries.slice(i, i + DEFAULT_BATCH_SIZE);
+        const stats = await Promise.all(
+          batch.map(async (entry) => {
+            const entryPath =
+              fullPath === "/" ? `/${entry}` : `${fullPath}/${entry}`;
+            try {
+              const s = await ctx.fs.stat(entryPath);
+              return {
+                name: entry,
+                isDirectory: s.isDirectory,
+                size: s.isDirectory ? undefined : s.size,
+              };
+            } catch {
+              return { name: entry, isDirectory: false, size: 0 };
+            }
+          }),
+        );
+        entryInfos.push(...stats);
+      }
+    }
+
+    // Sort entries for consistent output
+    entryInfos.sort((a, b) => a.name.localeCompare(b.name));
+
+    // Process files first (simple size addition)
+    const fileInfos = entryInfos.filter((e) => !e.isDirectory);
+    for (const file of fileInfos) {
+      const size = file.size ?? 0;
+      dirSize += size;
+      if (options.allFiles && !options.summarize) {
+        const entryDisplayPath =
+          displayPath === "." ? file.name : `${displayPath}/${file.name}`;
+        result.output +=
+          formatSize(size, options.humanReadable) +
+          "\t" +
+          entryDisplayPath +
+          "\n";
+      }
+    }
+
+    // Process directories in parallel batches
+    const dirInfos = entryInfos.filter((e) => e.isDirectory);
+    for (let i = 0; i < dirInfos.length; i += DEFAULT_BATCH_SIZE) {
+      const batch = dirInfos.slice(i, i + DEFAULT_BATCH_SIZE);
+      const subResults = await Promise.all(
+        batch.map(async (dir) => {
+          const entryPath =
+            fullPath === "/" ? `/${dir.name}` : `${fullPath}/${dir.name}`;
+          const entryDisplayPath =
+            displayPath === "." ? dir.name : `${displayPath}/${dir.name}`;
+          return {
+            name: dir.name,
+            result: await calculateSize(
+              ctx,
+              entryPath,
+              entryDisplayPath,
+              options,
+              depth + 1,
+            ),
+          };
+        }),
+      );
+
+      // Sort results for consistent order
+      subResults.sort((a, b) => a.name.localeCompare(b.name));
+
+      for (const { result: subResult } of subResults) {
+        dirSize += subResult.totalSize;
+        // Only output subdirectories if not summarizing and within depth limit
+        if (!options.summarize) {
+          if (options.maxDepth === null || depth + 1 <= options.maxDepth) {
+            result.output += subResult.output;
           }
         }
-      } catch {
-        // Skip entries we can't read
       }
     }