curl (#2)

Author: cramforce

AGENTS.md

@@ -20,7 +20,7 @@ echo 'false; echo $?' | pnpm dev:exec
 
 - Install packages via pnpm rather than editing package.json directly
 - Bias towards making new test files that are roughly logically grouped rather than letting test files gets too large. Try to stay below 300 lines. Prefer making a new file when you want to add a `describe()`
-- Prefer asserting the full STDOUT/STDERR output rather than using to.contain or to.not.contain
+- Prefer asserting the full STDOUT/STDERR output rather than using toContain or not.toContain
 - Always also add `comparison-tests` for major command functionality, but edge cases should always be covered in unit tests which are mush faster (`pnpm test:comparison`)
 - When you are unsure about bash/command behavior, create a `comparison-tests` test file to ensure compat.
 - `--help` does not need to pass comparison tests and should reflect actual capability

PROJECT.md

@@ -217,3 +217,25 @@ xargs — build argument lists
 ## All before this is done
 
 Woohoo
+
+## Implementation phase 16: curl
+
+- make a new non-standard command called html-to-markdown which uses turndown service (npm package) to turns HTML on STDIN to markdown
+- Lets implement curl as a wrapper around `fetch`
+- Start with the most common options: method, headers, etc.
+- `curl` should not be available by default. It should require explicit opt-in via argument to BashEnv or Sandbox.create
+- The optin requires an allow-list of allowed origin + oath (optional) prefixes. Only those must be accessible via `fetch`
+  - Must also be checked on redirects, so implement redirects in user land rather that relying on following
+  - This allow-list must be enforced at the fetch layer, not subject to parsing
+  - Implement extensive unit tests for the allow-list matching specifically
+  - add `dangerouslyAllowFullInternetAccess` option to bypass allow-list
+
+## Implementation phase 16.1: curl part 2
+
+- Make the usage statement for html-to-markdown more docs-like since the caller will not be aware of this
+- Write more adversarial tests against the allow-list enforcement
+- Write tests that check the allow-list is enforced e2e (via bash execution)
+- Make sure the tests have a really good mock of the underlying fetch and never actually go to the network
+- allow `pnpm shell` to access the internet and document it
+
+## Implementation phase 17: AI SDK Tool

README.md

@@ -103,6 +103,12 @@ console.log(finished.exitCode); // 0
 pnpm shell
 ```
 
+The interactive shell has full internet access enabled by default, allowing you to use `curl` to fetch data from any URL. Use `--no-network` to disable this:
+
+```bash
+pnpm shell --no-network
+```
+
 ## Supported Commands
 
 ### File Operations
@@ -121,6 +127,10 @@ pnpm shell
 
 `alias`, `bash`, `chmod`, `clear`, `false`, `history`, `sh`, `true`, `unalias`
 
+### Network Commands
+
+`curl`, `html-to-markdown`
+
 All commands support `--help` for usage information.
 
 ## Shell Features
@@ -149,6 +159,74 @@ When created without options, BashEnv provides a Unix-like directory structure:
 
 Commands can be invoked by path (e.g., `/bin/ls`) or by name.
 
+## Network Access
+
+Network access (and the `curl` command) is disabled by default for security. To enable it, configure the `network` option:
+
+```typescript
+// Allow specific URLs with GET/HEAD only (safest)
+const env = new BashEnv({
+  network: {
+    allowedUrlPrefixes: [
+      "https://api.github.com/repos/myorg/",
+      "https://api.example.com",
+    ],
+  },
+});
+
+// Allow specific URLs with additional methods
+const env = new BashEnv({
+  network: {
+    allowedUrlPrefixes: ["https://api.example.com"],
+    allowedMethods: ["GET", "HEAD", "POST"], // Default: ["GET", "HEAD"]
+  },
+});
+
+// Allow all URLs and methods (use with extreme caution)
+const env = new BashEnv({
+  network: { dangerouslyAllowFullInternetAccess: true },
+});
+```
+
+**Note:** The `curl` command only exists when network is configured. Without network configuration, `curl` returns "command not found".
+
+### Allow-List Security
+
+The allow-list enforces:
+- **Origin matching**: URLs must match the exact origin (scheme + host + port)
+- **Path prefix**: Only paths starting with the specified prefix are allowed
+- **HTTP method restrictions**: Only GET and HEAD by default (configure `allowedMethods` for more)
+- **Redirect protection**: Redirects to non-allowed URLs are blocked
+
+```typescript
+// Only allow GitHub repos in myorg
+const env = new BashEnv({
+  network: { allowedUrlPrefixes: ["https://api.github.com/repos/myorg/"] },
+});
+
+// These work:
+// curl https://api.github.com/repos/myorg/repo1/issues
+// curl https://api.github.com/repos/myorg/repo2/pulls
+
+// These are blocked:
+// curl https://api.github.com/repos/otherorg/repo
+// curl https://api.github.com/users/user
+```
+
+### Using curl
+
+```bash
+# Fetch and process data
+curl -s https://api.example.com/data | grep pattern
+
+# Download and convert HTML to Markdown
+curl -s https://example.com | html-to-markdown
+
+# POST JSON data
+curl -X POST -H "Content-Type: application/json" \
+  -d '{"key":"value"}' https://api.example.com/endpoint
+```
+
 ## Execution Protection
 
 BashEnv includes protection against infinite loops and deep recursion:

package.json

@@ -37,12 +37,14 @@
     "@biomejs/biome": "^2.3.10",
     "@types/node": "^25.0.3",
     "@types/sprintf-js": "^1.1.4",
+    "@types/turndown": "^5.0.6",
     "typescript": "^5.9.3",
     "vitest": "^4.0.16"
   },
   "dependencies": {
     "diff": "^8.0.2",
     "minimatch": "^10.1.1",
-    "sprintf-js": "^1.1.3"
+    "sprintf-js": "^1.1.3",
+    "turndown": "^7.2.2"
   }
 }

pnpm-lock.yaml

@@ -9,14 +9,22 @@
  */
 
 import type { FunctionDefNode } from "./ast/types.js";
-import { createLazyCommands } from "./commands/registry.js";
+import {
+  createLazyCommands,
+  createNetworkCommands,
+} from "./commands/registry.js";
 import { type IFileSystem, VirtualFs } from "./fs.js";
 import type { InitialFiles } from "./fs-interface.js";
 import {
   Interpreter,
   type InterpreterOptions,
   type InterpreterState,
 } from "./interpreter/index.js";
+import {
+  createSecureFetch,
+  type NetworkConfig,
+  type SecureFetch,
+} from "./network/index.js";
 import { type ParseException, parse } from "./parser/parser.js";
 import type { Command, CommandRegistry, ExecResult } from "./types.js";
 
@@ -33,6 +41,11 @@ export interface BashEnvOptions {
   maxCallDepth?: number;
   maxCommandCount?: number;
   maxLoopIterations?: number;
+  /**
+   * Network configuration for commands like curl.
+   * Network access is disabled by default - you must explicitly configure allowed URLs.
+   */
+  network?: NetworkConfig;
 }
 
 export class BashEnv {
@@ -42,6 +55,7 @@ export class BashEnv {
   private maxCallDepth: number;
   private maxCommandCount: number;
   private maxLoopIterations: number;
+  private secureFetch?: SecureFetch;
 
   // Interpreter state (shared with interpreter instances)
   private state: InterpreterState;
@@ -64,6 +78,11 @@ export class BashEnv {
     this.maxLoopIterations =
       options.maxLoopIterations ?? DEFAULT_MAX_LOOP_ITERATIONS;
 
+    // Create secure fetch if network is configured
+    if (options.network) {
+      this.secureFetch = createSecureFetch(options.network);
+    }
+
     // Initialize interpreter state
     this.state = {
       env,
@@ -105,6 +124,13 @@ export class BashEnv {
     for (const cmd of createLazyCommands()) {
       this.registerCommand(cmd);
     }
+
+    // Register network commands only when network is configured
+    if (options.network) {
+      for (const cmd of createNetworkCommands()) {
+        this.registerCommand(cmd);
+      }
+    }
   }
 
   registerCommand(command: Command): void {
@@ -156,6 +182,7 @@ export class BashEnv {
         maxCommandCount: this.maxCommandCount,
         maxLoopIterations: this.maxLoopIterations,
         exec: this.exec.bind(this),
+        fetch: this.secureFetch,
       };
 
       const interpreter = new Interpreter(interpreterOptions, this.state);

src/BashEnv.ts

@@ -29,6 +29,7 @@ interface ShellOptions {
   cwd?: string;
   files?: Record<string, string>;
   env?: Record<string, string>;
+  network?: boolean;
 }
 
 class VirtualShell {
@@ -57,6 +58,11 @@ class VirtualShell {
         TERM: "xterm-256color",
         ...options.env,
       },
+      // Enable network access if requested (default: enabled for interactive shell)
+      network:
+        options.network !== false
+          ? { dangerouslyAllowFullInternetAccess: true }
+          : undefined,
     });
 
     // Check if stdin is a TTY (interactive mode)
@@ -172,6 +178,10 @@ ${colors.bold}Navigation & environment:${colors.reset}
 ${colors.bold}Utilities:${colors.reset}
   find, tee, basename, dirname, chmod, clear, history, alias, unalias
 
+${colors.bold}Network commands:${colors.reset}
+  curl              Fetch data from URLs (with full internet access)
+  html-to-markdown  Convert HTML to Markdown (use with curl)
+
 ${colors.bold}Supported features:${colors.reset}
   - Pipes: cmd1 | cmd2
   - Redirections: >, >>, 2>, 2>&1, <
@@ -189,6 +199,7 @@ ${colors.bold}Example commands:${colors.reset}
   find . -name "*.txt"
   ln -s target.txt link.txt
   awk '{print $1}' file.txt
+  curl -s https://example.com | html-to-markdown
 `);
   }
 
@@ -242,7 +253,7 @@ All operations run on a virtual in-memory filesystem.
 // CLI argument parsing
 function parseArgs(): ShellOptions {
   const args = process.argv.slice(2);
-  const options: ShellOptions = {};
+  const options: ShellOptions = { network: true }; // Network enabled by default
 
   for (let i = 0; i < args.length; i++) {
     if (args[i] === "--cwd" && args[i + 1]) {
@@ -256,17 +267,26 @@ function parseArgs(): ShellOptions {
         console.error(`Error reading files from ${filePath}:`, error);
         process.exit(1);
       }
+    } else if (args[i] === "--no-network") {
+      options.network = false;
     } else if (args[i] === "--help" || args[i] === "-h") {
       console.log(`
 Usage: npx tsx src/cli/shell.ts [options]
 
 Options:
   --cwd <dir>         Set initial working directory (default: /home/user)
   --files <json>      Load initial files from JSON file
+  --no-network        Disable network access (curl commands disabled)
   --help, -h          Show this help message
 
+Network Access:
+  By default, the interactive shell has full internet access enabled,
+  allowing curl commands to fetch data from any URL. Use --no-network
+  to disable this for sandboxed execution.
+
 Example:
   npx tsx src/cli/shell.ts --cwd /app --files ./my-files.json
+  pnpm shell --no-network
 `);
       process.exit(0);
     }