runprompt

#!/usr/bin/env python3
"""
runprompt: A single-file runner for .prompt (Dotprompt) files.

This utility parses .prompt files containing YAML frontmatter and Handlebars
templates, executes pre-processing shell commands, attaches file context,
and manages LLM interactions with support for tool calling and caching.

Key Functions:
- main(): Entry point handling CLI arguments and the execution loop.
- render_template(): A subset of Handlebars for dynamic prompt generation.
- load_tools(): Dynamically imports Python and builtin functions as LLM tools.
- make_request(): Handles API communication across multiple providers.
- get_conf(): Implements the configuration priority cascade.
"""
import sys
import os
import json
import re
import hashlib
import time
import urllib.request
import urllib.error
import importlib.util
import inspect
import argparse
import glob
import base64
import mimetypes
import socket

# Try to use PyYAML if available, otherwise use minimal parser
try:
    import yaml as _yaml
    _HAS_PYYAML = True
except ImportError:
    _HAS_PYYAML = False

# Global config state (loaded once at startup)
CONFIG = {
    "files": {},      # Merged config from all config files
    "env": {},        # Parsed from RUNPROMPT_* env vars
    "args": {},       # From command line arguments
}

# Config keys that support the cascade
CONFIG_KEYS = {
    "model", "default_model", "tool_path", "base_url", "ollama_base_url", "cache", "cache_dir",
    "safe_yes", "verbose", "anthropic_api_key", "openai_api_key",
    "google_api_key", "openrouter_api_key", "chat",
}

PROVIDERS = {
    "openrouter": {
        "url": "https://openrouter.ai/api/v1/chat/completions",
        "env": "OPENROUTER_API_KEY",
    },
    "googleai": {
        "url": "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions",
        "env": "GOOGLE_API_KEY",
    },
    "anthropic": {
        "url": "https://api.anthropic.com/v1/messages",
        "env": "ANTHROPIC_API_KEY",
    },
    "openai": {
        "url": "https://api.openai.com/v1/chat/completions",
        "env": "OPENAI_API_KEY",
    },
}

RED = "\033[31m"
YELLOW = "\033[33m"
GREEN = "\033[32m"
CYAN = "\033[36m"
RESET = "\033[0m"
TIMEOUT = 120

# Session cost accumulator (None means unknown)
SESSION_COST = 0.0
SESSION_COST_KNOWN = True

# === CONFIGURATION CASCADE ===


def load_config_files():
    """Load config from cascade of config file locations (lowest to highest priority)."""
    config = {}
    locations = [
        os.path.join(os.path.expanduser("~"), ".runprompt", "config.yml"),
        os.path.join(
            os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")),
            "runprompt", "config.yml"
        ),
        os.path.join(".", ".runprompt", "config.yml"),
    ]
    for path in locations:
        if os.path.exists(path):
            try:
                with open(path, "r") as f:
                    file_config = parse_yaml(f.read())
                    if file_config:
                        for key, value in file_config.items():
                            config[normalize_key(key)] = value
            except Exception as e:
                print("%sWarning: Could not load config from %s: %s%s" %
                      (YELLOW, path, e, RESET), file=sys.stderr)
    return config


def load_config_env():
    """Load config from RUNPROMPT_* environment variables."""
    config = {}
    for env_key, env_value in os.environ.items():
        if env_key.startswith("RUNPROMPT_"):
            key = normalize_key(env_key[10:])
            if key in ("cache_dir",):
                config[key] = env_value
            else:
                config[key] = parse_yaml_value(env_value)
    return config


def normalize_key(key):
    """Normalize config key: lowercase, underscores instead of hyphens."""
    return key.lower().replace("-", "_")


def get_conf(key, default=None):
    """Get config value with cascade priority: files < env < args."""
    key = normalize_key(key)
    # Check args first (highest priority)
    if key in CONFIG["args"]:
        return CONFIG["args"][key]
    # Check env vars
    if key in CONFIG["env"]:
        return CONFIG["env"][key]
    # Check config files
    if key in CONFIG["files"]:
        return CONFIG["files"][key]
    return default


def get_api_key(provider):
    """Get API key for provider, checking config cascade then native env var."""
    key_name = provider + "_api_key"
    # First check config cascade
    from_conf = get_conf(key_name)
    if from_conf:
        return from_conf
    # Fall back to native env var (e.g. ANTHROPIC_API_KEY)
    env_var = PROVIDERS.get(provider, {}).get("env")
    if env_var:
        return os.environ.get(env_var)
    return None


def init_config(args):
    """Initialize global config from all sources."""
    CONFIG["files"] = load_config_files()
    CONFIG["env"] = load_config_env()
    # Convert args to dict
    args_dict = {}
    if args.verbose:
        args_dict["verbose"] = True
    if args.cache:
        args_dict["cache"] = True
    if args.safe_yes:
        args_dict["safe_yes"] = True
    if args.base_url:
        args_dict["base_url"] = args.base_url
    if args.ollama_base_url:
        args_dict["ollama_base_url"] = args.ollama_base_url
    if args.tool_path:
        args_dict["tool_path"] = args.tool_path
    if args.chat:
        args_dict["chat"] = True
    # Include overrides from --key=value
    for key, value in args.overrides.items():
        args_dict[normalize_key(key)] = value
    CONFIG["args"] = args_dict


# === MAIN ENTRY POINT ===


def main():
    args = parse_args(sys.argv[1:])
    init_config(args)

    if args.clear_cache:
        clear_cache()
        sys.exit(0)

    use_cache = get_conf("cache", False)

    if len(args.remaining) < 1 and not get_conf("chat"):
        print("Usage: runprompt [options] <prompt_file>", file=sys.stderr)
        print("Try 'runprompt --help' for more information.", file=sys.stderr)
        sys.exit(1)
    prompt_path = args.remaining[0] if args.remaining else None
    if prompt_path:
        meta, template = parse_prompt_file(prompt_path)
    else:
        meta, template = {}, ""
    meta = apply_overrides(meta)
    # Apply model from config cascade if not in frontmatter
    if not meta.get("model"):
        if get_conf("model"):
            meta["model"] = get_conf("model")
        elif get_conf("default_model"):
            meta["model"] = get_conf("default_model")
    for key, value in args.overrides.items():
        log("Override from arg --%s: %s" % (key, value))
        if key == "tools" and isinstance(value, str):
            # Parse comma-separated tools
            meta[key] = [t.strip() for t in value.split(",")]
        else:
            meta[key] = value
    model_str = meta.get("model", "")
    if not model_str:
        print("No model specified in prompt file", file=sys.stderr)
        sys.exit(1)
    provider, model = parse_model_string(model_str)
    if not provider:
        print("No provider in model string", file=sys.stderr)
        sys.exit(1)
    # Collect extra args after prompt file
    extra_args = args.remaining[1:] if len(args.remaining) > 1 else []
    args_str = " ".join(extra_args)
    log("Extra args: %s" % args_str)

    raw_stdin = read_stdin()
    variables = {"STDIN": raw_stdin or "", "ARGS": args_str}
    # Determine INPUT: prefer STDIN if provided, otherwise use ARGS
    if raw_stdin:
        variables["INPUT"] = raw_stdin
    else:
        variables["INPUT"] = args_str
    # Parse STDIN as JSON if provided
    if raw_stdin:
        try:
            parsed = json.loads(raw_stdin)
            variables.update(parsed)
            log("Parsed STDIN as JSON")
        except ValueError:
            log("STDIN is not JSON, treating as raw string")
            input_schema = meta.get("input", {}).get("schema", {})
            if input_schema:
                first_key = list(input_schema.keys())[0]
                variables[first_key] = raw_stdin
            else:
                variables["input"] = raw_stdin
    # Parse ARGS as JSON if provided (and no STDIN to avoid conflicts)
    elif args_str:
        try:
            parsed = json.loads(args_str)
            variables.update(parsed)
            log("Parsed ARGS as JSON")
        except ValueError:
            log("ARGS is not JSON, treating as raw string")
            input_schema = meta.get("input", {}).get("schema", {})
            if input_schema:
                first_key = list(input_schema.keys())[0]
                variables[first_key] = args_str
    
    # Execute before: shell commands if present
    before_commands = meta.get("before", {})
    if before_commands:
        execute_before_commands(before_commands, variables)
    
    validate_required_inputs(meta, variables)
    prompt = render_template(template, variables)
    log("Rendered prompt: %s" % prompt)
    output_config = meta.get("output", {})

    # Build tool search paths
    search_paths = []
    search_paths.append(os.getcwd())
    if prompt_path:
        prompt_dir = os.path.dirname(os.path.abspath(prompt_path))
        if prompt_dir not in search_paths:
            search_paths.append(prompt_dir)
        # If prompt is a symlink, also add the real file's directory
        real_prompt_path = os.path.realpath(prompt_path)
        if real_prompt_path != os.path.abspath(prompt_path):
            real_prompt_dir = os.path.dirname(real_prompt_path)
            if real_prompt_dir not in search_paths:
                search_paths.append(real_prompt_dir)
    # Add tool paths from config cascade
    conf_tool_paths = get_conf("tool_path", [])
    if isinstance(conf_tool_paths, str):
        conf_tool_paths = [conf_tool_paths]
    for tp in conf_tool_paths:
        abs_tp = os.path.abspath(tp)
        if abs_tp not in search_paths:
            search_paths.append(abs_tp)
    # Add default tool paths from config directories
    default_tool_dirs = [
        os.path.join(".", ".runprompt", "tools"),
        os.path.join(
            os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")),
            "runprompt", "tools"
        ),
        os.path.join(os.path.expanduser("~"), ".runprompt", "tools"),
    ]
    for td in default_tool_dirs:
        abs_td = os.path.abspath(td)
        if abs_td not in search_paths and os.path.isdir(abs_td):
            search_paths.append(abs_td)
    log("Tool search paths: %s" % search_paths)

    # Load tools
    tool_specs = meta.get("tools", [])
    if isinstance(tool_specs, str):
        tool_specs = [t.strip() for t in tool_specs.split(",")]
    tools = {}
    if tool_specs:
        tools = load_tools(tool_specs, search_paths)
        log("Loaded %d tools: %s" % (len(tools), list(tools.keys())))
    
    # Load shell_tools
    shell_tool_specs = meta.get("shell_tools", {})
    if shell_tool_specs:
        shell_tools = load_shell_tools(shell_tool_specs)
        tools.update(shell_tools)
        log("Loaded %d shell tools: %s" % (len(shell_tools), list(shell_tools.keys())))

    # Determine effective provider early for file reading
    base_url = (get_conf("base_url") or get_conf("ollama_base_url") or
                meta.get("base_url") or meta.get("ollama_base_url") or
                get_base_url())
    if base_url:
        effective_provider = "openai"
    else:
        effective_provider = provider

    read_patterns = []
    if meta.get("files") is not None:
        if not isinstance(meta.get("files"), list):
            print("%sError: 'files' frontmatter must be a list%s" %
                  (RED, RESET), file=sys.stderr)
            sys.exit(1)
        # Render template variables in files: patterns
        raw_patterns = meta.get("files") or []
        for pattern in raw_patterns:
            rendered_pattern = render_template(pattern, variables)
            read_patterns.append(rendered_pattern)

    if args.read:
        read_patterns.extend(args.read)

    # Read files from --read flags and prompt frontmatter
    read_files = []
    if read_patterns:
        read_files = read_files_for_context(read_patterns, effective_provider)
        log("Read %d file(s) for context" % len(read_files))

    is_chat = get_conf("chat") or meta.get("chat", False)

    if not model_str and is_chat:
        print("No model specified. Use --model or set one in config.",
              file=sys.stderr)
        sys.exit(1)
    if provider == "test":
        response = load_test_response(prompt_path)
        test_provider = response.get("_provider", "openai")
        result = extract_response(response, output_config, test_provider)
        print(result)
        return

    if base_url:
        url, api_key = get_provider_config(provider, base_url)
    else:
        url, api_key = get_provider_config(provider)

    # Check cache
    cached_response = None
    key = None
    if use_cache:
        key = cache_key(prompt, meta)
        log("Cache key: %s" % key)
        cached_response = cache_get(key)

    if cached_response:
        response = cached_response
        cached_provider = response.get("_provider", effective_provider)
        result = extract_response(response, output_config, cached_provider)
        print(result)
        return

    # Build initial messages with optional file attachments
    messages = []
    if prompt:
        user_content = build_content_with_files(prompt, read_files,
                                                effective_provider)
        messages = [{"role": "user", "content": user_content}]

    # In chat mode with no initial prompt, ask for first input now
    if is_chat and not messages:
        first_input = chat_read_input()
        if first_input is None:
            return
        messages = [{"role": "user", "content": first_input}]

    # Tool execution / Chat loop
    while True:
        response = make_request(url, api_key, model, messages, output_config,
                                effective_provider, tools if tools else None)

        # Print usage info
        print_usage(response)

        # Print any text content immediately
        text_content = extract_text_content(response, effective_provider)
        if text_content:
            print(text_content)

        # Check for tool calls
        tool_calls = extract_tool_calls(response, effective_provider)

        # Filter out 'extract' tool calls - those are for structured output
        user_tool_calls = [tc for tc in tool_calls if tc["name"] != "extract"]

        if not user_tool_calls:
            # No more tool calls
            if use_cache and key:
                cache_set(key, response, effective_provider)
            if args.save_response:
                save_response(response, effective_provider, args.save_response)
            
            # If there was an extract tool call, output that and exit
            for tc in tool_calls:
                if tc["name"] == "extract":
                    print(json.dumps(tc["arguments"], indent=2))
                    return
            
            # Otherwise we already printed text content above
            if not text_content:
                result = extract_response(response, output_config,
                                          effective_provider)
                if result:
                    print(result)
            
            if not is_chat:
                return
                
            # In chat mode, prompt for next input
            user_input = chat_read_input()
            if user_input is None:
                return
            messages.append(build_assistant_message(response, effective_provider))
            messages.append({"role": "user", "content": user_input})
            continue

        # Add assistant message to conversation
        messages.append(build_assistant_message(response, effective_provider))

        # Process each tool call
        for tc in user_tool_calls:
            tool_name = tc["name"]
            tool_args = tc["arguments"]

            if tool_name not in tools:
                error_msg = "Unknown tool: %s" % tool_name
                print("%s%s%s" % (RED, error_msg, RESET), file=sys.stderr)
                messages.append(build_tool_result_message(
                    tc, None, error_msg, effective_provider))
                continue

            tool_func = tools[tool_name]["func"]

            # Always print tool call summary
            print_tool_call(tool_name, tool_args)

            # Check if safe-yes is enabled and tool is safe
            if get_conf("safe_yes") and is_tool_safe(tool_func):
                log("Auto-approving safe tool: %s" % tool_name)
                approved = True
            else:
                # Prompt user for confirmation
                approved = prompt_user_for_tool(tool_name, tool_args)

            if not approved:
                error_msg = "Tool execution declined by user"
                messages.append(build_tool_result_message(
                    tc, None, error_msg, effective_provider))
                continue

            # Execute the tool
            result, error = execute_tool(tool_func, tool_args)

            if error:
                print("%s%s%s" % (RED, error, RESET), file=sys.stderr)

            messages.append(build_tool_result_message(
                tc, result, error, effective_provider))


HELP_EPILOG = """\
Input:
  Pipe JSON to set template variables: echo '{"name": "World"}' | runprompt hello.prompt
  Pipe text for {{STDIN}} variable:    echo "some text" | runprompt summarize.prompt
  Pass args for {{ARGS}} variable:     runprompt hello.prompt Some text here

Config files (lowest to highest priority):
  ~/.runprompt/config.yml
  $XDG_CONFIG_HOME/runprompt/config.yml (default: ~/.config/runprompt/config.yml)
  ./.runprompt/config.yml

Config file example:
  model: openai/gpt-4o
  tool_path:
    - ./tools
    - /shared/tools
  cache: true
  safe_yes: true
  openai_api_key: sk-...

Environment:
  ANTHROPIC_API_KEY       API key for Anthropic models
  OPENAI_API_KEY          API key for OpenAI models
  GOOGLE_API_KEY          API key for Google AI models
  OPENROUTER_API_KEY      API key for OpenRouter models
  OLLAMA_BASE_URL         Custom Ollama endpoint URL (e.g. http://localhost:11434/v1)
  OPENAI_BASE_URL         Custom OpenAI-compatible endpoint URL
  OPENAI_API_BASE         Custom endpoint URL (legacy OpenAI SDK v0.x style)
  BASE_URL                Custom endpoint URL (fallback)
  RUNPROMPT_<KEY>         Override config (e.g. RUNPROMPT_MODEL=openai/gpt-4o)

Config priority (highest wins): CLI flags > env vars > ./.runprompt > ~/.config > ~/.runprompt

Tools:
  Mark a function as safe by setting fn.safe = True after the definition:
  
    def my_safe_tool(x: str):
        \"\"\"A safe tool that only reads data.\"\"\"
        return "result"
    my_safe_tool.safe = True
  
  Safe tools are auto-approved when --safe-yes is passed.

Examples:
  runprompt hello.prompt
  runprompt hello.prompt Some text to process
  echo '{"name": "World"}' | runprompt hello.prompt
  runprompt --model openai/gpt-4o hello.prompt
  runprompt -v --save-response out.json hello.prompt
  runprompt --cache hello.prompt
  runprompt --safe-yes tool_prompt.prompt
  runprompt --read "src/*.py" review.prompt
  runprompt --read "*.png" describe.prompt
  runprompt --read README.md --read "src/**/*.py" analyze.prompt
  OPENAI_BASE_URL=http://localhost:11434/v1 runprompt hello.prompt

Overrides:
  --<key>=<value>         Override frontmatter value (e.g. --model=openai/gpt-4o)
  --<key> <value>         Override frontmatter value (e.g. --model openai/gpt-4o)
"""

def parse_args(args):
    parser = argparse.ArgumentParser(
        description="Run Dotprompt (.prompt) files from the command line.",
        add_help=True, formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=HELP_EPILOG)
    parser.add_argument("-v", "--verbose", action="store_true",
                        help="Show request/response details")
    parser.add_argument("-c", "--cache", action="store_true",
                        help="Enable response caching")
    parser.add_argument("--clear-cache", action="store_true",
                        help="Clear the response cache and exit")
    parser.add_argument("--safe-yes", action="store_true",
                        help="Auto-approve tool calls for safe functions")
    parser.add_argument("--save-response", metavar="FILE",
                        help="Save raw API response to file")
    parser.add_argument("--base-url", "--openai-base-url", metavar="URL",
                        help="Use custom OpenAI-compatible endpoint")
    parser.add_argument("--ollama-base-url", metavar="URL",
                        help="Use custom Ollama endpoint")
    parser.add_argument("--tool-path", action="append", default=[],
                        metavar="PATH", help="Add directory to tool import path")
    parser.add_argument("--chat", action="store_true",
                        help="Interactive chat mode: prompt for input after each response")
    parser.add_argument("--read", "--file", action="append", default=[],
                        metavar="FILE",
                        help="Read file(s) into context (supports globs)")
    parsed, unknown = parser.parse_known_args(args)
    # Parse unknown args for --key=value overrides and extra positional args
    parsed.overrides = {}
    parsed.remaining = []
    i = 0
    while i < len(unknown):
        arg = unknown[i]
        if arg.startswith("--"):
            if "=" in arg:
                key, val = arg[2:].split("=", 1)
                parsed.overrides[normalize_key(key)] = parse_yaml_value(val)
            elif i + 1 < len(unknown) and not unknown[i + 1].startswith("-"):
                parsed.overrides[normalize_key(arg[2:])] = parse_yaml_value(
                    unknown[i + 1])
                i += 1
            else:
                parsed.overrides[normalize_key(arg[2:])] = True
        else:
            parsed.remaining.append(arg)
        i += 1
    return parsed


def read_stdin():
    if sys.stdin.isatty():
        return None
    content = sys.stdin.read()
    if not content:
        return None
    return content.strip() or None


# === PROMPT FILE HANDLING ===


def parse_prompt_file(path):
    with open(path, "r") as f:
        content = f.read()
    # Skip shebang line if present
    if content.startswith("#!"):
        content = content.split("\n", 1)[1] if "\n" in content else ""
    # Handle ---frontmatter---template format
    if content.startswith("---"):
        parts = content.split("---", 2)
        if len(parts) >= 3:
            meta_str = parts[1].strip()
            template = parts[2].strip()
            meta = parse_yaml(meta_str)
            return meta, template
        return {}, content.strip()
    # Handle frontmatter---template format (no opening ---)
    if "---" in content:
        parts = content.split("---", 1)
        meta_str = parts[0].strip()
        template = parts[1].strip()
        meta = parse_yaml(meta_str)
        return meta, template
    # No frontmatter delimiter found
    return {}, content.strip()


def _parse_yaml_minimal(s):
    """Minimal YAML parser for dotprompt frontmatter."""
    result = {}
    stack = [(result, -1)]
    current_list = None
    current_list_indent = -1
    current_multiline = None
    current_multiline_indent = -1
    current_multiline_base_indent = -1
    lines = s.split("\n")
    i = 0
    while i < len(lines):
        line = lines[i]
        if not line.strip() or line.strip().startswith("#"):
            i += 1
            continue
        indent = len(line) - len(line.lstrip())
        
        # Handle multiline string continuation
        if current_multiline is not None:
            if indent > current_multiline_indent or not line.strip():
                # First content line - establish base indent
                if current_multiline_base_indent == -1 and line.strip():
                    current_multiline_base_indent = indent
                # Continuation of multiline string
                if line.strip():
                    current_multiline["lines"].append(line[current_multiline_base_indent:])
                else:
                    current_multiline["lines"].append("")
                i += 1
                continue
            else:
                # End of multiline string
                parent = current_multiline["parent"]
                key = current_multiline["key"]
                parent[key] = "\n".join(current_multiline["lines"])
                current_multiline = None
                current_multiline_indent = -1
                current_multiline_base_indent = -1
                # Don't increment i, process this line normally
        
        # Check if this is a list item
        list_match = re.match(r"^(\s*)-\s*(.*)", line)
        if list_match:
            item_value = list_match.group(2).strip()
            if current_list is not None and indent >= current_list_indent:
                current_list.append(parse_yaml_value(item_value) if item_value else item_value)
                i += 1
                continue
        # Not a list item - reset list tracking if we've dedented
        if current_list is not None and indent <= current_list_indent:
            current_list = None
            current_list_indent = -1
        # Pop stack for dedented lines
        while stack and indent <= stack[-1][1]:
            stack.pop()
        if not stack:
            stack = [(result, -1)]
        match = re.match(r"^(\s*)([^:]+):\s*(.*)", line)
        if not match:
            i += 1
            continue
        key = match.group(2).strip()
        value = match.group(3).strip()
        parent = stack[-1][0]
        
        # Check for multiline string indicator (| or >)
        if value in ("|", ">"):
            current_multiline = {
                "parent": parent,
                "key": key,
                "lines": [],
                "style": value
            }
            current_multiline_indent = indent
            current_multiline_base_indent = -1
            i += 1
            continue
        
        if value:
            parent[key] = parse_yaml_value(value)
            # Reset list tracking when we have a value
            current_list = None
            current_list_indent = -1
        else:
            # Check if next non-empty line is a list item
            is_list_parent = False
            if i + 1 < len(lines):
                for next_line in lines[i + 1:]:
                    if not next_line.strip() or next_line.strip().startswith("#"):
                        continue
                    next_indent = len(next_line) - len(next_line.lstrip())
                    if next_indent <= indent:
                        break
                    if re.match(r"^\s*-\s*", next_line):
                        is_list_parent = True
                    break
            if is_list_parent:
                parent[key] = []
                current_list = parent[key]
                current_list_indent = indent
            else:
                parent[key] = {}
                stack.append((parent[key], indent))
                current_list = None
                current_list_indent = -1
        i += 1
    
    # Handle any remaining multiline string
    if current_multiline is not None:
        parent = current_multiline["parent"]
        key = current_multiline["key"]
        parent[key] = "\n".join(current_multiline["lines"])
        current_multiline_base_indent = -1
    
    return result


def parse_yaml(s):
    """Parse YAML string. Uses PyYAML if available, otherwise minimal parser."""
    if _HAS_PYYAML:
        return _yaml.safe_load(s) or {}
    return _parse_yaml_minimal(s)


def parse_yaml_value(s):
    s = s.strip()
    if not s:
        return None
    if s.lower() == "true":
        return True
    if s.lower() == "false":
        return False
    if re.match(r"^-?\d+$", s):
        return int(s)
    if re.match(r"^-?\d+\.\d+$", s):
        return float(s)
    if "\n" in s or s.startswith("{"):
        try:
            return json.loads(s)
        except ValueError:
            pass
        parsed = parse_yaml(s)
        if parsed:
            return parsed
    # Strip surrounding quotes (single or double) to match PyYAML behavior
    if len(s) >= 2 and s[0] in ('"', "'") and s[-1] == s[0]:
        return s[1:-1]
    return s


def get_required_input_fields(meta):
    """Extract required field names from input schema (fields without ? suffix)."""
    input_schema = meta.get("input", {}).get("schema", {})
    required = []
    for key in input_schema:
        if not key.endswith("?"):
            required.append(key)
    return required


def validate_required_inputs(meta, variables):
    """Check that all required input fields are present. Exit with error if not."""
    log("DEBUG validate_required_inputs meta: %s" % meta)
    log("DEBUG validate_required_inputs variables: %s" % variables)
    required = get_required_input_fields(meta)
    log("DEBUG required fields: %s" % required)
    missing = []
    for field in required:
        if field not in variables or variables[field] == "":
            missing.append(field)
    if missing:
        print("%sError: Missing required input field(s): %s%s" %
              (RED, ", ".join(missing), RESET), file=sys.stderr)
        input_schema = meta.get("input", {}).get("schema", {})
        print("Expected input schema:", file=sys.stderr)
        for key, value in input_schema.items():
            opt = " (optional)" if key.endswith("?") else " (required)"
            clean_key = key.rstrip("?")
            print("  %s: %s%s" % (clean_key, value, opt), file=sys.stderr)
        sys.exit(1)


def apply_overrides(meta):
    """Apply RUNPROMPT_* env vars to prompt metadata (for prompt-specific overrides)."""
    for key in CONFIG["env"]:
        # Skip config-level keys, only apply prompt-specific overrides
        if key in CONFIG_KEYS:
            continue
        value = CONFIG["env"][key]
        if value is not None:
            log("Override from env RUNPROMPT_%s: %s" % (key.upper(), value))
            meta[key] = value
    return meta


# === TEMPLATE RENDERING ===


def render_template(template, variables):
    def lookup(name, ctx):
        name = name.strip()
        if name == ".":
            return ctx
        # Handle @index, @first, @last, @key
        if name.startswith("@"):
            return ctx.get(name, "")
        for part in name.split("."):
            if isinstance(ctx, dict):
                ctx = ctx.get(part, "")
            else:
                return ""
        return ctx

    def render(tmpl, ctx):
        # Remove comments: {{! ... }}
        tmpl = re.sub(r"\{\{!.*?\}\}", "", tmpl, flags=re.DOTALL)

        # Process {{#each key}}...{{/each}}
        def each_replace(match):
            key = match.group(1)
            inner = match.group(2)
            val = lookup(key, ctx)
            result = []
            if isinstance(val, list):
                for i, item in enumerate(val):
                    if isinstance(item, dict):
                        item_ctx = dict(item)
                    else:
                        item_ctx = {}
                    item_ctx["@index"] = i
                    item_ctx["@first"] = (i == 0)
                    item_ctx["@last"] = (i == len(val) - 1)
                    item_ctx["."] = item
                    result.append(render(inner, item_ctx))
            elif isinstance(val, dict):
                keys = list(val.keys())
                for i, k in enumerate(keys):
                    item = val[k]
                    if isinstance(item, dict):
                        item_ctx = dict(item)
                    else:
                        item_ctx = {}
                    item_ctx["@key"] = k
                    item_ctx["@index"] = i
                    item_ctx["@first"] = (i == 0)
                    item_ctx["@last"] = (i == len(keys) - 1)
                    item_ctx["."] = item
                    result.append(render(inner, item_ctx))
            return "".join(result)

        tmpl = re.sub(
            r"\{\{#each\s+(\w+)\}\}(.*?)\{\{/each\}\}",
            each_replace,
            tmpl,
            flags=re.DOTALL
        )

        # Process sections: {{#key}}...{{/key}}
        def section_replace(match):
            key = match.group(1)
            inner = match.group(2)
            val = lookup(key, ctx)
            if isinstance(val, list):
                result = []
                for i, item in enumerate(val):
                    if isinstance(item, dict):
                        item_ctx = dict(item)
                    else:
                        item_ctx = {"_value": item}
                    item_ctx["@index"] = i
                    item_ctx["@first"] = (i == 0)
                    item_ctx["@last"] = (i == len(val) - 1)
                    item_ctx["."] = item
                    result.append(render(inner, item_ctx))
                return "".join(result)
            if val:
                new_ctx = val if isinstance(val, dict) else ctx
                return render(inner, new_ctx)
            return ""

        # Process inverted sections: {{^key}}...{{/key}}
        def inverted_replace(match):
            key = match.group(1)
            inner = match.group(2)
            val = lookup(key, ctx)
            if not val or (isinstance(val, list) and len(val) == 0):
                return render(inner, ctx)
            return ""

        # Process {{#if}} and {{#unless}} with shared logic
        def conditional_replace(match, invert=False):
            key = match.group(1)
            inner = match.group(2)
            val = lookup(key, ctx)
            is_truthy = bool(val) and val != "" and \
                not (isinstance(val, list) and len(val) == 0)
            if invert:
                is_truthy = not is_truthy
            # Find {{else}} not inside nested conditionals
            depth = 0
            else_pos = None
            i = 0
            tag = "{{#unless" if invert else "{{#if"
            end_tag = "{{/unless}}" if invert else "{{/if}}"
            while i < len(inner):
                if inner[i:].startswith(tag):
                    depth += 1
                elif inner[i:].startswith(end_tag):
                    depth -= 1
                elif inner[i:].startswith("{{else}}") and depth == 0:
                    else_pos = i
                    break
                i += 1
            if else_pos is not None:
                return inner[:else_pos] if is_truthy else inner[else_pos + 8:]
            return inner if is_truthy else ""

        # Process if/unless in a single loop to handle mixed nesting
        if_pattern = re.compile(
            r"\{\{#if\s+([\w.]+)\}\}((?:(?!\{\{#if)(?!\{\{/if\}\})(?!\{\{#unless)(?!\{\{/unless\}\}).)*?)\{\{/if\}\}",
            re.DOTALL
        )
        unless_pattern = re.compile(
            r"\{\{#unless\s+([\w.]+)\}\}((?:(?!\{\{#if)(?!\{\{/if\}\})(?!\{\{#unless)(?!\{\{/unless\}\}).)*?)\{\{/unless\}\}",
            re.DOTALL
        )
        while True:
            if_match = if_pattern.search(tmpl)
            unless_match = unless_pattern.search(tmpl)
            if not if_match and not unless_match:
                break
            if if_match and (not unless_match or if_match.start() < unless_match.start()):
                tmpl = if_pattern.sub(lambda m: conditional_replace(m, False), tmpl, count=1)
            else:
                tmpl = unless_pattern.sub(lambda m: conditional_replace(m, True), tmpl, count=1)

        # Process sections first (innermost first via non-greedy)
        tmpl = re.sub(
            r"\{\{#(@?\w+)\}\}(.*?)\{\{/\1\}\}",
            section_replace,
            tmpl,
            flags=re.DOTALL
        )
        tmpl = re.sub(
            r"\{\{\^(@?\w+)\}\}(.*?)\{\{/\1\}\}",
            inverted_replace,
            tmpl,
            flags=re.DOTALL
        )

        # Process variables
        def var_replace(match):
            key = match.group(1).strip()
            val = lookup(key, ctx)
            # Handle special "." lookup for non-dict items in lists
            if key == "." and "." in ctx:
                return str(ctx["."])
            return str(val)
        tmpl = re.sub(r"\{\{([^#^/}]+)\}\}", var_replace, tmpl)

        return tmpl

    return render(template, variables)


# === MODEL/PROVIDER CONFIGURATION ===


def parse_model_string(model_str):
    if model_str == "test":
        return "test", None
    parts = model_str.split("/", 1)
    if len(parts) == 1:
        return None, parts[0]
    return parts[0], parts[1]


def get_base_url():
    """Get base URL from legacy env vars (fallback after config cascade)."""
    return (os.environ.get("OLLAMA_BASE_URL") or
            os.environ.get("OPENAI_BASE_URL") or
            os.environ.get("OPENAI_API_BASE") or
            os.environ.get("BASE_URL"))


def print_provider_help():
    """Print available providers and their API key URLs."""
    api_key_urls = {
        "anthropic": "https://console.anthropic.com/settings/keys",
        "openai": "https://platform.openai.com/api-keys",
        "googleai": "https://aistudio.google.com/app/apikey",
        "openrouter": "https://openrouter.ai/settings/keys",
    }
    print("\nAvailable providers:", file=sys.stderr)
    for name in sorted(PROVIDERS.keys()):
        env_var = PROVIDERS[name]["env"]
        url = api_key_urls.get(name, "")
        print("  %s (set %s)" % (name, env_var), file=sys.stderr)
        if url:
            print("    Get key: %s" % url, file=sys.stderr)


def get_provider_config(provider, base_url=None):
    if base_url:
        url = base_url.rstrip("/") + "/chat/completions"
        api_key = get_api_key("openai") or ""
        log("Using custom base URL: %s" % url)
        return url, api_key
    if provider not in PROVIDERS:
        print("%sUnknown provider: %s%s" % (RED, provider, RESET),
              file=sys.stderr)
        print_provider_help()
        sys.exit(1)
    config = PROVIDERS[provider]
    api_key = get_api_key(provider)
    if not api_key:
        api_key_urls = {
            "anthropic": "https://console.anthropic.com/settings/keys",
            "openai": "https://platform.openai.com/api-keys",
            "googleai": "https://aistudio.google.com/app/apikey",
            "openrouter": "https://openrouter.ai/settings/keys",
        }
        print("%sMissing API key: %s%s" % (RED, config["env"], RESET),
              file=sys.stderr)
        url = api_key_urls.get(provider, "")
        if url:
            print("Get key: %s" % url, file=sys.stderr)
        sys.exit(1)
    return config["url"], api_key


# === API REQUEST/RESPONSE ===


def make_request(url, api_key, model, messages, output_config, provider, tools=None):
    headers = {
        "Content-Type": "application/json",
    }
    if provider == "anthropic":
        headers["x-api-key"] = api_key
        headers["anthropic-version"] = "2023-06-01"
    else:
        headers["Authorization"] = "Bearer %s" % api_key
    if provider == "anthropic":
        # Convert messages to Anthropic format
        system_content = None
        anthropic_messages = []
        for msg in messages:
            if msg["role"] == "system":
                system_content = msg["content"]
            else:
                anthropic_messages.append(msg)
        body = {
            "model": model,
            "max_tokens": 4096,
            "messages": anthropic_messages,
        }
        if system_content:
            body["system"] = system_content
        all_tools = []
        if output_config and output_config.get("schema"):
            all_tools.append(to_anthropic_tool(build_schema_tool(
                output_config["schema"])))
        if tools:
            for tool_info in tools.values():
                all_tools.append(to_anthropic_tool(tool_info["schema"]))
        if all_tools:
            body["tools"] = all_tools
            if output_config and output_config.get("schema") and not tools:
                body["tool_choice"] = {"type": "tool", "name": "extract"}
    else:
        body = {
            "model": model,
            "messages": messages,
        }
        all_tools = []
        if output_config and output_config.get("schema"):
            all_tools.append(build_schema_tool(output_config["schema"]))
        if tools:
            for tool_info in tools.values():
                all_tools.append(tool_info["schema"])
        if all_tools:
            body["tools"] = all_tools
            if output_config and output_config.get("schema") and not tools:
                body["tool_choice"] = {
                    "type": "function",
                    "function": {"name": "extract"}
                }
    data = json.dumps(body).encode("utf-8")
    log("Request URL: %s" % url)
    log("Request body: %s" % json.dumps(body, indent=2))
    req = urllib.request.Request(url, data=data, headers=headers, method="POST")
    try:
        start_time = time.time()
        with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
            response_body = resp.read().decode("utf-8")
        elapsed = time.time() - start_time
        log("Response: %s" % response_body)
        response = json.loads(response_body)
        response["_elapsed"] = elapsed
        return response
    except (socket.gaierror, socket.timeout, urllib.error.URLError) as e:
        error_detail = str(e.reason) if hasattr(e, 'reason') else str(e)
        print("%sNetwork error: %s%s" % (RED, error_detail, RESET),
              file=sys.stderr)
        print("Please check your network connection and verify the remote "
              "server is accessible.", file=sys.stderr)
        log("Network error details: %s" % e)
        sys.exit(1)
    except urllib.error.HTTPError as e:
        error_body = e.read().decode("utf-8")
        log("Error response: %s" % error_body)
        message = extract_error_message(error_body)

        req_preview = data[:200].decode("utf-8", errors="replace")
        req_preview = req_preview.replace("\r", " ").replace("\n", " ")

        print("%sLLM request failed (model=%s): %s%s" %
              (RED, model, message, RESET), file=sys.stderr)
        print("%sRequest preview: %s%s" %
              (YELLOW, req_preview, RESET), file=sys.stderr)

        # Check if this is an invalid model error and suggest alternatives
        msg_lower = message.lower()
        if "is not a valid model" in msg_lower or "model not found" in msg_lower \
           or "does not exist" in msg_lower or "invalid model" in msg_lower:
            if "openrouter.ai" in url:
                suggest_openrouter_models(model)
            elif "api.openai.com" in url:
                suggest_openai_models(api_key)
            elif "generativelanguage.googleapis.com" in url:
                suggest_googleai_models(api_key)
        sys.exit(1)


def extract_response(response, output_config, provider):
    if provider == "anthropic":
        content = response.get("content", [])
        for block in content:
            if block.get("type") == "tool_use" and block.get("name") == "extract":
                return json.dumps(block.get("input", {}), indent=2)
    else:
        choices = response.get("choices", [])
        if choices:
            message = choices[0].get("message", {})
            tool_calls = message.get("tool_calls", [])
            for tc in tool_calls:
                if tc.get("function", {}).get("name") == "extract":
                    return tc.get("function", {}).get("arguments", "{}")
    return extract_text_content(response, provider)


def extract_text_content(response, provider):
    if provider == "anthropic":
        content = response.get("content", [])
        texts = []
        for block in content:
            if block.get("type") == "text":
                texts.append(block.get("text", ""))
        return "".join(texts)
    else:
        choices = response.get("choices", [])
        if choices:
            message = choices[0].get("message", {})
            return message.get("content", "") or ""
        return ""


def extract_tool_calls(response, provider):
    tool_calls = []
    if provider == "anthropic":
        content = response.get("content", [])
        for block in content:
            if block.get("type") == "tool_use":
                tool_calls.append({
                    "id": block.get("id", ""),
                    "name": block.get("name", ""),
                    "arguments": block.get("input", {}),
                })
    else:
        choices = response.get("choices", [])
        if choices:
            message = choices[0].get("message", {})
            for tc in message.get("tool_calls", []):
                args_str = tc.get("function", {}).get("arguments", "{}")
                try:
                    args = json.loads(args_str)
                except ValueError:
                    args = {}
                tool_calls.append({
                    "id": tc.get("id", ""),
                    "name": tc.get("function", {}).get("name", ""),
                    "arguments": args,
                })
    return tool_calls


def format_tokens(n):
    """Format token count with k suffix for thousands."""
    if n >= 1000:
        return "%.1fk" % (n / 1000) if n < 10000 else "%dk" % (n // 1000)
    return str(n)


def print_usage(response):
    """Print token usage, cost, and timing info from response."""
    global SESSION_COST, SESSION_COST_KNOWN
    usage = response.get("usage", {})
    elapsed = response.get("_elapsed")
    if not usage and elapsed is None:
        return
    sent = usage.get("prompt_tokens") or usage.get("input_tokens") or 0
    recv = usage.get("completion_tokens") or usage.get("output_tokens") or 0
    total = usage.get("total_tokens") or (sent + recv)
    cost = usage.get("cost")
    if cost is not None:
        SESSION_COST += cost
    else:
        SESSION_COST_KNOWN = False
    msg = "Tokens: %s sent, %s received, %s total." % (
        format_tokens(sent), format_tokens(recv), format_tokens(total))
    if cost is not None or SESSION_COST > 0:
        cost_str = "$%.2f" % cost if cost >= 0.01 else "$%.4f" % cost
        session_str = "$%.2f" % SESSION_COST if SESSION_COST >= 0.01 else "$%.4f" % SESSION_COST
        msg += " Cost: %s message, %s session." % (cost_str, session_str)
    if elapsed is not None:
        msg += " Time: %.3fs." % elapsed
    print("%s%s%s" % (YELLOW, msg, RESET), file=sys.stderr)


def extract_error_message(error_body):
    try:
        data = json.loads(error_body)
        if "error" in data:
            err = data["error"]
            if isinstance(err, dict):
                err_type = err.get("type", "")
                message = err.get("message", "")
                if err_type and message:
                    return "%s: %s" % (err_type, message)
                if message:
                    return message
                if err_type:
                    return err_type
            if isinstance(err, str):
                return err
        if "message" in data:
            return data["message"]
    except ValueError:
        pass
    return error_body


def fetch_openrouter_models(prefix=None):
    """Fetch available models from OpenRouter, optionally filtered by prefix."""
    url = "https://openrouter.ai/api/v1/models"
    try:
        req = urllib.request.Request(url)
        req.add_header('User-Agent', 'runprompt/1.0')
        with urllib.request.urlopen(req, timeout=10) as resp:
            data = json.loads(resp.read().decode('utf-8'))
        models = data.get("data", [])
        if prefix:
            models = [m for m in models if m.get("id", "").startswith(prefix)]
        # Sort by context length (proxy for capability/popularity)
        models.sort(key=lambda m: m.get("context_length", 0), reverse=True)
        return models
    except (urllib.error.URLError, urllib.error.HTTPError, OSError, ValueError):
        return []


def fetch_openai_models(api_key):
    """Fetch available models from OpenAI."""
    url = "https://api.openai.com/v1/models"
    try:
        req = urllib.request.Request(url)
        req.add_header('Authorization', 'Bearer %s' % api_key)
        req.add_header('User-Agent', 'runprompt/1.0')
        with urllib.request.urlopen(req, timeout=10) as resp:
            data = json.loads(resp.read().decode('utf-8'))
        models = data.get("data", [])
        # Filter to chat models (gpt-*, o1-*, chatgpt-*)
        chat_prefixes = ('gpt-', 'o1-', 'o3-', 'chatgpt-')
        models = [m for m in models if any(
            m.get("id", "").startswith(p) for p in chat_prefixes)]
        # Sort alphabetically
        models.sort(key=lambda m: m.get("id", ""))
        return models
    except (urllib.error.URLError, urllib.error.HTTPError, OSError, ValueError):
        return []


def fetch_googleai_models(api_key):
    """Fetch available models from Google AI."""
    url = "https://generativelanguage.googleapis.com/v1beta/models?key=%s" % api_key
    try:
        req = urllib.request.Request(url)
        req.add_header('User-Agent', 'runprompt/1.0')
        with urllib.request.urlopen(req, timeout=10) as resp:
            data = json.loads(resp.read().decode('utf-8'))
        models = data.get("models", [])
        # Filter to generative models
        models = [m for m in models if "generateContent" in
                  m.get("supportedGenerationMethods", [])]
        # Sort by name
        models.sort(key=lambda m: m.get("name", ""))
        return models
    except (urllib.error.URLError, urllib.error.HTTPError, OSError, ValueError):
        return []


def suggest_openrouter_models(invalid_model):
    """Suggest similar OpenRouter models when an invalid model is specified."""
    # Extract provider prefix (e.g., "google" from "google/gemini-3-pro")
    parts = invalid_model.split("/", 1)
    if not parts:
        return
    prefix = parts[0] + "/"
    print("Fetching available %s models..." % parts[0], file=sys.stderr)
    models = fetch_openrouter_models(prefix)
    if not models:
        return
    print("\nAvailable %s models on OpenRouter:" % parts[0], file=sys.stderr)
    for m in models:
        model_id = m.get("id", "")
        ctx = m.get("context_length", 0)
        ctx_str = "%dk" % (ctx // 1000) if ctx >= 1000 else str(ctx)
        print("  openrouter/%s (%s context)" % (model_id, ctx_str),
              file=sys.stderr)


def suggest_openai_models(api_key):
    """Suggest available OpenAI models."""
    print("Fetching available models...", file=sys.stderr)
    models = fetch_openai_models(api_key)
    if not models:
        return
    print("\nAvailable OpenAI models:", file=sys.stderr)
    for m in models:
        model_id = m.get("id", "")
        print("  openai/%s" % model_id, file=sys.stderr)


def suggest_googleai_models(api_key):
    """Suggest available Google AI models."""
    print("Fetching available models...", file=sys.stderr)
    models = fetch_googleai_models(api_key)
    if not models:
        return
    print("\nAvailable Google AI models:", file=sys.stderr)
    for m in models:
        # Model name is like "models/gemini-1.5-pro"
        name = m.get("name", "").replace("models/", "")
        print("  googleai/%s" % name, file=sys.stderr)


def to_anthropic_tool(tool_schema):
    func = tool_schema["function"]
    return {
        "name": func["name"],
        "description": func["description"],
        "input_schema": func["parameters"],
    }


def build_assistant_message(response, provider):
    if provider == "anthropic":
        return {
            "role": "assistant",
            "content": response.get("content", []),
        }
    else:
        choices = response.get("choices", [])
        if choices:
            return choices[0].get("message", {"role": "assistant", "content": ""})
        return {"role": "assistant", "content": ""}


def build_tool_result_message(tool_call, result, error, provider):
    if error:
        content = json.dumps({"error": error})
    else:
        content = json.dumps(result) if not isinstance(result, str) else result
    if provider == "anthropic":
        return {
            "role": "user",
            "content": [{
                "type": "tool_result",
                "tool_use_id": tool_call["id"],
                "content": content,
            }]
        }
    return {
        "role": "tool",
        "tool_call_id": tool_call["id"],
        "content": content,
    }


# === TOOL LOADING & SCHEMA ===


def load_shell_tools(shell_tool_specs):
    """Load shell script tools from shell_tools: frontmatter."""
    tools = {}
    
    for name, spec in shell_tool_specs.items():
        # Handle short form: name: "command"
        if isinstance(spec, str):
            cmd = spec
            safe = False
            description = cmd
        # Handle long form: name: {cmd: ..., safe: ..., description: ...}
        elif isinstance(spec, dict):
            cmd = spec.get("cmd")
            if not cmd:
                print("%sWarning: shell_tool '%s' missing 'cmd' field%s" %
                      (YELLOW, name, RESET), file=sys.stderr)
                continue
            safe = spec.get("safe", False)
            description = spec.get("description", cmd)
        else:
            print("%sWarning: shell_tool '%s' has invalid spec%s" %
                  (YELLOW, name, RESET), file=sys.stderr)
            continue
        
        # Create the tool function
        func = create_shell_tool(name, cmd, description, safe)
        schema = function_to_tool_schema(func)
        if schema:
            schema['function']['name'] = name
            tools[name] = {"schema": schema, "func": func}
            log("Loaded shell tool: %s" % name)
    
    return tools


def create_shell_tool(name, cmd, description, safe):
    """Create a shell tool function with the given command."""
    def shell_tool(args: str = "", **env_vars):
        """Execute shell command with optional args and environment variables."""
        import subprocess
        
        # Build environment with provided env vars
        env = os.environ.copy()
        for key, value in env_vars.items():
            env[key] = str(value)
        
        # Build full command
        full_cmd = cmd
        if args:
            full_cmd = "%s %s" % (cmd, args)
        
        # Use user's configured shell, fallback to /bin/sh
        shell = env.get("SHELL", "/bin/sh")
        
        log("Running shell tool '%s': %s" % (name, full_cmd))
        try:
            result = subprocess.run(
                [shell, "-c", full_cmd],
                capture_output=True,
                text=True,
                timeout=30,
                env=env,
            )
            if result.returncode == 0:
                return result.stdout.rstrip('\n')
            else:
                return {
                    "returncode": result.returncode,
                    "stdout": result.stdout.rstrip('\n'),
                    "stderr": result.stderr.rstrip('\n'),
                }
        except subprocess.TimeoutExpired:
            raise RuntimeError("Command timed out after 30 seconds")
        except Exception as e:
            raise RuntimeError("Command failed: %s" % str(e))
    
    # Set docstring and metadata
    shell_tool.__doc__ = description
    shell_tool.safe = safe
    shell_tool.__name__ = name
    
    return shell_tool


def load_tools(tool_specs, search_paths):
    tools = {}  # name -> {"schema": ..., "func": ...}
    for spec in tool_specs:
        # Handle builtin tools
        if spec.startswith("builtin."):
            builtin_spec = spec[8:]  # Remove "builtin." prefix
            if builtin_spec == "*":
                # Import all builtin tools (skip factories)
                for name, func in BUILTIN_TOOLS.items():
                    if getattr(func, 'is_factory', False):
                        continue
                    schema = function_to_tool_schema(func)
                    if schema:
                        schema['function']['name'] = name
                        tools[name] = {"schema": schema, "func": func}
                        log("Loaded builtin tool: %s" % name)
            else:
                # Check for parameterized builtin: func(args)
                factory_match = re.match(r"(\w+)\((.+)\)$", builtin_spec)
                if factory_match:
                    func_name = factory_match.group(1)
                    args_str = factory_match.group(2)
                    if func_name in BUILTIN_TOOLS:
                        factory = BUILTIN_TOOLS[func_name]
                        if getattr(factory, 'is_factory', False):
                            # Parse arguments and call factory
                            try:
                                factory_args = parse_factory_args(args_str)
                                func = factory(*factory_args)
                                # Use factory-provided name or generate one
                                tool_name = getattr(func, 'tool_name', func_name)
                                schema = function_to_tool_schema(func)
                                if schema:
                                    # Override schema name with tool_name
                                    schema['function']['name'] = tool_name
                                    tools[tool_name] = {
                                        "schema": schema, "func": func}
                                    log("Loaded builtin tool: %s" % tool_name)
                            except Exception as e:
                                print("%sWarning: Failed to create tool "
                                      "'%s': %s%s" %
                                      (YELLOW, spec, e, RESET), file=sys.stderr)
                        else:
                            print("%sWarning: '%s' is not a factory tool%s" %
                                  (YELLOW, func_name, RESET), file=sys.stderr)
                    else:
                        print("%sWarning: Unknown builtin tool '%s'%s" %
                              (YELLOW, func_name, RESET), file=sys.stderr)
                # Import specific builtin tool
                elif builtin_spec in BUILTIN_TOOLS:
                    func = BUILTIN_TOOLS[builtin_spec]
                    if getattr(func, 'is_factory', False):
                        print("%sWarning: '%s' requires arguments, "
                              "e.g. builtin.%s('path')%s" %
                              (YELLOW, builtin_spec, builtin_spec, RESET),
                              file=sys.stderr)
                    else:
                        schema = function_to_tool_schema(func)
                        if schema:
                            schema['function']['name'] = builtin_spec
                            tools[builtin_spec] = {"schema": schema, "func": func}
                            log("Loaded builtin tool: %s" % builtin_spec)
                else:
                    print("%sWarning: Unknown builtin tool '%s'%s" %
                          (YELLOW, builtin_spec, RESET), file=sys.stderr)
            continue
        if spec.endswith(".*"):
            # Import all functions from module
            module_name = spec[:-2]
            try:
                module = load_module_from_path(module_name, search_paths)
                if module is None:
                    raise ImportError("No module named '%s'" % module_name)
                for name in dir(module):
                    if name.startswith("_"):
                        continue
                    obj = getattr(module, name)
                    if (inspect.isfunction(obj) and
                            obj.__module__ == module.__name__ and
                            inspect.getdoc(obj)):
                        schema = function_to_tool_schema(obj)
                        if schema:
                            schema['function']['name'] = name
                            tools[name] = {"schema": schema, "func": obj}
                            log("Loaded tool: %s from %s" % (name, module_name))
            except Exception as e:
                print("%sWarning: Could not import tool '%s': %s%s" %
                      (YELLOW, spec, e, RESET), file=sys.stderr)
        else:
            # Import specific function
            parts = spec.rsplit(".", 1)
            if len(parts) != 2:
                print("%sWarning: Invalid tool spec '%s'%s" %
                      (YELLOW, spec, RESET), file=sys.stderr)
                continue
            module_name, func_name = parts
            try:
                module = load_module_from_path(module_name, search_paths)
                if module is None:
                    raise ImportError("No module named '%s'" % module_name)
                if not hasattr(module, func_name):
                    raise AttributeError("Module '%s' has no function '%s'" %
                                         (module_name, func_name))
                func = getattr(module, func_name)
                if not callable(func):
                    raise TypeError("'%s' is not callable" % func_name)
                schema = function_to_tool_schema(func)
                if schema:
                    schema['function']['name'] = func_name
                    tools[func_name] = {"schema": schema, "func": func}
                    log("Loaded tool: %s from %s" % (func_name, module_name))
            except Exception as e:
                print("%sWarning: Could not import tool '%s': %s%s" %
                      (YELLOW, spec, e, RESET), file=sys.stderr)
    return tools


def parse_factory_args(args_str):
    """Parse factory arguments from a string like "'hello.txt'" or "1, 'two'"."""
    # Use ast.literal_eval for safe parsing of literals
    import ast
    # Wrap in tuple if multiple args, or just eval single arg
    if ',' in args_str:
        return ast.literal_eval('(' + args_str + ')')
    else:
        return (ast.literal_eval(args_str),)


def load_module_from_path(module_name, search_paths):
    parts = module_name.split(".")
    for base_path in search_paths:
        # Build list of candidate paths to try
        candidates = [os.path.join(base_path, module_name + ".py")]
        if len(parts) > 1:
            candidates.append(os.path.join(base_path, *parts[:-1], parts[-1] + ".py"))
            candidates.append(os.path.join(base_path, *parts, "__init__.py"))
        for file_path in candidates:
            if os.path.exists(file_path):
                spec = importlib.util.spec_from_file_location(module_name, file_path)
                module = importlib.util.module_from_spec(spec)
                sys.modules[module_name] = module
                spec.loader.exec_module(module)
                return module
    return None


def function_to_tool_schema(func):
    doc = inspect.getdoc(func)
    sig = inspect.signature(func)
    if not doc:
        # Auto-generate minimal description from signature
        params = ", ".join(
            "%s: %s" % (p, python_type_to_json_type(
                sig.parameters[p].annotation
                if sig.parameters[p].annotation is not inspect.Parameter.empty
                else None))
            for p in sig.parameters if p not in ("self", "cls"))
        doc = "%s(%s)" % (func.__name__, params)
        print("%sWarning: Function '%s' has no docstring, using: %s%s" %
              (YELLOW, func.__name__, doc, RESET), file=sys.stderr)
    properties = {}
    required = []
    for param_name, param in sig.parameters.items():
        if param_name in ("self", "cls"):
            continue
        param_type = param.annotation
        if param_type is inspect.Parameter.empty:
            param_type = None
        json_type = python_type_to_json_type(param_type)
        properties[param_name] = {"type": json_type}
        if param.default is inspect.Parameter.empty:
            required.append(param_name)
    return build_tool_schema(func.__name__, doc, properties, required)


def python_type_to_json_type(py_type):
    if py_type is None:
        return "string"
    type_name = getattr(py_type, "__name__", str(py_type))
    mapping = {
        "str": "string",
        "int": "integer",
        "float": "number",
        "bool": "boolean",
        "list": "array",
        "dict": "object",
    }
    return mapping.get(type_name, "string")


def build_tool_schema(name, description, properties, required):
    return {
        "type": "function",
        "function": {
            "name": name,
            "description": description,
            "parameters": {
                "type": "object",
                "properties": properties,
                "required": required,
            },
        },
    }


def build_schema_tool(schema):
    properties = {}
    required = []
    for key, value in schema.items():
        clean_key = key.rstrip("?")
        is_optional = key.endswith("?")
        parts = value.split(",", 1) if isinstance(value, str) else [value]
        type_str = parts[0].strip() if parts else "string"
        description = parts[1].strip() if len(parts) > 1 else ""
        json_type = "string"
        if type_str == "number":
            json_type = "number"
        elif type_str == "boolean":
            json_type = "boolean"
        prop = {"type": json_type}
        if description:
            prop["description"] = description
        properties[clean_key] = prop
        if not is_optional:
            required.append(clean_key)
    return build_tool_schema("extract", "Extract structured data", properties,
                             required)


# === TOOL EXECUTION & USER INTERACTION ===


def execute_tool(tool_func, args):
    try:
        result = tool_func(**args)
        return result, None
    except Exception as e:
        error_msg = "%s: %s" % (type(e).__name__, str(e))
        return None, error_msg


def prompt_user_for_tool(tool_name, args):
    prompt_text = "Run this tool? [Y/n]: "
    while True:
        if (tool_name == "shell" and isinstance(args, dict) and
                isinstance(args.get("command"), str)):
            sys.stderr.write("%s%s%s\n" % (CYAN, args.get("command"), RESET))
            sys.stderr.flush()

        response = read_tty_line(prompt_text)

        if response is None:
            return True
        if response == "" or response.lower() in ("y", "yes"):
            return True
        if response.lower() in ("n", "no"):
            return False
        print("Please enter 'y' or 'n'", file=sys.stderr)


def is_tool_safe(tool_func):
    return getattr(tool_func, 'safe', False)


def print_tool_call(tool_name, args):
    summary = format_tool_call_summary(tool_name, args)
    print("%s%s%s" % (YELLOW, summary, RESET), file=sys.stderr)


def format_tool_call_summary(tool_name, args):
    if not args:
        return "Tool: %s()" % tool_name
    arg_parts = []
    for k, v in args.items():
        q = '"' if isinstance(v, str) else ''
        arg_parts.append('%s=%s%s%s' % (k, q, truncate_value(v, 30), q))
    args_str = ", ".join(arg_parts)
    if len(args_str) > 60:
        args_str = args_str[:57] + "..."
    return "Tool: %s(%s)" % (tool_name, args_str)


def truncate_value(value, max_len=50):
    s = str(value).replace('\r\n', '\\n').replace('\r', '\\n').replace('\n', '\\n')
    return s if len(s) <= max_len else s[:max_len - 3] + "..."


def chat_read_input():
    """Read a line of user input for chat mode. Returns None on EOF/empty."""
    try:
        sys.stderr.write("%s> %s" % (GREEN, RESET))
        sys.stderr.flush()
        if sys.stdin.isatty():
            line = sys.stdin.readline()
        else:
            try:
                tty = open("/dev/tty", "r")
                line = tty.readline()
                tty.close()
            except (IOError, OSError):
                return None
        line = line.strip()
        return line if line else None
    except (KeyboardInterrupt, EOFError):
        sys.stderr.write("\n")
        return None


def read_tty_line(prompt_text):
    # If stdin is a tty (e.g. running in a pty for testing), read from it directly
    if sys.stdin.isatty():
        sys.stderr.write(prompt_text)
        sys.stderr.flush()
        return sys.stdin.readline().strip()
    # Try /dev/tty for interactive input when stdin is a pipe
    try:
        tty = open("/dev/tty", "r")
        sys.stderr.write(prompt_text)
        sys.stderr.flush()
        response = tty.readline().strip()
        tty.close()
        return response
    except (IOError, OSError):
        pass
    # No input available
    return None


# === CACHING ===


def cache_key(rendered_prompt, effective_meta):
    data = json.dumps({
        "prompt": rendered_prompt,
        "meta": effective_meta,
    }, sort_keys=True)
    return hashlib.sha256(data.encode()).hexdigest()


def cache_get(key):
    cache_dir = get_cache_dir()
    cache_file = os.path.join(cache_dir, "%s.json" % key)
    if os.path.exists(cache_file):
        log("Cache hit: %s" % cache_file)
        with open(cache_file, "r") as f:
            return json.load(f)
    log("Cache miss")
    return None


def cache_set(key, response, provider):
    cache_dir = get_cache_dir()
    if not os.path.exists(cache_dir):
        os.makedirs(cache_dir)
    cache_file = os.path.join(cache_dir, "%s.json" % key)
    response_with_provider = {"_provider": provider}
    response_with_provider.update(response)
    with open(cache_file, "w") as f:
        json.dump(response_with_provider, f, indent=2)
    log("Cached response: %s" % cache_file)


def clear_cache():
    cache_dir = get_cache_dir()
    if not os.path.exists(cache_dir):
        print("Cache directory does not exist: %s" % cache_dir)
        return
    count = 0
    for filename in os.listdir(cache_dir):
        if filename.endswith(".json"):
            os.remove(os.path.join(cache_dir, filename))
            count += 1
    print("Cleared %d cached response(s) from %s" % (count, cache_dir))


def get_cache_dir():
    from_conf = get_conf("cache_dir")
    if from_conf:
        return from_conf
    xdg_cache = os.environ.get("XDG_CACHE_HOME", os.path.expanduser("~/.cache"))
    return os.path.join(xdg_cache, "runprompt")


# === TEST/DEBUG UTILITIES ===


def load_test_response(prompt_path):
    test_file = prompt_path + ".test-response"
    if not os.path.exists(test_file):
        print("Test response file not found: %s" % test_file, file=sys.stderr)
        sys.exit(1)
    with open(test_file, "r") as f:
        content = f.read()
    log("Loaded test response from: %s" % test_file)
    return json.loads(content)


def save_response(response, provider, save_path):
    response_with_provider = {"_provider": provider}
    response_with_provider.update(response)
    with open(save_path, "w") as f:
        json.dump(response_with_provider, f, indent=2)
    log("Saved response to: %s" % save_path)


def log(msg):
    if get_conf("verbose", False):
        print(msg, file=sys.stderr)


# === FILE READING FOR --read ===


def is_url(path):
    """Check if path is a URL."""
    return path.startswith("http://") or path.startswith("https://")


def is_binary_file(path):
    """Check if a file is binary by looking for null bytes."""
    try:
        with open(path, 'rb') as f:
            chunk = f.read(8192)
        return b'\x00' in chunk
    except (IOError, OSError):
        return False


def get_mime_type(path):
    """Get MIME type for a file, with sensible defaults."""
    mime_type, _ = mimetypes.guess_type(path)
    if mime_type:
        return mime_type
    # Default based on binary detection
    return "application/octet-stream" if is_binary_file(path) else "text/plain"


def is_supported_media_type(mime_type):
    """Check if MIME type is supported for media blocks."""
    supported_prefixes = ("image/", "audio/", "video/")
    supported_types = ("application/pdf",)
    return (mime_type.startswith(supported_prefixes) or
            mime_type in supported_types)


def get_url_mime_type(url):
    """Get MIME type for URL, using HEAD request if extension unknown."""
    mime_type, _ = mimetypes.guess_type(url)
    if mime_type:
        return mime_type
    # Try HEAD request for Content-Type
    try:
        req = urllib.request.Request(url, method='HEAD')
        req.add_header('User-Agent', 'runprompt/1.0')
        with urllib.request.urlopen(req, timeout=10) as resp:
            content_type = resp.headers.get('Content-Type', '')
            if content_type:
                return content_type.split(';')[0].strip()
    except (urllib.error.URLError, urllib.error.HTTPError, OSError):
        pass
    return "text/plain"


def fetch_url_content(url):
    """Fetch URL content, return (content_bytes, mime_type)."""
    req = urllib.request.Request(url)
    req.add_header('User-Agent', 'runprompt/1.0')
    with urllib.request.urlopen(req, timeout=30) as resp:
        content = resp.read()
        content_type = resp.headers.get('Content-Type', '')
        mime_type = content_type.split(';')[0].strip() if content_type else None
        if not mime_type:
            mime_type, _ = mimetypes.guess_type(url)
        if not mime_type:
            mime_type = "application/octet-stream" if b'\x00' in content[:8192] else "text/plain"
        return content, mime_type


def read_files_for_context(patterns, provider=None):
    """Read files matching patterns, return list of file info dicts.
    
    Each dict has: path, content (bytes or str or None), is_binary, is_url, mime_type.
    For URLs with non-Anthropic providers, content may be None (use URL directly).
    """
    files = []
    seen = set()
    for pattern in patterns:
        # Handle URLs (no glob expansion)
        if is_url(pattern):
            if pattern in seen:
                continue
            seen.add(pattern)
            mime_type = mimetypes.guess_type(pattern)[0]
            # OpenRouter proxies URLs for us; all other providers need base64
            if provider == "openrouter":
                # Use URL directly (OpenRouter fetches for us)
                if not mime_type:
                    mime_type = get_url_mime_type(pattern)
                is_binary = is_supported_media_type(mime_type)
                files.append({
                    "path": pattern,
                    "content": None,  # Will use URL directly
                    "is_binary": is_binary,
                    "is_url": True,
                    "mime_type": mime_type,
                })
                print("%sLoaded URL: %s%s" % (YELLOW, pattern, RESET),
                      file=sys.stderr)
                log("URL (no download): %s (%s)" % (pattern, mime_type))
            else:
                # Download for anthropic, openai, googleai, etc.
                if not mime_type:
                    mime_type = None  # Will get from download
                print("%sDownloading: %s%s" % (YELLOW, pattern, RESET),
                      file=sys.stderr)
                try:
                    content, mime_type = fetch_url_content(pattern)
                    is_binary = b'\x00' in content[:8192] if content else False
                    files.append({
                        "path": pattern,
                        "content": content,
                        "is_binary": is_binary,
                        "is_url": True,
                        "mime_type": mime_type,
                    })
                    print("%sLoaded URL: %s%s" % (YELLOW, pattern, RESET),
                          file=sys.stderr)
                    log("Downloaded URL: %s (%s)" % (pattern, mime_type))
                except (urllib.error.URLError, urllib.error.HTTPError, OSError) as e:
                    print("%sWarning: Cannot fetch '%s': %s%s" %
                          (YELLOW, pattern, e, RESET), file=sys.stderr)
            continue
        # Expand glob pattern for local files
        matches = glob.glob(pattern, recursive=True)
        if not matches:
            if os.path.exists(pattern):
                matches = [pattern]
            else:
                print("%sWarning: No files match '%s'%s" %
                      (YELLOW, pattern, RESET), file=sys.stderr)
                continue
        for path in sorted(matches):
            if path in seen:
                continue
            seen.add(path)
            if os.path.isdir(path):
                continue
            try:
                is_binary = is_binary_file(path)
                if is_binary:
                    with open(path, 'rb') as f:
                        content = f.read()
                else:
                    with open(path, 'r', encoding='utf-8', errors='replace') as f:
                        content = f.read()
                files.append({
                    "path": path,
                    "content": content,
                    "is_binary": is_binary,
                    "is_url": False,
                    "mime_type": get_mime_type(path),
                })
                print("%sLoaded file: %s%s" % (YELLOW, path, RESET),
                      file=sys.stderr)
                log("Read file: %s (%s)" %
                    (path, "binary" if is_binary else "text"))
            except (IOError, OSError) as e:
                print("%sWarning: Cannot read '%s': %s%s" %
                      (YELLOW, path, e, RESET), file=sys.stderr)
    return files


def build_media_block(file_info, provider):
    """Build a media content block for binary files or URLs."""
    path = file_info["path"]
    content = file_info["content"]
    mime_type = file_info["mime_type"]
    is_url = file_info["is_url"]
    
    # For non-Anthropic providers, use URL directly if available
    if is_url and content is None and provider != "anthropic":
        if mime_type and mime_type.startswith("image/"):
            return {"type": "image_url", "image_url": {"url": path}}
        # Non-image URL without content - just describe it
        return {"type": "text", "text": "[URL: %s (%s)]" % (path, mime_type)}
    
    # Need actual content for base64 encoding
    if content is None:
        return {"type": "text", "text": "[URL: %s (not downloaded)]" % path}
    
    b64_data = base64.b64encode(content).decode('ascii')
    if provider == "anthropic":
        if mime_type.startswith("audio/"):
            return {
                "type": "audio",
                "source": {
                    "type": "base64",
                    "media_type": mime_type,
                    "data": b64_data,
                }
            }
        if mime_type == "application/pdf":
            return {
                "type": "document",
                "source": {
                    "type": "base64",
                    "media_type": mime_type,
                    "data": b64_data,
                }
            }
        return {
            "type": "image",
            "source": {
                "type": "base64",
                "media_type": mime_type,
                "data": b64_data,
            }
        }
    else:
        # OpenAI format (also used by Google AI compat layer, OpenRouter)
        if mime_type.startswith("audio/"):
            # OpenAI / OpenRouter format
            audio_format = "mp3"
            if "wav" in mime_type:
                audio_format = "wav"
            return {
                "type": "input_audio",
                "input_audio": {
                    "data": b64_data,
                    "format": audio_format
                }
            }

        data_url = "data:%s;base64,%s" % (mime_type, b64_data)
        if mime_type.startswith("image/"):
            return {"type": "image_url", "image_url": {"url": data_url}}
        # For other types, fall back to text description
        return {"type": "text", "text": "[Binary file: %s (%s, %d bytes)]" %
                (path, mime_type, len(content))}


def build_content_with_files(prompt, files, provider):
    """Build multi-part content array with files and prompt."""
    if not files:
        return prompt  # Simple string content
    content = []
    # Add header
    content.append({"type": "text", "text": "# Attached files\n"})
    # Add each file
    for file_info in files:
        path = file_info["path"]
        file_content = file_info["content"]
        is_binary = file_info["is_binary"]
        mime_type = file_info["mime_type"]
        content.append({"type": "text", "text": "## File: %s\n" % path})
        if is_binary:
            if is_supported_media_type(mime_type):
                content.append(build_media_block(file_info, provider))
            else:
                size = len(file_content) if file_content else 0
                content.append({
                    "type": "text",
                    "text": "[Binary file: %s (%s, %d bytes)]" %
                            (path, mime_type, size)
                })
        else:
            if file_content is not None:
                # Handle bytes content from URL downloads
                if isinstance(file_content, bytes):
                    file_content = file_content.decode('utf-8', errors='replace')
                # Use placeholder for empty files
                if not file_content or not file_content.strip():
                    file_content = "[EMPTY FILE]"
                content.append({"type": "text", "text": file_content})
            else:
                content.append({"type": "text", "text": "[Content not available]"})
    # Add the main prompt
    content.append({"type": "text", "text": "\n# Prompt\n\n" + prompt})
    return content


# === BEFORE COMMANDS ===


def execute_before_commands(before_commands, variables):
    """Execute shell commands from before: frontmatter and add outputs to variables."""
    import subprocess
    
    if not isinstance(before_commands, dict):
        print("%sError: 'before' must be a dict of name: command pairs%s" %
              (RED, RESET), file=sys.stderr)
        sys.exit(1)
    
    # Build environment with current variables
    env = os.environ.copy()
    for key, value in variables.items():
        env[key] = str(value)
    
    # Use user's configured shell, fallback to /bin/sh
    shell = env.get("SHELL", "/bin/sh")
    
    before_outputs = []
    for name, command in before_commands.items():
        log("Running before command '%s': %s" % (name, command))
        try:
            result = subprocess.run(
                [shell, "-c", command],
                capture_output=True,
                text=True,
                timeout=30,
                env=env,
            )
            if result.returncode == 0:
                output = result.stdout.rstrip('\n')
                variables[name] = output
                before_outputs.append(output)
                log("Command '%s' succeeded: %s" % (name, output[:100]))
            else:
                error = result.stderr.rstrip('\n')
                variables[name] = error
                before_outputs.append(error)
                log("Command '%s' failed: %s" % (name, error[:100]))
        except subprocess.TimeoutExpired:
            error = "Command timed out after 30 seconds"
            variables[name] = error
            before_outputs.append(error)
            log("Command '%s' timed out" % name)
        except Exception as e:
            error = str(e)
            variables[name] = error
            before_outputs.append(error)
            log("Command '%s' error: %s" % (name, error))
    
    # Add combined BEFORE variable
    variables["BEFORE"] = "\n".join(before_outputs)


# === BUILTIN TOOLS ===


def _fetch_clean_simple(url: str):
    """Simple HTML fetcher that extracts visible text (fallback)."""
    from html.parser import HTMLParser

    class TextExtractor(HTMLParser):
        SKIP_TAGS = {'script', 'style', 'head', 'meta', 'link', 'noscript', 'svg'}

        def __init__(self):
            super().__init__()
            self.text = []
            self.skip_depth = 0

        def handle_starttag(self, tag, attrs):
            if tag == 'body':
                self.skip_depth = 0
            elif tag in self.SKIP_TAGS:
                self.skip_depth += 1
            elif tag in ('p', 'br', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4',
                         'h5', 'h6'):
                self.text.append('\n')

        def handle_endtag(self, tag):
            if tag in self.SKIP_TAGS and self.skip_depth > 0:
                self.skip_depth -= 1
            elif tag in ('p', 'div', 'li', 'tr', 'h1', 'h2', 'h3', 'h4', 'h5',
                         'h6'):
                self.text.append('\n')

        def handle_data(self, data):
            if self.skip_depth == 0:
                self.text.append(data)

        def get_text(self):
            import html
            text = ''.join(self.text)
            lines = [' '.join(line.split()) for line in text.split('\n')]
            text = '\n'.join(lines)
            while '\n\n\n' in text:
                text = text.replace('\n\n\n', '\n\n')
            return html.unescape(text.strip())

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                      'AppleWebKit/537.36 (KHTML, like Gecko) '
                      'Chrome/120.0.0.0 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.5',
    }
    req = urllib.request.Request(url, headers=headers)
    with urllib.request.urlopen(req, timeout=30) as resp:
        html_content = resp.read().decode('utf-8', errors='replace')
    extractor = TextExtractor()
    extractor.feed(html_content)
    return extractor.get_text()


def calculator(expression: str):
    """Evaluate a mathematical expression safely.

    Supports arithmetic (+, -, *, /, //, %, **), scientific functions
    (sin, cos, tan, log, sqrt, exp, etc.), and constants (pi, e, tau).
    """
    import ast
    import operator
    import math

    allowed_ops = {
        ast.Add: operator.add,
        ast.Sub: operator.sub,
        ast.Mult: operator.mul,
        ast.Div: operator.truediv,
        ast.FloorDiv: operator.floordiv,
        ast.Mod: operator.mod,
        ast.Pow: operator.pow,
        ast.USub: operator.neg,
        ast.UAdd: operator.pos,
    }

    allowed_functions = {
        'sin': math.sin, 'cos': math.cos, 'tan': math.tan,
        'asin': math.asin, 'acos': math.acos, 'atan': math.atan,
        'atan2': math.atan2,
        'sinh': math.sinh, 'cosh': math.cosh, 'tanh': math.tanh,
        'asinh': math.asinh, 'acosh': math.acosh, 'atanh': math.atanh,
        'exp': math.exp, 'log': math.log, 'log10': math.log10,
        'log2': math.log2, 'sqrt': math.sqrt, 'pow': pow,
        'abs': abs, 'ceil': math.ceil, 'floor': math.floor,
        'trunc': math.trunc, 'round': round,
        'degrees': math.degrees, 'radians': math.radians,
        'factorial': math.factorial, 'gcd': math.gcd,
        'max': max, 'min': min, 'sum': sum,
    }
    if hasattr(math, 'lcm'):
        allowed_functions['lcm'] = math.lcm

    allowed_constants = {
        'pi': math.pi, 'e': math.e, 'tau': math.tau,
        'inf': math.inf, 'nan': math.nan,
    }

    def _eval(node):
        if isinstance(node, ast.Expression):
            return _eval(node.body)
        elif isinstance(node, ast.Constant):
            if isinstance(node.value, (int, float, complex)):
                return node.value
            raise ValueError("Only numbers allowed, not %s" %
                             type(node.value).__name__)
        elif hasattr(ast, 'Num') and isinstance(node, ast.Num):
            return node.n
        elif hasattr(ast, 'Str') and isinstance(node, ast.Str):
            raise ValueError("Only numbers allowed, not str")
        elif isinstance(node, ast.BinOp):
            if type(node.op) not in allowed_ops:
                raise ValueError("Operation not allowed: %s" %
                                 type(node.op).__name__)
            return allowed_ops[type(node.op)](_eval(node.left),
                                              _eval(node.right))
        elif isinstance(node, ast.UnaryOp):
            if type(node.op) not in allowed_ops:
                raise ValueError("Operation not allowed: %s" %
                                 type(node.op).__name__)
            return allowed_ops[type(node.op)](_eval(node.operand))
        elif isinstance(node, ast.Call):
            if not isinstance(node.func, ast.Name):
                raise ValueError("Only simple function calls allowed")
            func_name = node.func.id
            if func_name not in allowed_functions:
                raise ValueError("Function not allowed: %s" % func_name)
            args = [_eval(arg) for arg in node.args]
            return allowed_functions[func_name](*args)
        elif isinstance(node, ast.Name):
            if node.id in allowed_constants:
                return allowed_constants[node.id]
            raise ValueError("Name not allowed: %s" % node.id)
        elif isinstance(node, ast.List):
            return [_eval(item) for item in node.elts]
        elif isinstance(node, ast.Tuple):
            return tuple(_eval(item) for item in node.elts)
        else:
            raise ValueError("Node type not allowed: %s" % type(node).__name__)

    tree = ast.parse(expression, mode='eval')
    return _eval(tree)


calculator.safe = True


def fetch_clean(url: str):
    """Fetch a web page and extract its main content as HTML.

    Uses Firefox Reader View via Playwright for best results. Falls back to
    simple text extraction if Playwright is not installed.
    """
    try:
        from playwright.sync_api import sync_playwright
    except ImportError:
        print("%sWarning: playwright not installed. Using simple scraper. "
              "For better results install with:\n"
              "  pip install playwright && playwright install firefox%s" %
              (YELLOW, RESET), file=sys.stderr)
        return _fetch_clean_simple(url)
    with sync_playwright() as p:
        browser = p.firefox.launch(headless=True)
        context = browser.new_context()
        page = context.new_page()
        reader_url = "about:reader?url=%s" % url
        page.goto(reader_url)
        page.wait_for_timeout(1000)
        page.wait_for_selector(
            '[data-l10n-id="about-reader-loading"]',
            state='hidden',
            timeout=20000
        )
        html_content = page.evaluate("""() => {
            const header = document.querySelector('div.header');
            const content = document.querySelector('div.content');
            let result = '';
            if (header) result += header.innerHTML;
            if (content) result += content.innerHTML;
            return result || document.body.innerHTML;
        }""")
        browser.close()
        return html_content


fetch_clean.safe = True


def _slugify_filename_for_tool(path: str):
    """Return a slug suitable for tool naming from a file path."""
    filename = os.path.basename(path).lower()
    slug = re.sub(r"[^a-z0-9]+", "_", filename)
    slug = re.sub(r"_+", "_", slug).strip("_")
    return slug or "file"


def write_file(path: str):
    """Factory that creates a file writer tool for a specific path.

    Usage in prompt: builtin.write_file('hello.txt')
    """
    tool_suffix = _slugify_filename_for_tool(path)
    tool_name = "write_file_%s" % tool_suffix

    def writer(content: str):
        """Write content to the file."""
        with open(path, 'w') as f:
            f.write(content)
        return "Wrote %d bytes to %s" % (len(content), path)
    writer.__doc__ = "Write content to %s." % path
    writer.tool_name = tool_name
    return writer


write_file.is_factory = True


def datetime_now():
    """Return current date and time in ISO format (YYYY-MM-DDTHH:MM:SS)."""
    from datetime import datetime
    return datetime.now().isoformat(timespec='seconds')


datetime_now.safe = True


def sleep(seconds: float):
    """Sleep for the specified number of seconds."""
    time.sleep(seconds)
    return "Slept for %s second(s)" % seconds


sleep.safe = True


def ask_user(question: str):
    """Ask the user a question and return their response.

    This tool reads from the user's terminal (TTY). If no TTY is available, it
    raises an error.
    """
    try:
        import readline  # noqa: F401
    except ImportError:
        pass

    tty = None
    original_stdin = None
    if not sys.stdin.isatty():
        try:
            tty = open("/dev/tty", "r")
        except (IOError, OSError):
            tty = None
        if tty is None:
            raise RuntimeError("No TTY available for user input")
        original_stdin = sys.stdin
        sys.stdin = tty

    try:
        sys.stderr.write("%s\n" % question)
        sys.stderr.flush()

        original_stdout = None
        if sys.stdout is not sys.stderr:
            original_stdout = sys.stdout
            sys.stdout = sys.stderr

        prompt = "\001%s\002> " % GREEN

        try:
            response = input(prompt)
        except EOFError:
            response = ""
        finally:
            sys.stderr.write(RESET)
            sys.stderr.flush()
            if original_stdout is not None:
                sys.stdout = original_stdout

        return response
    finally:
        if original_stdin is not None:
            sys.stdin = original_stdin
        if tty is not None:
            tty.close()


ask_user.safe = True


def shell(command: str, timeout: float = 30):
    """Run a shell command and return stdout/stderr/returncode.

    This tool executes the command using /bin/bash -lc. It is intentionally not
    marked safe.
    """
    import subprocess
    completed = subprocess.run(
        ["/bin/bash", "-lc", command],
        capture_output=True,
        text=True,
        timeout=timeout,
    )
    return {
        "command": command,
        "returncode": completed.returncode,
        "stdout": completed.stdout,
        "stderr": completed.stderr,
    }


BUILTIN_TOOLS = {
    "calculator": calculator,
    "datetime": datetime_now,
    "fetch_clean": fetch_clean,
    "sleep": sleep,
    "ask_user": ask_user,
    "shell": shell,
    "write_file": write_file,
}


if __name__ == "__main__":
    main()