refactor!: rework pipeline and runtime into component-based engine architecture (#1)

Reworks the project from the initial single-purpose script into a component-based
engine: a framework-free core pipeline, descriptor-driven built-ins (source
providers, LLM providers, steps, renderers), and HTTP adapters wired through
bundles. Also documents the squash-merge strategy for the project.

Breaking; pre-1.0, no compatibility with 0.1.x internals.

Co-authored-by: David Soušek <15769039+sousekd@users.noreply.github.com>

Author: sousekd

.dockerignore

@@ -1,4 +1,4 @@
-# Build artifacts and dependencies (rebuilt inside the image)
+# Build artifacts and dependencies rebuilt inside the image
 node_modules
 dist
 coverage
@@ -10,11 +10,11 @@ coverage
 .gitattributes
 .github
 
-# Local config / secrets
+# Environment files are supplied at runtime, never baked into the image
 .env
 .env.*
 
-# Tests and dev tooling (not needed in runtime image)
+# Tests and dev tooling not needed in the runtime image
 tests
 vitest.config.ts
 tsconfig.test.json
@@ -27,7 +27,7 @@ npm-debug.log*
 .idea
 .DS_Store
 
-# Repo docs (kept in the source tree, excluded from the image to keep it small)
+# Repository docs stay outside the runtime image
 README.md
 docs
 AGENTS.md

.env.example

@@ -1,108 +1,84 @@
-# --- Docker Compose ------------------------------------------------------
-# Local image name used by compose.yaml. Ignored by the Node app.
+# --- Compose images ------------------------------------------------------
+# Local image name used by compose.yaml. Ignored by the Node app. Default: llm-context-loader:local
 LLMC_LOCAL_IMAGE=llm-context-loader:local
-# Published image tag used by compose.deploy.yaml. Ignored by the Node app.
+
+# Published image tag used by compose.deploy.yaml. Pin a release tag for repeatable deployments. Default: latest
 LLMC_IMAGE_TAG=latest
 
-# --- Server --------------------------------------------------------------
-# Bind address. 0.0.0.0 listens on every interface; use 127.0.0.1 for local only.
-SERVER_HOST=0.0.0.0
-# TCP port the HTTP server listens on.
-SERVER_PORT=3010
-# pino log level: trace, debug, info, warn, error, fatal, silent.
+# --- Bootstrap -----------------------------------------------------------
+# YAML configuration file loaded at startup. Default: config/llm-context-loader.yaml
+CONFIG_FILE=config/llm-context-loader.yaml
+
+# Bind address. Use 127.0.0.1 for local-only development. Default: 0.0.0.0
+HOST=0.0.0.0
+
+# HTTP server listen port. Default: 3010
+PORT=3010
+
+# pino log level: trace, debug, info, warn, error, fatal, or silent. Default: info
 LOG_LEVEL=info
 
-# --- Auth / Diagnostics --------------------------------------------------
-# Require Authorization: Bearer <API_KEY> on real client routes.
-AUTH_ENABLED=false
-# Shared bearer token for POST / and GET /r/* when AUTH_ENABLED=true.
-API_KEY=
-# Append <context_loader_info /> and return diagnostic document on fetch failures.
-# When false, fetch failures throw HTTP errors and no footer is appended.
-DIAGNOSTIC_FOOTER_ENABLED=true
-
-# --- Clients -------------------------------------------------------------
-# Comma-separated client plugins to register. Empty means only /health is served.
-# Built-ins: openwebui, jina.
-CLIENTS=openwebui,jina
-
-# --- Fetch ---------------------------------------------------------------
-# Fetch provider implementation. v1 supports only "firecrawl".
-FETCH_PROVIDER=firecrawl
-# Per-URL fetch timeout in seconds.
-FETCH_TIMEOUT_SECONDS=60
-# Max parallel in-flight fetches across all requests.
-FETCH_CONCURRENCY=4
-# Base URL of Firecrawl instance.
-FIRECRAWL_BASE_URL=http://firecrawl-api:3002
-# Optional Firecrawl bearer token.
+# Pretty logs: auto detects TTY, true forces pretty output, false writes JSON. Default: auto
+LOG_PRETTY=auto
+
+# --- Pipeline ------------------------------------------------------------
+# Maximum concurrent source-loading groups. Default: 1
+SOURCE_CONCURRENCY=4
+
+# Maximum concurrent LLM workflow groups. Default: 1
+LLM_CONCURRENCY=2
+
+# Output renderer used by the default pipeline. Default: debug-xml
+DEFAULT_OUTPUT_RENDERER=debug-xml
+
+# Include skipped steps in the debug-xml footer. Default: false
+DEBUG_XML_INCLUDE_SKIPPED=false
+
+# Desired maximum output size used by summarize and truncate steps. Default: 25000
+OUTPUT_TARGET_CHARS=35000
+
+# --- Source provider -----------------------------------------------------
+# Firecrawl base URL used by the default source provider. Required, no default.
+FIRECRAWL_BASE_URL=https://firecrawl.example
+
+# Optional Firecrawl bearer token. Default: empty (no token)
 FIRECRAWL_API_KEY=
-# Ask Firecrawl to strip boilerplate before returning markdown.
-FIRECRAWL_ONLY_MAIN_CONTENT=true
-
-# --- LLM (shared) --------------------------------------------------------
-# LLM provider implementation. v1 supports only "openai_chat".
-LLM_PROVIDER=openai_chat
-# OpenAI-compatible Chat Completions base URL.
-LLM_BASE_URL=http://localhost:8080/v1
-# Optional LLM bearer token.
+
+# --- LLM provider --------------------------------------------------------
+# OpenAI-compatible Chat Completions /v1 base URL. Required, no default.
+LLM_BASE_URL=https://openai-compatible.example/v1
+
+# Optional LLM bearer token. Default: empty (no token)
 LLM_API_KEY=
-# Model identifier sent in chat-completions requests.
-LLM_MODEL=local-model
-# Model context window in tokens for stage eligibility checks.
+
+# Model identifier sent to the chat-completions endpoint. Required, no default.
+LLM_MODEL=model-name
+
+# Model context window in tokens. Default: empty (disables the context-fit gate)
 LLM_CONTEXT_TOKENS=131072
-# Rough chars-per-token estimate.
-LLM_CHARS_PER_TOKEN=4
-# Max parallel per-URL LLM workflows (clean + summarize together).
-LLM_CONCURRENCY=1
-# Extra JSON object merged into every chat-completions request body.
-# Any key wins. Example: {"temperature":0.7, ...}
-LLM_EXTRA_BODY=
-
-# --- Clean stage ---------------------------------------------------------
-# Run a clean pass over fetched markdown.
-CLEAN_ENABLED=true
-# Skip cleanup below this compacted char count.
+
+# Conservative chars-per-token estimate used by the context-fit gate. Default: 3.5
+LLM_CHARS_PER_TOKEN=3.5
+
+# Extra tokens reserved for chat-template framing and estimator drift. Default: 128
+LLM_SAFETY_MARGIN_TOKENS=128
+
+# --- Step thresholds and timeouts ---------------------------------------
+# Timeout in seconds for the source-loading step. Default: 20
+LOAD_SOURCE_TIMEOUT_SECONDS=20
+
+# Timeout in seconds for the clean LLM pass. Default: 60
+CLEAN_TIMEOUT_SECONDS=90
+
+# Minimum body length before the clean LLM pass runs. Default: 1000
 CLEAN_MIN_INPUT_CHARS=500
-# Hard cap on clean input size. 0 means no cap.
-CLEAN_MAX_INPUT_CHARS=75000
-# Output token budget reserved as a fraction of input tokens for context-window eligibility.
-CLEAN_OUTPUT_RATIO=0.5
-# Per-call timeout in seconds.
-CLEAN_TIMEOUT_SECONDS=60
-# Reject output when output/source ratio drops below this threshold.
-CLEAN_QUALITY_MIN_RATIO=0.02
-# Reject clean output that introduces URLs absent from source.
-CLEAN_CHECK_URLS=true
-
-# --- Summarize stage -----------------------------------------------------
-# Run a summarize pass over current best content.
-SUMMARIZE_ENABLED=false
-# Skip summarization below this compacted char count.
-SUMMARIZE_MIN_INPUT_CHARS=25000
-# Hard cap on summarize input size. 0 means no cap.
-SUMMARIZE_MAX_INPUT_CHARS=0
-# Output token budget reserved as a fraction of input tokens for context-window eligibility.
-SUMMARIZE_OUTPUT_RATIO=0.15
-# Per-call timeout in seconds.
+
+# Timeout in seconds for the summarize LLM pass. Default: 60
 SUMMARIZE_TIMEOUT_SECONDS=60
-# Reject output when output/source ratio drops below this threshold.
-SUMMARIZE_QUALITY_MIN_RATIO=0.01
-# Reject summarize output that introduces URLs absent from source.
-SUMMARIZE_CHECK_URLS=false
-
-# --- Truncate stage ------------------------------------------------------
-# Hard cap on final response body length in characters.
-TRUNCATE_TARGET_CHARS=25000
-
-# --- Templates -----------------------------------------------------------
-# Directory containing the Mustache templates below.
-TEMPLATE_DIR=./templates
-# Clean stage prompts (relative to TEMPLATE_DIR).
-CLEAN_SYSTEM_TEMPLATE=clean.system.md
-CLEAN_USER_TEMPLATE=clean.user.md
-# Summarize stage prompts (relative to TEMPLATE_DIR).
-SUMMARIZE_SYSTEM_TEMPLATE=summarize.system.md
-SUMMARIZE_USER_TEMPLATE=summarize.user.md
-# XML diagnostic footer template (relative to TEMPLATE_DIR).
-FOOTER_TEMPLATE=footer.md
+
+# --- Inbound auth --------------------------------------------------------
+# Open WebUI adapter bearer token. Default: empty (leaves POST / open)
+OWUI_AUTH_TOKEN=
+
+# Jina-style adapter bearer token. Default: empty (leaves GET /r open)
+JINA_AUTH_TOKEN=

.gitignore

@@ -1,12 +1,16 @@
-node_modules/
+# Dependencies and build output
+node_modules/
 dist/
 coverage/
 *.tsbuildinfo
 
+# Local environment files. Keep .env.example tracked.
 .env
 .env.local
 .env.*.local
+!.env.example
 
+# Logs and OS noise
 *.log
 .DS_Store
 

.prettierrc.json

@@ -0,0 +1,14 @@
+{
+  "printWidth": 120,
+  "tabWidth": 2,
+  "useTabs": false,
+  "semi": true,
+  "singleQuote": false,
+  "quoteProps": "as-needed",
+  "trailingComma": "none",
+  "bracketSpacing": true,
+  "bracketSameLine": false,
+  "arrowParens": "avoid",
+  "endOfLine": "lf",
+  "proseWrap": "preserve"
+}

.vscode/extensions.json

@@ -1,6 +1,7 @@
 {
   "recommendations": [
     "editorconfig.editorconfig",
+    "esbenp.prettier-vscode",
     "vitest.explorer",
     "ms-azuretools.vscode-docker",
     "github.vscode-github-actions",

.vscode/settings.json

@@ -1,6 +1,8 @@
 {
   "editor.formatOnSave": true,
+  "editor.defaultFormatter": "esbenp.prettier-vscode",
   "js/ts.tsdk.path": "node_modules/typescript/lib",
+  "vitest.cliArguments": "--project unit",
   "search.exclude": {
     "**/node_modules": true,
     "**/dist": true,
@@ -13,4 +15,4 @@
     "**/dist/**": true,
     "**/coverage/**": true
   }
-}
+}