feat!: signal-driven conditional pipeline with single full/smoke config (#4)

* feat: conditional step execution with URL classification and fallback routing

Add classify-url pipeline step, runIf/skipIf gates, and fallback load-source routing.

- src/contracts/pipeline/condition.ts — recursive Condition type (string leaf, all/any/not combinators)
- src/core/pipeline/conditions.ts — evaluateCondition, resolveStepGate, isTruthySignal
- src/core/pipeline/compiled.ts — runIf/skipIf on CompiledPipelineStep
- src/core/pipeline/orchestrator.ts — gate evaluation before step execution
- src/builtins/pipeline-steps/classify-url/ — config, step, and descriptor
- src/config/yaml/yaml-config.ts — conditionSchema with diagnostic-name validation
- src/engine/engine-config.ts — runIf/skipIf on EngineStepConfig
- src/shared/diagnostic-names.ts — isDiagnosticName helper
- config/llm-context-loader.yaml — clean-combined rewired with classify → docling (runIf:binary_doc) → firecrawl (fallback) → clean (skipIf:code_host)
- docs: ARCHITECTURE.md, CUSTOMIZATION.md, ROADMAP.md updated
- 69 new focused tests; 388 total passing

* feat: compile-time step disable, full/smoke pipeline split, env cleanup

* feat(docling): opaque passthrough options with JSON-string coercion and standalone docs

* feat(firecrawl): opaque passthrough options with PDF sniffing

---------

Co-authored-by: David Soušek <15769039+sousekd@users.noreply.github.com>

Author: sousekd

.env.example

@@ -22,8 +22,8 @@ LOG_LEVEL=info
 LOG_PRETTY=auto
 
 # --- Pipeline ------------------------------------------------------------
-# Active pipeline selected by HTTP adapters. Default: truncate (load-source + truncate)
-DEFAULT_PIPELINE=clean-combined
+# Active pipeline selected by HTTP adapters. Default: full (full-featured; external deps disabled by default).
+DEFAULT_PIPELINE=full
 
 # Maximum concurrent source-loading groups. Default: 1; raise only if providers can handle it.
 SOURCE_CONCURRENCY=5
@@ -44,28 +44,37 @@ DEBUG_XML_INCLUDE_SKIPPED=false
 OUTPUT_TARGET_CHARS=35000
 
 # --- Source provider -----------------------------------------------------
-# Optional source-provider override. Leave empty to use the selected pipeline's fallback.
-SOURCE_PROVIDER=
+# Enable Firecrawl source loading. Disabled by default. Default: false
+FIRECRAWL_ENABLED=false
 
-# Firecrawl base URL. Required only when a referenced provider uses it (e.g. firecrawl-markdown, firecrawl-html).
+# Firecrawl base URL. Required only when FIRECRAWL_ENABLED=true.
 FIRECRAWL_BASE_URL=https://firecrawl.example
 
 # Optional Firecrawl bearer token. Default: empty (no token)
 FIRECRAWL_API_KEY=
 
-# Firecrawl PDF parsing. Set to false to return raw PDF bytes (binary body) instead of parsed text.
-FIRECRAWL_PARSE_PDF=
+# Opaque Firecrawl scrape options, keeping real API parameter names.
+FIRECRAWL_OPTIONS=
 
-# Docling Serve base URL. Required only when a referenced provider uses it (e.g. docling-ocr).
+# Enable Docling OCR source loading for binary documents. Disabled by default. Default: false
+DOCLING_ENABLED=false
+
+# Docling Serve base URL. Required only when DOCLING_ENABLED=true.
 DOCLING_BASE_URL=https://docling.example
 
 # Optional Docling API key (X-Api-Key header). Default: empty (no token)
 DOCLING_API_KEY=
 
+# Opaque JSON object of Docling convert options, merged into the convert body.
+DOCLING_OPTIONS=
+
 # --- Content transformers ------------------------------------------------
 # Post-conversion link and whitespace cleanup for mdream-convert. Ignored when minimal=true (mdream-aggressive). Default: true
 MDREAM_CLEAN=true
 
+# Enable Readability HTML-to-article extraction. Enabled by default (local; no external service needed). Default: true
+READABILITY_ENABLED=true
+
 # Minimum content length for Readability's isProbablyReaderable gate. Default: 140
 READABILITY_MIN_CONTENT_CHARS=140
 
@@ -76,7 +85,13 @@ READABILITY_MIN_SCORE=20
 READABILITY_MAX_ELEMENTS=0
 
 # --- LLM provider --------------------------------------------------------
-# OpenAI-compatible Chat Completions /v1 base URL. Required only when a referenced provider uses it (e.g. llm-default).
+# Enable the clean LLM pass. Disabled by default. Default: false
+LLM_CLEAN_ENABLED=false
+
+# Enable the summarize LLM pass. Disabled by default. Default: false
+LLM_SUMMARIZE_ENABLED=false
+
+# OpenAI-compatible Chat Completions /v1 base URL. Required only when LLM_CLEAN_ENABLED or LLM_SUMMARIZE_ENABLED.
 LLM_BASE_URL=https://openai-compatible.example/v1
 
 # Optional LLM bearer token. Default: empty (no token)
@@ -95,17 +110,23 @@ LLM_CHARS_PER_TOKEN=3.5
 LLM_SAFETY_MARGIN_TOKENS=128
 
 # --- Step thresholds and timeouts ---------------------------------------
-# Timeout in seconds for the source-loading step. Default: 20
-LOAD_SOURCE_TIMEOUT_SECONDS=20
+# Timeout in seconds for the native HTTP source-loading step. Default: 20
+FETCH_TIMEOUT_SECONDS=20
+
+# Timeout in seconds for the Firecrawl source-loading step. Default: 20
+FIRECRAWL_TIMEOUT_SECONDS=20
+
+# Timeout in seconds for the Docling OCR source-loading step. Default: 60; OCR is slower.
+DOCLING_TIMEOUT_SECONDS=60
 
 # Timeout in seconds for the clean LLM pass. Default: 60; raise for slower models.
-CLEAN_TIMEOUT_SECONDS=90
+LLM_CLEAN_TIMEOUT_SECONDS=90
 
 # Minimum body length before the clean LLM pass runs. Default: 1000; lower values run cleanup more often.
-CLEAN_MIN_INPUT_CHARS=500
+LLM_CLEAN_MIN_INPUT_CHARS=500
 
 # Timeout in seconds for the summarize LLM pass. Default: 60
-SUMMARIZE_TIMEOUT_SECONDS=60
+LLM_SUMMARIZE_TIMEOUT_SECONDS=60
 
 # --- Inbound auth --------------------------------------------------------
 # Open WebUI adapter bearer token. Default: empty (leaves POST / open)

README.md

@@ -11,7 +11,7 @@ LLM Context Loader is a small HTTP service that turns URLs into markdown for LLM
 
 ## Current Shape
 
-- **Pipelines:** `truncate` (default, no LLM), `clean-deterministic` (Firecrawl HTML + Readability + mdream, no LLM), `clean-llm` (Firecrawl + LLM), and `clean-combined` (Firecrawl HTML + Readability + mdream + LLM summarize), selected via `DEFAULT_PIPELINE`.
+- **Pipelines:** `full` (default; full-featured with toggleable Docling, Firecrawl, Readability, and LLM passes) and `smoke` (zero-dependency HTTP fetch + aggressive markdown + truncation), selected via `DEFAULT_PIPELINE`.
 - **Source providers:** native HTTP fetch, Firecrawl, Docling.
 - **Content transformers:** `readability` (article HTML extraction), `mdream` (HTML to markdown).
 - **LLM providers:** OpenAI-compatible `/chat/completions`.
@@ -64,7 +64,7 @@ docker compose -f compose.deploy.yaml up -d
 
 The deploy compose file requires `LLMC_IMAGE_TAG`. The example env file uses `latest`, but repeatable deployments should pin it to an immutable release tag.
 
-The out-of-box `truncate` pipeline needs no provider config. Switch to `clean-llm` with `DEFAULT_PIPELINE=clean-llm` and the appropriate provider vars. The Compose files pass all variables through without failing early; the service validates only the active pipeline's providers at startup. If Firecrawl or the LLM server runs on the host, use a LAN address or `host.docker.internal` instead of `localhost`.
+The `smoke` pipeline needs no provider config — it uses native HTTP fetch only. The Compose files pass all variables through without failing early; the service validates only the active pipeline's providers at startup. If Firecrawl or the LLM server runs on the host, use a LAN address or `host.docker.internal` instead of `localhost`.
 
 ## API
 

compose.deploy.yaml

@@ -11,7 +11,7 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l
   LOG_PRETTY: "${LOG_PRETTY:-auto}"
 
   # Active pipeline selected by HTTP adapters.
-  DEFAULT_PIPELINE: "${DEFAULT_PIPELINE:-truncate}"
+  DEFAULT_PIPELINE: "${DEFAULT_PIPELINE:-full}"
 
   # Pipeline concurrency and renderer selection.
   SOURCE_CONCURRENCY: "${SOURCE_CONCURRENCY:-1}"
@@ -21,13 +21,20 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l
   DEBUG_XML_INCLUDE_SKIPPED: "${DEBUG_XML_INCLUDE_SKIPPED:-false}"
   OUTPUT_TARGET_CHARS: "${OUTPUT_TARGET_CHARS:-25000}"
 
+  # Step toggles (disabled by default).
+  DOCLING_ENABLED: "${DOCLING_ENABLED:-false}"
+  FIRECRAWL_ENABLED: "${FIRECRAWL_ENABLED:-false}"
+  READABILITY_ENABLED: "${READABILITY_ENABLED:-true}"
+  LLM_CLEAN_ENABLED: "${LLM_CLEAN_ENABLED:-false}"
+  LLM_SUMMARIZE_ENABLED: "${LLM_SUMMARIZE_ENABLED:-false}"
+
   # Source provider.
-  SOURCE_PROVIDER: "${SOURCE_PROVIDER:-}"
   FIRECRAWL_BASE_URL: "${FIRECRAWL_BASE_URL:-}"
   FIRECRAWL_API_KEY: "${FIRECRAWL_API_KEY:-}"
-  FIRECRAWL_PARSE_PDF: "${FIRECRAWL_PARSE_PDF:-}"
+  FIRECRAWL_OPTIONS: "${FIRECRAWL_OPTIONS:-}"
   DOCLING_BASE_URL: "${DOCLING_BASE_URL:-}"
   DOCLING_API_KEY: "${DOCLING_API_KEY:-}"
+  DOCLING_OPTIONS: "${DOCLING_OPTIONS:-}"
 
   # Content transformers.
   MDREAM_CLEAN: "${MDREAM_CLEAN:-true}"
@@ -44,10 +51,12 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l
   LLM_SAFETY_MARGIN_TOKENS: "${LLM_SAFETY_MARGIN_TOKENS:-128}"
 
   # Step thresholds and timeouts.
-  LOAD_SOURCE_TIMEOUT_SECONDS: "${LOAD_SOURCE_TIMEOUT_SECONDS:-20}"
-  CLEAN_TIMEOUT_SECONDS: "${CLEAN_TIMEOUT_SECONDS:-60}"
-  CLEAN_MIN_INPUT_CHARS: "${CLEAN_MIN_INPUT_CHARS:-1000}"
-  SUMMARIZE_TIMEOUT_SECONDS: "${SUMMARIZE_TIMEOUT_SECONDS:-60}"
+  FETCH_TIMEOUT_SECONDS: "${FETCH_TIMEOUT_SECONDS:-20}"
+  FIRECRAWL_TIMEOUT_SECONDS: "${FIRECRAWL_TIMEOUT_SECONDS:-20}"
+  DOCLING_TIMEOUT_SECONDS: "${DOCLING_TIMEOUT_SECONDS:-60}"
+  LLM_CLEAN_TIMEOUT_SECONDS: "${LLM_CLEAN_TIMEOUT_SECONDS:-60}"
+  LLM_CLEAN_MIN_INPUT_CHARS: "${LLM_CLEAN_MIN_INPUT_CHARS:-1000}"
+  LLM_SUMMARIZE_TIMEOUT_SECONDS: "${LLM_SUMMARIZE_TIMEOUT_SECONDS:-60}"
 
   # Per-adapter inbound bearer tokens. Empty means open route.
   OWUI_AUTH_TOKEN: "${OWUI_AUTH_TOKEN:-}"

compose.yaml

@@ -11,7 +11,7 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l
   LOG_PRETTY: "${LOG_PRETTY:-auto}"
 
   # Active pipeline selected by HTTP adapters.
-  DEFAULT_PIPELINE: "${DEFAULT_PIPELINE:-truncate}"
+  DEFAULT_PIPELINE: "${DEFAULT_PIPELINE:-full}"
 
   # Pipeline concurrency and renderer selection.
   SOURCE_CONCURRENCY: "${SOURCE_CONCURRENCY:-1}"
@@ -21,13 +21,20 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l
   DEBUG_XML_INCLUDE_SKIPPED: "${DEBUG_XML_INCLUDE_SKIPPED:-false}"
   OUTPUT_TARGET_CHARS: "${OUTPUT_TARGET_CHARS:-25000}"
 
+  # Step toggles (disabled by default).
+  DOCLING_ENABLED: "${DOCLING_ENABLED:-false}"
+  FIRECRAWL_ENABLED: "${FIRECRAWL_ENABLED:-false}"
+  READABILITY_ENABLED: "${READABILITY_ENABLED:-true}"
+  LLM_CLEAN_ENABLED: "${LLM_CLEAN_ENABLED:-false}"
+  LLM_SUMMARIZE_ENABLED: "${LLM_SUMMARIZE_ENABLED:-false}"
+
   # Source provider.
-  SOURCE_PROVIDER: "${SOURCE_PROVIDER:-}"
   FIRECRAWL_BASE_URL: "${FIRECRAWL_BASE_URL:-}"
   FIRECRAWL_API_KEY: "${FIRECRAWL_API_KEY:-}"
-  FIRECRAWL_PARSE_PDF: "${FIRECRAWL_PARSE_PDF:-}"
+  FIRECRAWL_OPTIONS: "${FIRECRAWL_OPTIONS:-}"
   DOCLING_BASE_URL: "${DOCLING_BASE_URL:-}"
   DOCLING_API_KEY: "${DOCLING_API_KEY:-}"
+  DOCLING_OPTIONS: "${DOCLING_OPTIONS:-}"
 
   # Content transformers.
   MDREAM_CLEAN: "${MDREAM_CLEAN:-true}"
@@ -44,10 +51,12 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l
   LLM_SAFETY_MARGIN_TOKENS: "${LLM_SAFETY_MARGIN_TOKENS:-128}"
 
   # Step thresholds and timeouts.
-  LOAD_SOURCE_TIMEOUT_SECONDS: "${LOAD_SOURCE_TIMEOUT_SECONDS:-20}"
-  CLEAN_TIMEOUT_SECONDS: "${CLEAN_TIMEOUT_SECONDS:-60}"
-  CLEAN_MIN_INPUT_CHARS: "${CLEAN_MIN_INPUT_CHARS:-1000}"
-  SUMMARIZE_TIMEOUT_SECONDS: "${SUMMARIZE_TIMEOUT_SECONDS:-60}"
+  FETCH_TIMEOUT_SECONDS: "${FETCH_TIMEOUT_SECONDS:-20}"
+  FIRECRAWL_TIMEOUT_SECONDS: "${FIRECRAWL_TIMEOUT_SECONDS:-20}"
+  DOCLING_TIMEOUT_SECONDS: "${DOCLING_TIMEOUT_SECONDS:-60}"
+  LLM_CLEAN_TIMEOUT_SECONDS: "${LLM_CLEAN_TIMEOUT_SECONDS:-60}"
+  LLM_CLEAN_MIN_INPUT_CHARS: "${LLM_CLEAN_MIN_INPUT_CHARS:-1000}"
+  LLM_SUMMARIZE_TIMEOUT_SECONDS: "${LLM_SUMMARIZE_TIMEOUT_SECONDS:-60}"
 
   # Per-adapter inbound bearer tokens. Empty means open route.
   OWUI_AUTH_TOKEN: "${OWUI_AUTH_TOKEN:-}"