feat: add experimental Rust MCP runtime and session core

.env.example

@@ -211,6 +211,50 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 # MCP_SESSION_POOL_ENABLED=false
 # ANYIO_CANCEL_DELIVERY_PATCH_ENABLED=false
 
+# Rust MCP (simple)
+# RUST_MCP_BUILD=false                # build the Rust MCP runtime into Containerfile.lite images
+# RUST_MCP_MODE=off                   # off | shadow | edge | full
+# RUST_MCP_LOG=warn                   # default Rust sidecar log filter for the simple mode flow
+#
+# RUST_MCP_MODE=shadow -> Rust sidecar enabled, but public /mcp stays on Python for safe fallback
+# RUST_MCP_MODE=edge   -> direct public /mcp on Rust with managed UDS sidecar defaults
+# RUST_MCP_MODE=full   -> edge + Rust session/event-store/resume/live-stream/affinity cores
+#
+# Advanced Rust MCP overrides
+# RUST_MCP_SESSION_AUTH_REUSE=false         # advanced override for the fast direct public Rust session-auth path; prefer RUST_MCP_MODE presets above
+# EXPERIMENTAL_RUST_MCP_RUNTIME_ENABLED=
+# EXPERIMENTAL_RUST_MCP_RUNTIME_URL=http://127.0.0.1:8787
+# EXPERIMENTAL_RUST_MCP_RUNTIME_UDS=/tmp/contextforge-mcp-rust.sock
+# EXPERIMENTAL_RUST_MCP_RUNTIME_TIMEOUT_SECONDS=30
+# EXPERIMENTAL_RUST_MCP_SESSION_CORE_ENABLED=       # enable Rust-owned MCP session metadata/lifecycle increment
+# EXPERIMENTAL_RUST_MCP_EVENT_STORE_ENABLED=        # enable Rust-owned resumable event-store backend
+# EXPERIMENTAL_RUST_MCP_RESUME_CORE_ENABLED=        # enable Rust-owned public GET /mcp replay/resume path
+# EXPERIMENTAL_RUST_MCP_LIVE_STREAM_CORE_ENABLED=   # enable Rust-owned public GET /mcp live SSE path
+# EXPERIMENTAL_RUST_MCP_AFFINITY_CORE_ENABLED=      # enable Rust-owned session-affinity forwarding path
+# EXPERIMENTAL_RUST_MCP_SESSION_AUTH_REUSE_ENABLED= # enable Rust-owned session-bound auth-context reuse
+# EXPERIMENTAL_RUST_MCP_RUNTIME_MANAGED=            # launcher env, not a Pydantic setting
+# ENABLE_RUST_MCP_RMCP_BUILD=             # container build arg override for rmcp-enabled Rust MCP binary
+# MCP_RUST_USE_RMCP_UPSTREAM_CLIENT=      # runtime override for official rust-sdk upstream tools/call client
+# MCP_RUST_LISTEN_HTTP=127.0.0.1:8787     # runtime env for bundled Rust sidecar
+# MCP_RUST_LISTEN_UDS=/tmp/contextforge-mcp-rust.sock
+# MCP_RUST_SESSION_CORE_ENABLED=          # explicit sidecar env; defaults from EXPERIMENTAL_RUST_MCP_SESSION_CORE_ENABLED
+# MCP_RUST_SESSION_TTL_SECONDS=3600
+# MCP_RUST_EVENT_STORE_ENABLED=           # explicit sidecar env; defaults from EXPERIMENTAL_RUST_MCP_EVENT_STORE_ENABLED
+# MCP_RUST_RESUME_CORE_ENABLED=           # explicit sidecar env; defaults from EXPERIMENTAL_RUST_MCP_RESUME_CORE_ENABLED
+# MCP_RUST_LIVE_STREAM_CORE_ENABLED=      # explicit sidecar env; defaults from EXPERIMENTAL_RUST_MCP_LIVE_STREAM_CORE_ENABLED
+# MCP_RUST_AFFINITY_CORE_ENABLED=         # explicit sidecar env; defaults from EXPERIMENTAL_RUST_MCP_AFFINITY_CORE_ENABLED
+# MCP_RUST_SESSION_AUTH_REUSE_ENABLED=    # explicit sidecar env; defaults from EXPERIMENTAL_RUST_MCP_SESSION_AUTH_REUSE_ENABLED
+# MCP_RUST_SESSION_AUTH_REUSE_TTL_SECONDS=30
+# MCP_RUST_EVENT_STORE_MAX_EVENTS_PER_STREAM=100
+# MCP_RUST_EVENT_STORE_TTL_SECONDS=3600
+# MCP_RUST_EVENT_STORE_POLL_INTERVAL_MS=250
+# MCP_RUST_LOG=                             # advanced runtime log override for the bundled Rust sidecar
+# MCP_RUST_BACKEND_RPC_URL=http://127.0.0.1:4444/_internal/mcp/rpc
+# MCP_RUST_REDIS_URL=redis://redis:6379/0
+# MCP_RUST_CACHE_PREFIX=mcpgw:
+# MCP_RUST_DATABASE_URL=postgresql://postgres:mysecretpassword@pgbouncer:6432/mcp
+# MCP_RUST_DB_POOL_MAX_SIZE=20
+
 # =============================================================================
 # Performance Tuning (quick reference)
 # =============================================================================
@@ -1773,16 +1817,20 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 # =============================================================================
 # Caches authentication data (user, team, revocation) to reduce database queries
 # Uses Redis when available, falls back to in-memory cache
+# Applies to both Python MCP and Rust MCP because public MCP auth still runs in Python first
 
 # Enable Redis/in-memory caching for authentication data (default: true)
 # Significantly reduces database queries during authentication
+# Disabling this also disables the shared auth cache benefit for RUST_MCP_MODE=edge/full
 # AUTH_CACHE_ENABLED=true
 
 # TTL in seconds for cached user data (default: 60, range: 10-300)
+# Also affects MCP Streamable HTTP auth, including Rust-fronted MCP requests
 # AUTH_CACHE_USER_TTL=60
 
 # TTL in seconds for token revocation cache (default: 30, range: 5-120)
 # Security-critical: keep short to limit exposure window for revoked tokens
+# Also affects MCP auth on both Python and Rust runtime modes
 # AUTH_CACHE_REVOCATION_TTL=30
 
 # TTL in seconds for team membership cache (default: 60, range: 10-300)
@@ -1794,6 +1842,7 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 
 # Enable caching for get_user_teams() (default: true)
 # Set to false to disable teams list caching (useful for debugging)
+# Also affects session-token MCP auth on Python and Rust modes
 # AUTH_CACHE_TEAMS_ENABLED=true
 
 # TTL in seconds for user teams list cache (default: 60, range: 10-300)
@@ -1802,6 +1851,7 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 
 # Batch auth DB queries into single call (default: true)
 # Reduces 3 separate queries to 1, improving performance under load
+# Streamable HTTP MCP auth uses this too before falling back to per-query checks
 # AUTH_CACHE_BATCH_QUERIES=true
 
 # Registry Cache Configuration

.gitignore

@@ -298,6 +298,7 @@ TODO.md
 FIXMEs
 
 # Upgrade validation outputs
+artifacts/
 artifacts/upgrade-validation*/
 
 # Debug & profiling artifacts
@@ -354,6 +355,12 @@ docs/docs/test/license-check-report.json
 nginx.conf
 docker-compose.perf.yml
 
+# Rust MCP runtime profiling artifacts
+tools_rust/mcp_runtime/profiles/
+tools_rust/mcp_runtime/flamegraph*.svg
+tools_rust/mcp_runtime/flamegraph*.html
+tools_rust/mcp_runtime/perf.data*
+
 # JMeter test results and local installation
 tests/jmeter/results/*.jtl
 tests/jmeter/results/*/

Containerfile.lite

@@ -20,6 +20,7 @@
 # Python major.minor series to track
 ARG PYTHON_VERSION=3.12
 ARG ENABLE_RUST=false
+ARG ENABLE_RUST_MCP_RMCP=false
 # Enable profiling tools (memray, py-spy) - off by default for smaller images
 # To enable: docker build --build-arg ENABLE_PROFILING=true -f Containerfile.lite .
 # Usage after enabling:
@@ -36,14 +37,19 @@ ARG ENABLE_PROFILING=false
 ###############################################################################
 FROM quay.io/pypa/manylinux2014:2026.03.06-3 AS rust-builder-base
 ARG ENABLE_RUST
+ARG ENABLE_RUST_MCP_RMCP
 
 # Set shell with pipefail for safety
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 # Only build if ENABLE_RUST=true
 RUN if [ "$ENABLE_RUST" != "true" ]; then \
         echo "⏭️  Rust builds disabled (set --build-arg ENABLE_RUST=true to enable)"; \
-        mkdir -p /build/rust-wheels; \
+        mkdir -p /build/rust-wheels /build/tools_rust/mcp_runtime/target/release; \
+        printf '#!/usr/bin/env sh\n' > /build/tools_rust/mcp_runtime/target/release/contextforge-mcp-runtime; \
+        printf 'echo "Rust MCP runtime not built into this image. Rebuild with --build-arg ENABLE_RUST=true." >&2\n' >> /build/tools_rust/mcp_runtime/target/release/contextforge-mcp-runtime; \
+        printf 'exit 1\n' >> /build/tools_rust/mcp_runtime/target/release/contextforge-mcp-runtime; \
+        chmod +x /build/tools_rust/mcp_runtime/target/release/contextforge-mcp-runtime; \
         exit 0; \
     fi
 
@@ -55,8 +61,9 @@ ENV PATH="/root/.cargo/bin:$PATH"
 
 WORKDIR /build
 
-# Copy only Rust plugin files (only if ENABLE_RUST=true)
+# Copy only Rust plugin/runtime files (only if ENABLE_RUST=true)
 COPY plugins_rust/ /build/plugins_rust/
+COPY tools_rust/mcp_runtime/ /build/tools_rust/mcp_runtime/
 
 # Build each Rust plugin independently using Python 3.12 from manylinux image
 RUN if [ "$ENABLE_RUST" = "true" ]; then \
@@ -74,6 +81,21 @@ RUN if [ "$ENABLE_RUST" = "true" ]; then \
         echo "⏭️  Skipping Rust plugin build"; \
     fi
 
+WORKDIR /build/tools_rust/mcp_runtime
+
+# Build the experimental Rust MCP runtime binary (only if ENABLE_RUST=true)
+RUN if [ "$ENABLE_RUST" = "true" ]; then \
+        if [ "$ENABLE_RUST_MCP_RMCP" = "true" ]; then \
+            cargo build --release --features rmcp-upstream-client; \
+        else \
+            cargo build --release; \
+        fi && \
+        cp target/release/contextforge_mcp_runtime target/release/contextforge-mcp-runtime && \
+        echo "✅ Rust MCP runtime built successfully"; \
+    else \
+        echo "⏭️  Skipping Rust MCP runtime build"; \
+    fi
+
 FROM rust-builder-base AS rust-builder
 
 ###########################
@@ -127,6 +149,7 @@ COPY pyproject.toml /app/
 # Copy Rust plugin wheels from rust-builder stage (if any exist)
 # ----------------------------------------------------------------------------
 COPY --from=rust-builder /build/rust-wheels/ /tmp/rust-wheels/
+COPY --from=rust-builder /build/tools_rust/mcp_runtime/target/release/contextforge-mcp-runtime /app/bin/contextforge-mcp-runtime
 
 # ----------------------------------------------------------------------------
 # Create and populate virtual environment
@@ -139,6 +162,7 @@ COPY --from=rust-builder /build/rust-wheels/ /tmp/rust-wheels/
 #  - Remove build caches and build artifacts
 # ----------------------------------------------------------------------------
 ARG ENABLE_RUST=false
+ARG ENABLE_RUST_MCP_RMCP=false
 ARG ENABLE_PROFILING=false
 RUN set -euo pipefail \
     && . /etc/profile.d/use-openssl.sh \
@@ -219,6 +243,8 @@ RUN chown -R 1001:0 /app \
 FROM registry.access.redhat.com/ubi10/ubi-minimal:10.1-1772441549 AS runtime
 
 ARG PYTHON_VERSION=3.12
+ARG ENABLE_RUST=false
+ARG ENABLE_RUST_MCP_RMCP=false
 ARG ENABLE_PROFILING=false
 
 # ----------------------------------------------------------------------------
@@ -285,6 +311,8 @@ COPY --from=builder --chown=1001:0 /app /app
 # - Disable pip version check to reduce startup time
 # ----------------------------------------------------------------------------
 ENV PATH="/app/.venv/bin:${PATH}" \
+    CONTEXTFORGE_ENABLE_RUST_BUILD=${ENABLE_RUST} \
+    CONTEXTFORGE_ENABLE_RUST_MCP_RMCP_BUILD=${ENABLE_RUST_MCP_RMCP} \
     PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
     PYTHONHASHSEED=random \