feat(api): comprehensive Virtual Meta-Server with 12 meta-tools

Implements the Virtual Meta-Server feature (IBM#2230) — a tool aggregation
layer that enables AI agents to discover and invoke thousands of underlying
tools through a unified interface.

Meta-tools:
- search_tools: hybrid semantic + keyword search with scope filtering
- list_tools: paginated tool listing with sorting and filtering
- describe_tool: detailed tool info with schema and metadata
- execute_tool: tool execution with JSON schema validation and routing
- get_tool_categories: aggregated categories with counts
- get_similar_tools: vector similarity search for related tools
- authorize_gateway: interactive OAuth authorization with token refresh
- authorize_all_gateways: one-click authorization for all OAuth gateways
- list_resources: paginated MCP resource listing
- read_resource: read MCP resource content by URI
- list_prompts: paginated MCP prompt listing
- get_prompt: prompt template retrieval with optional rendering

Features:
- OAuth integration: propagates user identity through the call chain
- Chained OAuth flow: authorize-all endpoint chains multiple gateways
- camelCase normalization for MCP clients
- Flat argument tolerance for Copilot Studio
- Post-login redirect via cookie with safe path validation
- Observability: prompt.render and resource.read spans
- JSON serialization: orjson.dumps() for proper JSON output
- Admin UI: meta-server checkbox and hide-underlying-tools in server forms
- Preserves protect_oauth_config_for_storage() on server update
- RBAC enforcement via middleware on all meta endpoints

Closes IBM#2230
Supersedes IBM#3653

Signed-off-by: Olivier Gintrand <olivier.gintrand@forterro.com>

Author: Olivier Gintrand

.env.example

@@ -2136,6 +2136,11 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
 # TOOL_CONCURRENT_LIMIT=10
 # GATEWAY_TOOL_NAME_SEPARATOR=-
 
+# Maximum length of response text returned for non-JSON REST API responses
+# Longer responses are truncated to prevent exposing excessive sensitive data
+# Default: 5000 characters, Range: 1000-100000
+# REST_RESPONSE_TEXT_MAX_LENGTH=5000
+
 # Prompt Configuration
 # PROMPT_CACHE_SIZE=100
 # MAX_PROMPT_SIZE=102400

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline|package-lock.json|Cargo.lock|scripts/sign_image.sh|scripts/zap|sonar-project.properties|uv.lock|go.sum|mcpgateway/sri_hashes.json|^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2026-04-14T13:09:46Z",
+  "generated_at": "2026-04-14T14:08:10Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -4830,7 +4830,7 @@
         "hashed_secret": "ff37a98a9963d347e9749a5c1b3936a4a245a6ff",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 2228,
+        "line_number": 2236,
         "type": "Secret Keyword",
         "verified_result": null
       }
@@ -8624,39 +8624,39 @@
         "hashed_secret": "ee977806d7286510da8b9a7492ba58e2484c0ecc",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 6376,
+        "line_number": 6907,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "f2e7745f43b0ef0e2c2faf61d6c6a28be2965750",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 6868,
+        "line_number": 7399,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "4a249743d4d2241bd2ae085b4fe654d089488295",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 8215,
+        "line_number": 8746,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "0c8d051d3c7eada5d31b53d9936fce6bcc232ae2",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 8357,
+        "line_number": 8888,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "f2b14f68eb995facb3a1c35287b778d5bd785511",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 8733,
+        "line_number": 9264,
         "type": "Secret Keyword",
         "verified_result": null
       }

mcpgateway/admin.py

@@ -1406,6 +1406,50 @@ def validate_password_strength(password: str) -> tuple[bool, str]:
 ADMIN_CSRF_FORM_FIELD = "csrf_token"
 
 
+def _resolve_root_path(request: Request) -> str:
+    """Resolve the application root path from the request scope with fallback.
+
+    Some embedded/proxy deployments do not populate ``scope["root_path"]``
+    consistently.  This helper checks the ASGI scope first and falls back
+    to ``settings.app_root_path`` when the scope value is empty.
+
+    Args:
+        request: Incoming request used to read ASGI ``root_path``.
+
+    Returns:
+        Normalized root path (leading ``/``, no trailing ``/``), or empty
+        string when no root path is configured.
+    """
+    root_path = request.scope.get("root_path", "") or ""
+    if not root_path or not str(root_path).strip():
+        root_path = settings.app_root_path or ""
+    root_path = str(root_path).strip()
+    if root_path:
+        root_path = "/" + root_path.lstrip("/")
+    return root_path.rstrip("/")
+
+
+def _is_safe_local_path(path: str) -> bool:
+    """Validate that a path is a safe local redirect target (no open redirect).
+
+    Args:
+        path: The path to validate.
+
+    Returns:
+        True if the path is a safe relative path starting with ``/``.
+    """
+    if not path or not isinstance(path, str):
+        return False
+    if not path.startswith("/"):
+        return False
+    # Block protocol-relative URLs (//evil.com), authority injection (@), backslash tricks
+    if path.startswith("//") or "@" in path or "\\" in path:
+        return False
+    parsed = urllib.parse.urlparse(path)
+    if parsed.scheme or parsed.netloc:
+        return False
+    return True
+
 def _admin_cookie_path(request: Request) -> str:
     """Build admin cookie path honoring ASGI root_path.
 
@@ -2978,6 +3022,8 @@ async def admin_add_server(request: Request, db: Session = Depends(get_db), user
             visibility=visibility,
             oauth_enabled=oauth_enabled,
             oauth_config=oauth_config,
+            server_type="meta" if form.get("meta_server_enabled") else str(form.get("server_type", "standard")),
+            hide_underlying_tools=form.get("hide_underlying_tools") == "true" or form.get("hide_underlying_tools") == "on",
         )
     except KeyError as e:
         # Convert KeyError to ValidationError-like response
@@ -3141,6 +3187,8 @@ async def admin_edit_server(
             owner_email=user_email,
             oauth_enabled=oauth_enabled,
             oauth_config=oauth_config,
+            server_type="meta" if form.get("meta_server_enabled") else str(form.get("server_type", "standard")),
+            hide_underlying_tools=form.get("hide_underlying_tools") == "true" or form.get("hide_underlying_tools") == "on",
         )
 
         await server_service.update_server(
@@ -4211,6 +4259,21 @@ async def admin_login_page(request: Request) -> Response:
         response.delete_cookie("jwt_token", path="/")
         response.delete_cookie("access_token", path="/")
 
+    # Preserve ?next= parameter as a short-lived cookie so SSO callback can redirect
+    # back to the original URL (e.g. /oauth/authorize/{gateway_id}) after login.
+    next_url = request.query_params.get("next", "")
+    if next_url and _is_safe_local_path(next_url):
+        use_secure = (settings.environment == "production") or settings.secure_cookies
+        response.set_cookie(
+            key="post_login_next",
+            value=next_url,
+            max_age=300,  # 5 minutes — enough for SSO round-trip
+            httponly=True,
+            secure=use_secure,
+            samesite=settings.cookie_samesite,
+            path=settings.app_root_path or "/",
+        )
+
     return response
 
 

mcpgateway/admin_ui/formSubmitHandlers.js

@@ -438,6 +438,17 @@ export const handleServerFormSubmit = async function (e) {
       }
     }
 
+    // Handle Meta-Server configuration
+    const metaEnabledCheckbox = safeGetElement("server-meta-enabled");
+    if (metaEnabledCheckbox && metaEnabledCheckbox.checked) {
+      formData.set("server_type", "meta");
+      const hideToolsCheckbox = safeGetElement("server-hide-underlying-tools");
+      formData.set("hide_underlying_tools", hideToolsCheckbox && hideToolsCheckbox.checked ? "true" : "false");
+    } else {
+      formData.set("server_type", "standard");
+      formData.delete("hide_underlying_tools");
+    }
+
     const response = await fetch(`${window.ROOT_PATH}/admin/servers`, {
       method: "POST",
       body: formData,
@@ -1008,6 +1019,17 @@ export const handleEditServerFormSubmit = async function (e) {
       }
     });
 
+    // Handle Meta-Server configuration
+    const metaEnabledCheckbox = safeGetElement("edit-server-meta-enabled");
+    if (metaEnabledCheckbox && metaEnabledCheckbox.checked) {
+      formData.set("server_type", "meta");
+      const hideToolsCheckbox = safeGetElement("edit-server-hide-underlying-tools");
+      formData.set("hide_underlying_tools", hideToolsCheckbox && hideToolsCheckbox.checked ? "true" : "false");
+    } else {
+      formData.set("server_type", "standard");
+      formData.delete("hide_underlying_tools");
+    }
+
     // Submit via fetch
     const response = await fetch(form.action, {
       method: "POST",

mcpgateway/admin_ui/servers.js

@@ -864,6 +864,29 @@ export const editServer = async function (serverId) {
       if (oauthTokenEndpointField) oauthTokenEndpointField.value = "";
     }
 
+    // Set Meta-Server configuration fields
+    const metaEnabledCheckbox = safeGetElement("edit-server-meta-enabled");
+    const metaConfigSection = safeGetElement("edit-server-meta-config-section");
+    const hideUnderlyingToolsCheckbox = safeGetElement("edit-server-hide-underlying-tools");
+    const isMeta = server.serverType === "meta" || server.server_type === "meta";
+
+    if (metaEnabledCheckbox) {
+      metaEnabledCheckbox.checked = isMeta;
+    }
+    if (metaConfigSection) {
+      if (isMeta) {
+        metaConfigSection.classList.remove("hidden");
+      } else {
+        metaConfigSection.classList.add("hidden");
+      }
+    }
+    if (hideUnderlyingToolsCheckbox) {
+      const hideTools = server.hideUnderlyingTools !== undefined
+        ? server.hideUnderlyingTools
+        : (server.hide_underlying_tools !== undefined ? server.hide_underlying_tools : true);
+      hideUnderlyingToolsCheckbox.checked = isMeta ? hideTools : true;
+    }
+
     // Store server data for modal population
     window.Admin.currentEditingServer = server;
 

mcpgateway/alembic/versions/5126ced48fd0_add_meta_server_fields.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+"""Add meta-server fields to servers table
+
+Revision ID: 5126ced48fd0
+Revises: 64acf94cb7f2
+Create Date: 2026-02-12 10:00:00.000000
+
+"""
+
+# Third-Party
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "5126ced48fd0"
+down_revision = "64acf94cb7f2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    """Add server_type, hide_underlying_tools, meta_config, and meta_scope columns to servers."""
+    inspector = sa.inspect(op.get_bind())
+
+    # Skip if table doesn't exist (fresh DB uses db.py models directly)
+    if "servers" not in inspector.get_table_names():
+        return
+
+    columns = [col["name"] for col in inspector.get_columns("servers")]
+
+    if "server_type" not in columns:
+        op.add_column("servers", sa.Column("server_type", sa.String(20), nullable=False, server_default="standard"))
+
+    if "hide_underlying_tools" not in columns:
+        op.add_column("servers", sa.Column("hide_underlying_tools", sa.Boolean(), nullable=False, server_default=sa.text("true")))
+
+    if "meta_config" not in columns:
+        op.add_column("servers", sa.Column("meta_config", sa.JSON(), nullable=True))
+
+    if "meta_scope" not in columns:
+        op.add_column("servers", sa.Column("meta_scope", sa.JSON(), nullable=True))
+
+
+def downgrade() -> None:
+    """Remove meta-server fields from servers table."""
+    inspector = sa.inspect(op.get_bind())
+
+    if "servers" not in inspector.get_table_names():
+        return
+
+    columns = [col["name"] for col in inspector.get_columns("servers")]
+
+    if "meta_scope" in columns:
+        op.drop_column("servers", "meta_scope")
+    if "meta_config" in columns:
+        op.drop_column("servers", "meta_config")
+    if "hide_underlying_tools" in columns:
+        op.drop_column("servers", "hide_underlying_tools")
+    if "server_type" in columns:
+        op.drop_column("servers", "server_type")

mcpgateway/common/validators.py

@@ -50,13 +50,12 @@
 # Standard
 from html.parser import HTMLParser
 import ipaddress
-import json
 import logging
 from pathlib import Path
 import re
 import shlex
 import socket
-from typing import Any, Dict, Iterable, List, Optional, Pattern
+from typing import Any, Iterable, List, Optional, Pattern
 from urllib.parse import urlparse
 import uuid
 
@@ -77,7 +76,9 @@
 _HTML_SPECIAL_CHARS_RE: Pattern[str] = re.compile(r'[<>"\']')  # / removed per SEP-986
 _DANGEROUS_TEMPLATE_TAGS_RE: Pattern[str] = re.compile(r"<(script|iframe|object|embed|link|meta|base|form)\b", re.IGNORECASE)
 _EVENT_HANDLER_RE: Pattern[str] = re.compile(r"on\w+\s*=", re.IGNORECASE)
-_MIME_TYPE_RE: Pattern[str] = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9!#$&\-\^_+\.]*\/[a-zA-Z0-9][a-zA-Z0-9!#$&\-\^_+\.]*(?:\s*;\s*[a-zA-Z0-9!#$&\-\^_+\.]+=(?:[a-zA-Z0-9!#$&\-\^_+\.]+|"[^"\r\n]*"))*$')
+_MIME_TYPE_RE: Pattern[str] = re.compile(
+    r'^[a-zA-Z0-9][a-zA-Z0-9!#$&\-\^_+\.]*\/[a-zA-Z0-9][a-zA-Z0-9!#$&\-\^_+\.]*(?:\s*;\s*[a-zA-Z0-9!#$&\-\^_+\.]+=(?:[a-zA-Z0-9!#$&\-\^_+\.]+|"[^"\r\n]*"))*$'
+)
 _URI_SCHEME_RE: Pattern[str] = re.compile(r"^[a-zA-Z][a-zA-Z0-9+\-.]*://")
 _SHELL_DANGEROUS_CHARS_RE: Pattern[str] = re.compile(r"[;&|`$(){}\[\]<>]")
 _ANSI_ESCAPE_RE: Pattern[str] = re.compile(r"\x1B\[[0-9;]*[A-Za-z]")
@@ -1814,60 +1815,3 @@ def validate_core_url(value: str, field_name: str = "URL") -> str:
         The validated URL string.
     """
     return SecurityValidator.validate_url(value, field_name)
-
-
-# CWE-400: Limits for user-supplied meta_data forwarded to upstream MCP servers.
-# Keeps arbitrarily large dicts from amplifying into downstream network/DB load.
-# These are now read from config (settings.meta_max_keys, etc.) but kept as
-# module-level aliases for backward-compatible imports.
-META_MAX_KEYS: int = settings.meta_max_keys
-META_MAX_DEPTH: int = settings.meta_max_depth
-META_MAX_BYTES: int = settings.meta_max_bytes
-
-
-def validate_meta_data(meta_data: Optional[Dict[str, Any]]) -> None:
-    """Enforce size, key-count, and depth limits on user-supplied meta_data (CWE-400).
-
-    Args:
-        meta_data: The metadata dictionary to validate. ``None`` is always accepted.
-
-    Raises:
-        ValueError: if any limit is exceeded.
-    """
-    max_keys = settings.meta_max_keys
-    max_depth = settings.meta_max_depth
-    max_bytes = settings.meta_max_bytes
-
-    if not meta_data:
-        return
-    if len(meta_data) > max_keys:
-        raise ValueError(f"meta_data exceeds maximum key count ({max_keys}): got {len(meta_data)}")
-
-    def _check_depth(obj: Any, depth: int) -> None:
-        """Recursively enforce nesting depth, traversing both dicts and lists (CWE-400).
-
-        Lists are traversed without incrementing the depth counter so that a
-        list-of-dicts does not hide an extra level of dict nesting — e.g.
-        ``{"k": [{"l2": {"l3": "x"}}]}`` is correctly caught as depth 3.
-        """
-        if depth > max_depth:
-            raise ValueError(f"meta_data exceeds maximum nesting depth ({max_depth})")
-        if isinstance(obj, dict):
-            for v in obj.values():
-                _check_depth(v, depth + 1)
-        elif isinstance(obj, list):
-            for item in obj:
-                _check_depth(item, depth)
-
-    for v in meta_data.values():
-        _check_depth(v, 1)
-
-    try:
-        # CWE-20: Use strict json.dumps (no default=str) so non-serializable objects
-        # raise TypeError rather than being silently coerced — keeps the byte limit
-        # meaningful and matches the strict rejection behaviour used in prompt_service.
-        size = len(json.dumps(meta_data))
-        if size > max_bytes:
-            raise ValueError(f"meta_data exceeds maximum size ({max_bytes} bytes): got {size}")
-    except (TypeError, ValueError) as exc:
-        raise ValueError(f"meta_data is not serializable: {exc}") from exc

mcpgateway/config.py

@@ -360,9 +360,6 @@ class Settings(BaseSettings):
     allowed_roots: List[str] = Field(default_factory=list, description="Allowed root paths for resource access")
     max_path_depth: int = Field(default=10, description="Maximum allowed path depth")
     max_param_length: int = Field(default=10000, description="Maximum parameter length")
-    meta_max_keys: int = Field(default=16, description="Maximum number of keys in user-supplied meta_data forwarded to upstream MCP servers (CWE-400)")
-    meta_max_depth: int = Field(default=2, description="Maximum nesting depth for user-supplied meta_data forwarded to upstream MCP servers (CWE-400)")
-    meta_max_bytes: int = Field(default=4096, description="Maximum JSON-encoded byte size for user-supplied meta_data forwarded to upstream MCP servers (CWE-400)")
     dangerous_patterns: List[str] = Field(
         default_factory=lambda: [
             r"[;&|`$(){}\[\]<>]",  # Shell metacharacters
@@ -1603,6 +1600,7 @@ def parse_issuers(cls, v: Any) -> list[str]:
     max_tool_retries: int = 3
     tool_rate_limit: int = 100  # requests per minute
     tool_concurrent_limit: int = 10
+    semantic_search_rate_limit: int = 30  # requests per minute for semantic search
 
     # Content Security - Size Limits
     content_max_resource_size: int = Field(default=102400, ge=1024, le=10485760, description="Maximum size in bytes for resource content (default: 100KB)")  # 100KB  # Minimum 1KB  # Maximum 10MB