Merge remote-tracking branch 'origin/main' into unpin-ea1

Author: jland-redhat

README.md

@@ -16,15 +16,44 @@ Our goal is to create a comprehensive platform for **Models as a Service** with
 
 ## 📋 Prerequisites
 
-- **Openshift cluster** (4.19.9+) with kubectl/oc access
+- **OpenShift cluster** (4.19.9+) with kubectl/oc access
+- **Kuadrant v1.4.2+** (ODH) or **RHCL v1.3+** (RHOAI) - **Required for MaaS v0.2.0+**
 - **PostgreSQL database** (for production ODH/RHOAI deployments)
 
-!!! warning "Database Required for Production"
-    MaaS requires a PostgreSQL database for API key management. For production ODH/RHOAI deployments, you must create a Secret with the database connection URL **before** enabling modelsAsService.
+### ⚠️ Important Version Requirements
 
-    See [Database Prerequisites](docs/content/install/prerequisites.md#database-prerequisite) for details.
+#### Kuadrant 1.4.2+ Required (MaaS v0.2.0+)
 
-    Note: The `scripts/deploy.sh` script creates a development PostgreSQL instance automatically.
+**MaaS v0.2.0 and later requires Kuadrant 1.4.2+ (ODH) or RHCL 1.3+ (RHOAI).**
+
+**Why Kuadrant 1.4.2+ is required:**
+
+MaaS v0.2.0 requires the authorization header stripping capability added in Authorino v0.23.1 (shipped with Kuadrant 1.4.2) to protect user credentials from potential exfiltration to model backends.
+
+**Security Context:**
+
+When a user makes an inference request with their OpenShift token or API key, that credential must be validated by Authorino but should NOT be forwarded to model backends (whether internal KServe models or external providers). Kuadrant 1.4.2+ allows Authorino to:
+
+1. Validate the incoming user credential (OpenShift token or MaaS API key)
+2. Strip/replace the Authorization header before forwarding to model backends
+3. Optionally inject model-specific credentials from Kubernetes Secrets (credentialRef) for ExternalModel resources
+
+This prevents credential exfiltration where a malicious or compromised model service could capture and misuse user tokens.
+
+**Migration Notes:**
+
+- The deployment script (`scripts/deploy.sh`) automatically installs Kuadrant 1.4.2 for new deployments
+- For existing deployments, upgrade Kuadrant/RHCL before upgrading to MaaS v0.2.0+
+
+For detailed version compatibility, see [Version Compatibility](docs/content/install/prerequisites.md#version-compatibility).
+
+#### Database Required for Production
+
+MaaS requires a PostgreSQL database for API key management. For production ODH/RHOAI deployments, you must create a Secret with the database connection URL **before** enabling modelsAsService.
+
+See [Database Prerequisites](docs/content/install/prerequisites.md#database-prerequisite) for details.
+
+Note: The `scripts/deploy.sh` script creates a development PostgreSQL instance automatically.
 
 ## 🚀 Quick Start
 

deployment/base/maas-api/policies/auth-policy.yaml

@@ -12,7 +12,9 @@ spec:
       # API key authentication (for sk-oai-* tokens)
       api-keys:
         when:
-          - predicate: request.headers.authorization.startsWith("Bearer sk-oai-")
+          - selector: request.headers.authorization
+            operator: matches
+            value: "^Bearer sk-oai-.*"
         plain:
           selector: request.headers.authorization
         priority: 0
@@ -27,7 +29,9 @@ spec:
       # Validate API key via HTTP callback (only runs for API key auth)
       apiKeyValidation:
         when:
-          - predicate: request.headers.authorization.startsWith("Bearer sk-oai-")
+          - selector: request.headers.authorization
+            operator: matches
+            value: "^Bearer sk-oai-.*"
         http:
           # Placeholder URL - gets patched based on deployment mode:
           #   - Operator mode (ODH/RHOAI): ODH overlay replacement (app-namespace param)
@@ -42,7 +46,9 @@ spec:
       # Check API key is valid (only for API key auth)
       api-key-valid:
         when:
-          - predicate: request.headers.authorization.startsWith("Bearer sk-oai-")
+          - selector: request.headers.authorization
+            operator: matches
+            value: "^Bearer sk-oai-.*"
         patternMatching:
           patterns:
             - selector: auth.metadata.apiKeyValidation.valid
@@ -55,29 +61,31 @@ spec:
           # Username: from API key validation (when API key used)
           X-MaaS-Username:
             when:
-              - predicate: request.headers.authorization.startsWith("Bearer sk-oai-")
+              - selector: request.headers.authorization
+                operator: matches
+                value: "^Bearer sk-oai-.*"
             plain:
               selector: auth.metadata.apiKeyValidation.username
             priority: 0
-          # Username: from OpenShift identity (when OC token used)
+          # Username: from OpenShift identity (fallback for non-API-key tokens)
+          # No 'when' clause - always tries to inject, but priority 1 means API key wins
           X-MaaS-Username-OC:
-            when:
-              - predicate: '!request.headers.authorization.startsWith("Bearer sk-oai-")'
             plain:
               selector: auth.identity.user.username
             key: X-MaaS-Username
             priority: 1
           # Groups: from API key validation as JSON array (when API key used)
           X-MaaS-Group:
             when:
-              - predicate: request.headers.authorization.startsWith("Bearer sk-oai-")
+              - selector: request.headers.authorization
+                operator: matches
+                value: "^Bearer sk-oai-.*"
             plain:
               selector: auth.metadata.apiKeyValidation.groups.@tostr
             priority: 0
-          # Groups: from OpenShift identity as JSON array (when OC token used)
+          # Groups: from OpenShift identity as JSON array (fallback for non-API-key tokens)
+          # No 'when' clause - always tries to inject, but priority 1 means API key wins
           X-MaaS-Group-OC:
-            when:
-              - predicate: '!request.headers.authorization.startsWith("Bearer sk-oai-")'
             plain:
               selector: auth.identity.user.groups.@tostr
             key: X-MaaS-Group
@@ -86,8 +94,12 @@ spec:
           # This header is used by /v1/models to determine which subscription's models to return
           X-MaaS-Subscription:
             when:
-              - predicate: request.headers.authorization.startsWith("Bearer sk-oai-")
-              - predicate: auth.metadata.apiKeyValidation.subscription != ""
+              - selector: request.headers.authorization
+                operator: matches
+                value: "^Bearer sk-oai-.*"
+              - selector: auth.metadata.apiKeyValidation.subscription
+                operator: neq
+                value: ""
             plain:
               selector: auth.metadata.apiKeyValidation.subscription
             priority: 0

deployment/base/observability/gateway-telemetry-policy.yaml

@@ -12,8 +12,8 @@ spec:
         user: auth.identity.userid
         # Subscription metadata for usage attribution and billing
         subscription: auth.identity.selected_subscription
-        organization_id: auth.identity.organizationId
-        cost_center: auth.identity.costCenter
+        organization_id: auth.identity.subscription_info.organizationId
+        cost_center: auth.identity.subscription_info.costCenter
   targetRef:
     group: gateway.networking.k8s.io
     kind: Gateway

docs/content/architecture.md

@@ -214,7 +214,7 @@ The MaaSAuthPolicy delegates to the MaaS API for key validation and subscription
 2. MaaS API validates the key (format, not revoked, not expired) and returns username, groups, and subscription.
 3. Authorino calls MaaS API to check subscription (groups, username, requested subscription from the key).
 4. If the user lacks access to the requested subscription → error (403).
-5. On success, returns selected subscription; Authorino caches the result (e.g., 60s TTL). AuthPolicy may inject `X-MaaS-Subscription` **server-side** for downstream rate limiting and metrics. Clients do not send this header on inference; subscription comes from the API key record created at mint time.
+5. On success, returns selected subscription; Authorino caches the result (e.g., 60s TTL). Identity information (username, groups, subscription, key ID) is made available to TokenRateLimitPolicy and observability through AuthPolicy's `filters.identity` mechanism, but is **not forwarded** as HTTP headers to upstream model workloads (defense-in-depth security). Clients do not send subscription headers on inference; subscription comes from the API key record created at mint time.
 
 ```mermaid
 graph TB

docs/content/configuration-and-management/maas-controller-overview.md

@@ -231,6 +231,34 @@ The Kuadrant AuthPolicy validates API keys via the MaaS API and validates user t
 
 ---
 
+## 9.1. Identity Headers and Defense-in-Depth
+
+**For model inference routes** (HTTPRoutes targeting model workloads):
+
+The controller-generated AuthPolicies do **not** inject most identity-related HTTP headers (`X-MaaS-Username`, `X-MaaS-Group`, `X-MaaS-Key-Id`) into requests forwarded to upstream model pods. This is a defense-in-depth security measure to prevent accidental disclosure of user identity, group membership, and key identifiers in:
+
+- Model runtime logs
+- Upstream debug dumps
+- Misconfigured proxies or sidecars
+
+**Exception:** `X-MaaS-Subscription` **is** injected for Istio Telemetry to enable per-subscription latency tracking. Istio runs in the Envoy gateway and cannot access Authorino's `auth.identity` context—it can only read request headers. The injected subscription value is server-controlled (resolved by Authorino from validated subscriptions), not client-provided.
+
+All identity information remains available to **gateway-level features** through Authorino's `auth.identity` and `auth.metadata` contexts, which are consumed by:
+
+- **TokenRateLimitPolicy (TRLP)**: Uses `selected_subscription_key`, `userid`, `groups`, and `subscription_info` from `filters.identity` (access `subscription_info.labels` for tier-based rate limiting)
+- **Gateway telemetry/metrics**: Accesses identity fields with `metrics: true` enabled on `filters.identity`
+- **Authorization policies**: OPA/Rego rules evaluate `auth.identity` and `auth.metadata` directly
+
+**For maas-api routes**:
+
+The static AuthPolicy for maas-api (`deployment/base/maas-api/policies/auth-policy.yaml`) still injects `X-MaaS-Username` and `X-MaaS-Group` headers, as maas-api's `ExtractUserInfo` middleware requires them. This is separate from model inference routes and follows a different security model (maas-api is a trusted internal service).
+
+**Security motivation:**
+
+Model workloads (vLLM, Llama.cpp, etc.) do not require strong identity claims in cleartext headers. By keeping identity at the gateway layer, we reduce the attack surface and limit the blast radius of potential log leaks or upstream vulnerabilities.
+
+---
+
 ## 10. Summary
 
 | Topic | Summary |

docs/content/configuration-and-management/tls-configuration.md

@@ -152,12 +152,14 @@ This section covers how `maas-api` is configured to use TLS certificates. These
 
 The `maas-api` component accepts TLS configuration via environment variables:
 
-| Variable | Description | Example |
-|----------|-------------|---------|
-| `TLS_CERT` | Path to TLS certificate file | `/etc/maas-api/tls/tls.crt` |
-| `TLS_KEY` | Path to TLS private key file | `/etc/maas-api/tls/tls.key` |
-
-When both variables are set, the API server listens on HTTPS (port 8443) instead of HTTP (port 8080).
+| Variable | Description | Default | Example |
+|----------|-------------|---------|---------|
+| `TLS_CERT` | Path to TLS certificate file | (none) | `/etc/maas-api/tls/tls.crt` |
+| `TLS_KEY` | Path to TLS private key file | (none) | `/etc/maas-api/tls/tls.key` |
+| `TLS_SELF_SIGNED` | Generate a self-signed certificate at startup | `false` | `true` |
+| `TLS_MIN_VERSION` | Minimum accepted TLS version (`1.2` or `1.3`) | `1.2` | `1.3` |
+
+When `TLS_CERT` and `TLS_KEY` are both set, the API server listens on HTTPS (port 8443) instead of HTTP (port 8080). If `TLS_SELF_SIGNED` is set to `true`, a self-signed certificate is generated automatically and explicit cert/key paths are not required. When both cert/key files and `TLS_SELF_SIGNED` are provided, the cert/key files take precedence.
 
 ### Volume Mounts
 

docs/content/install/prerequisites.md

@@ -12,6 +12,7 @@ Red Hat OpenShift AI (RHOAI). MaaS is installed by enabling it in the DataScienc
 |--------------|-----|-------------------------------|-------------|
 | v0.0.2       | 4.19.9+ | v1.3+ / v1.2+             | v1.2+       |
 | v0.1.0       | 4.19.9+ | v1.3+ / v1.2+             | v1.2+       |
+| v0.2.0+      | 4.19.9+ | v1.4.2+ / v1.3+           | v1.2+       |
 
 !!! note "Other Kubernetes flavors"
     Other Kubernetes flavors (e.g., upstream Kubernetes, other distributions) are currently being validated.
@@ -34,14 +35,12 @@ MaaS requires Open Data Hub version 3.0 or later, with the Model Serving compone
 enabled (KServe) and properly configured for deploying models with `LLMInferenceService`
 resources.
 
-A specific requirement for MaaS is to set up ODH's Model Serving with Kuadrant v1.3+, even
-though ODH can work with earlier Kuadrant versions.
+A specific requirement for MaaS v0.2.0+ is to set up ODH's Model Serving with Kuadrant v1.4.2 or later.
 
 ## Requirements for Red Hat OpenShift AI
 
 MaaS requires Red Hat OpenShift AI (RHOAI) version 3.0 or later, with the Model Serving
 component enabled (KServe) and properly configured for deploying models with
 `LLMInferenceService` resources.
 
-A specific requirement for MaaS is to set up RHOAI Model Serving with Red Hat Connectivity
-Link (RHCL) v1.2+, even though RHOAI can work with earlier RHCL versions.
+A specific requirement for MaaS v0.2.0+ is to set up RHOAI Model Serving with Red Hat Connectivity Link (RHCL) v1.3 or later.

maas-api/internal/handlers/models.go

@@ -187,7 +187,17 @@ func (h *ModelsHandler) ListLLMs(c *gin.Context) {
 	// Extract x-maas-subscription header.
 	// For API keys: Authorino injects this from auth.metadata.apiKeyValidation.subscription
 	// For user tokens: This header is not present (Authorino doesn't inject it)
-	requestedSubscription := strings.TrimSpace(c.GetHeader("x-maas-subscription"))
+	// Note: If client sends x-maas-subscription header, there may be multiple values.
+	// Authorino appends its value, so we take the last non-empty value.
+	requestedSubscription := ""
+	headerValues := c.Request.Header.Values("X-Maas-Subscription")
+	for i := len(headerValues) - 1; i >= 0; i-- {
+		trimmed := strings.TrimSpace(headerValues[i])
+		if trimmed != "" {
+			requestedSubscription = trimmed
+			break
+		}
+	}
 	isAPIKeyRequest := strings.HasPrefix(authHeader, "Bearer sk-oai-")
 
 	// Fail closed: API keys without a bound subscription must be rejected

maas-controller/pkg/controller/maas/maasauthpolicy_controller.go

@@ -448,32 +448,22 @@ allow {
 		// match against subscription groups (which may differ from auth policy groups).
 		// Also inject subscription metadata from subscription-info for Limitador metrics.
 		// For API keys: username/groups come from apiKeyValidation metadata
-		// For K8s tokens: username/groups come from auth.identity
+		// Identity headers intentionally removed for defense-in-depth:
+		// User identity, groups, and key IDs are not forwarded to upstream model workloads
+		// to prevent accidental disclosure in logs or dumps. All identity information remains
+		// available to TRLP and telemetry via auth.identity and filters.identity below.
+		// Exception: X-MaaS-Subscription is injected for Istio Telemetry (per-subscription latency tracking).
 		rule["response"] = map[string]interface{}{
 			"success": map[string]interface{}{
 				"headers": map[string]interface{}{
-					// Username from API key validation or K8s token identity
-					"X-MaaS-Username": map[string]interface{}{
+					// Strip Authorization header to prevent token exfiltration to model backends
+					// Both API keys and OpenShift tokens are validated by Authorino, but should
+					// not be forwarded to model services to prevent credential theft
+					"Authorization": map[string]interface{}{
 						"plain": map[string]interface{}{
-							"expression": `(has(auth.metadata) && has(auth.metadata.apiKeyValidation)) ? auth.metadata.apiKeyValidation.username : auth.identity.user.username`,
-						},
-						"metrics":  false,
-						"priority": int64(0),
-					},
-					// Groups - serialize to JSON array string from API key validation or K8s identity
-					// Using string() conversion of JSON-serialized groups for proper escaping
-					"X-MaaS-Group": map[string]interface{}{
-						"plain": map[string]interface{}{
-							"expression": `string(((has(auth.metadata) && has(auth.metadata.apiKeyValidation)) ? auth.metadata.apiKeyValidation.groups : auth.identity.user.groups))`,
-						},
-						"metrics":  false,
-						"priority": int64(0),
-					},
-					// Key ID for tracking (only for API keys)
-					"X-MaaS-Key-Id": map[string]interface{}{
-						"plain": map[string]interface{}{
-							"expression": `(has(auth.metadata) && has(auth.metadata.apiKeyValidation)) ? auth.metadata.apiKeyValidation.keyId : ""`,
+							"value": "",
 						},
+						"key":      "authorization",
 						"metrics":  false,
 						"priority": int64(0),
 					},
@@ -510,14 +500,11 @@ allow {
 										ref.Namespace, ref.Name,
 									),
 								},
-								"organizationId": map[string]interface{}{
-									"expression": `has(auth.metadata["subscription-info"].organizationId) ? auth.metadata["subscription-info"].organizationId : ""`,
-								},
-								"costCenter": map[string]interface{}{
-									"expression": `has(auth.metadata["subscription-info"].costCenter) ? auth.metadata["subscription-info"].costCenter : ""`,
-								},
-								"subscription_labels": map[string]interface{}{
-									"expression": `has(auth.metadata["subscription-info"].labels) ? auth.metadata["subscription-info"].labels : {}`,
+								// Full subscription-info object from subscription-select endpoint
+								// Contains: name, namespace, labels, organizationId, costCenter, error, message
+								// Consumers should access nested fields (e.g., subscription_info.organizationId)
+								"subscription_info": map[string]interface{}{
+									"expression": `has(auth.metadata["subscription-info"].name) ? auth.metadata["subscription-info"] : {}`,
 								},
 								// Error information (for debugging - only populated when selection fails)
 								"subscription_error": map[string]interface{}{