chore: promote stable to rhoai (#964)

## Promotion: stable → rhoai

Automated promotion of **11 commit(s)** from `stable` to `rhoai`.

| Detail | Value |
| --- | --- |
| Promotion | `stable` → `rhoai` |
| Commits to merge | **11** |
| Conflict check | ✅ Passed |

### Commits included

````
81628d8 feat: add pre-auth ext_proc stage for body-based model routing (#948)
5127624 fix: use tiny-gpt2 URI for premium simulator in CI (#961)
3bb5b77 fix: route subscription latency to odh monitoring stack (#906)
06ffb2d fix: revert "feat: move external cronjob to run timely cleanup to goroutin… (#953)
44613ee feat: move external cronjob to run timely cleanup to goroutine+configable for internval of cleanup (#934)
a7f20f0 fix: update ODH version in scripts to get up-to-date TLS feature (#945)
dfb6bb3 test: enable OIDC by default in E2E and expand test coverage (3→16 tests) (#679)
37e9be9 fix: remove namespace field from model refs in GET /subscriptions response (#915)
ae511cc feat: alert on authorino maas-api metadata evaluator failures (#895)
d1cfced docs: add payload processing namespace configuration reference (#822)
````

---
⚠️ **Merge this PR with a merge commit** (do not squash or rebase).

Author: chaitanya1731

.github/hack/install-odh.sh

@@ -8,7 +8,8 @@
 #   OPERATOR_CATALOG - Custom catalog image (optional). When unset, uses community-operators.
 #                      Set to e.g. quay.io/opendatahub/opendatahub-operator-catalog:latest for custom builds.
 #   OPERATOR_CHANNEL   - Subscription channel (default: fast-3)
-#   OPERATOR_STARTING_CSV - Pin Subscription startingCSV (default: opendatahub-operator.v3.4.0-ea.1). Set to "-" to omit.
+#   OPERATOR_STARTING_CSV - Pin Subscription startingCSV (optional). When unset,
+#                           follow the channel head (latest in fast-3 by default).
 #   OPERATOR_INSTALL_PLAN_APPROVAL - Manual (default) or Automatic; use "-" to omit.
 #     Manual: blocks auto-upgrades; this script auto-approves only the first InstallPlan so install does not stall.
 #   OPERATOR_IMAGE   - Custom operator image to patch into CSV (optional)
@@ -124,9 +125,9 @@ else
   channel="${OPERATOR_CHANNEL:-fast-3}"
 fi
 
-# Pin to ODH 3.4 EA1 unless overridden (omit with OPERATOR_STARTING_CSV=- to follow channel head)
-starting_csv="${OPERATOR_STARTING_CSV:-opendatahub-operator.v3.4.0-ea.1}"
-[[ "$starting_csv" == "-" ]] && starting_csv=""
+# Follow the configured channel head by default unless OPERATOR_STARTING_CSV
+# explicitly pins a CSV.
+starting_csv="${OPERATOR_STARTING_CSV:-}"
 
 # Manual = no auto-upgrades; install_olm_operator still approves the first InstallPlan programmatically
 plan_approval="${OPERATOR_INSTALL_PLAN_APPROVAL:-Manual}"

.github/workflows/openapi-validation.yml

@@ -112,8 +112,9 @@ jobs:
           git fetch origin "$BASE_REF"
           git show "origin/$BASE_REF:maas-api/openapi3.yaml" > base-spec.yaml
 
-          # Run breaking change detection
-          oasdiff breaking base-spec.yaml maas-api/openapi3.yaml > breaking-changes.txt || true
+          # Run breaking change detection (--warn-ignore suppresses acknowledged warnings)
+          oasdiff breaking base-spec.yaml maas-api/openapi3.yaml \
+            --warn-ignore maas-api/oasdiff-warn-ignore.txt > breaking-changes.txt || true
 
           # Check if actual breaking changes were detected.
           # oasdiff outputs "No changes detected" when specs are identical, and

deployment/base/observability/authorino-maas-metadata-evaluator-prometheusrule.yaml

@@ -0,0 +1,37 @@
+# PrometheusRule: alert when Authorino HTTP metadata evaluators for maas-api fail often.
+#
+# Scraped metrics come from Authorino /server-metrics (see authorino-server-metrics-servicemonitor.yaml).
+# Requires metadata rules apiKeyValidation + subscription-info with metrics enabled (MaaS controller).
+#
+# Deployed by: scripts/observability/install-observability.sh → kustomize build deployment/base/observability
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: authorino-maas-metadata-evaluator-high-failure-rate
+  namespace: kuadrant-system
+  labels:
+    app.kubernetes.io/part-of: maas-observability
+    app.kubernetes.io/component: alerting
+spec:
+  groups:
+    - name: maas.authorino.metadata
+      rules:
+        - alert: MaaSAuthorinoMetadataEvaluatorHighFailureRate
+          annotations:
+            summary: Authorino maas-api metadata HTTP evaluators are failing often
+            description: >-
+              Over the last 5 minutes, more than 10% of Authorino HTTP metadata evaluations
+              for apiKeyValidation or subscription-info (calls to maas-api) were cancelled/failed.
+              Investigate maas-api health, network policies, TLS trust between Authorino and maas-api,
+              and subscription/API-key validation paths.
+          expr: |
+            (
+              sum(rate(auth_server_evaluator_cancelled{namespace=~"rh-connectivity-link|kuadrant-system",evaluator_type="METADATA_GENERIC_HTTP",evaluator_name=~"apiKeyValidation|subscription-info"}[5m]))
+              /
+              sum(rate(auth_server_evaluator_total{namespace=~"rh-connectivity-link|kuadrant-system",evaluator_type="METADATA_GENERIC_HTTP",evaluator_name=~"apiKeyValidation|subscription-info"}[5m]))
+            ) > 0.1
+            and
+            sum(rate(auth_server_evaluator_total{namespace=~"rh-connectivity-link|kuadrant-system",evaluator_type="METADATA_GENERIC_HTTP",evaluator_name=~"apiKeyValidation|subscription-info"}[5m])) > 0.001
+          for: 5m
+          labels:
+            severity: warning

deployment/base/observability/istio-gateway-servicemonitor.yaml

@@ -7,6 +7,9 @@ metadata:
   labels:
     app.kubernetes.io/part-of: maas-observability
     app.kubernetes.io/managed-by: maas-observability
+    monitoring.opendatahub.io/scrape: 'true'
+    monitoring.openshift.io/collection-profile: minimal
+    openshift.io/user-monitoring: 'false'
 spec:
   selector:
     matchLabels:

deployment/base/observability/kustomization.yaml

@@ -14,6 +14,11 @@ resources:
 # NOTE: istio-gateway-service.yaml and istio-gateway-servicemonitor.yaml are NOT
 # included here. They are applied conditionally by install-observability.sh only
 # when the gateway deployment exists.
+#
+# NOTE: authorino-maas-metadata-evaluator-prometheusrule.yaml is NOT included here.
+# It has a hardcoded namespace (kuadrant-system) which causes problems in RHOAI
+# (rh-connectivity-link). The install-observability.sh script deploys it with
+# dynamic namespace substitution.
 - gateway-telemetry-policy.yaml
 - istio-gateway-telemetry.yaml
 

deployment/base/payload-processing/after/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../instance

…yload-processing/manager/deployment.yaml …load-processing/instance/deployment.yamldeployment/base/payload-processing/manager/deployment.yaml renamed to deployment/base/payload-processing/instance/deployment.yaml deployment/base/payload-processing/manager/deployment.yaml renamed to deployment/base/payload-processing/instance/deployment.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - deployment.yaml
+  - service.yaml
+  - destination-rule.yaml