Merge pull request #400 from hbelmiro/main-to-stable

Main to stable

Author: alyssacgoins

.github/resources/scripts/configure-mlflow.sh

@@ -29,6 +29,10 @@ KFP_NAMESPACE="${1:?KFP namespace required}"
 MLFLOW_NAMESPACE="${2:?MLflow namespace required}"
 CONFIG_JSON_PATH="${3:?Path to source config.json required}"
 
+MLFLOW_CA_CONFIGMAP="mlflow-ca-cert"
+KFP_CA_BUNDLE_DIR="/kfp/certs"
+KFP_CA_BUNDLE_PATH="${KFP_CA_BUNDLE_DIR}/ca.crt"
+
 echo "Services in ${MLFLOW_NAMESPACE} namespace:"
 kubectl get svc -n "$MLFLOW_NAMESPACE" --no-headers
 MLFLOW_SVC=$(kubectl get svc -n "$MLFLOW_NAMESPACE" --no-headers -o custom-columns=":metadata.name" | grep -i mlflow | head -1)
@@ -42,9 +46,41 @@ MLFLOW_STATIC_PREFIX="/mlflow"
 MLFLOW_ENDPOINT="https://${MLFLOW_HOST}:${MLFLOW_PORT}${MLFLOW_STATIC_PREFIX}"
 echo "MLflow service: $MLFLOW_SVC port=$MLFLOW_PORT endpoint=$MLFLOW_ENDPOINT"
 
-MLFLOW_PATCH=$(jq -n --arg endpoint "$MLFLOW_ENDPOINT" '{
+# --- Extract CA certificate from the MLflow TLS secret ---
+CA_CERT_FILE="/tmp/mlflow-ca.crt"
+
+echo "Extracting MLflow CA certificate..."
+TLS_SECRET=$(kubectl get secret -n "$MLFLOW_NAMESPACE" -o custom-columns=":metadata.name" --no-headers | grep -i "mlflow.*tls\|tls.*mlflow" | head -1)
+if [ -z "$TLS_SECRET" ]; then
+  TLS_SECRET=$(kubectl get secret -n "$MLFLOW_NAMESPACE" --field-selector type=kubernetes.io/tls -o custom-columns=":metadata.name" --no-headers | head -1)
+fi
+if [ -z "$TLS_SECRET" ]; then
+  echo "ERROR: No TLS secret found in namespace $MLFLOW_NAMESPACE"
+  exit 1
+fi
+echo "Found TLS secret: $TLS_SECRET"
+
+# cert-manager stores the CA in the ca.crt key; fall back to tls.crt if ca.crt is absent
+CA_DATA=$(kubectl get secret -n "$MLFLOW_NAMESPACE" "$TLS_SECRET" -o jsonpath='{.data.ca\.crt}' 2>/dev/null)
+if [ -z "$CA_DATA" ]; then
+  echo "ca.crt not found in secret, falling back to tls.crt"
+  CA_DATA=$(kubectl get secret -n "$MLFLOW_NAMESPACE" "$TLS_SECRET" -o jsonpath='{.data.tls\.crt}')
+fi
+if [ -z "$CA_DATA" ]; then
+  echo "ERROR: Could not extract CA certificate from secret $TLS_SECRET"
+  exit 1
+fi
+echo "$CA_DATA" | base64 -d > "$CA_CERT_FILE"
+echo "CA certificate extracted to $CA_CERT_FILE ($(wc -l < "$CA_CERT_FILE") lines)"
+
+# ConfigMap consumed by the API server (plugin config) and compiler (driver/launcher pods).
+kubectl create configmap "$MLFLOW_CA_CONFIGMAP" -n "$KFP_NAMESPACE" \
+  --from-file=ca.crt="$CA_CERT_FILE" --dry-run=client -o yaml | kubectl apply -f -
+
+# --- Build the MLflow plugin config with caBundlePath ---
+MLFLOW_PATCH=$(jq -n --arg endpoint "$MLFLOW_ENDPOINT" --arg caBundlePath "$KFP_CA_BUNDLE_PATH" '{
   endpoint: $endpoint,
-  tls: { insecureSkipVerify: true },
+  tls: { caBundlePath: $caBundlePath },
   settings: { workspacesEnabled: true }
 }')
 
@@ -54,10 +90,11 @@ jq --argjson mlflow "$MLFLOW_PATCH" '. + { plugins: { mlflow: $mlflow } }' \
 echo "Patched config.json plugins.mlflow:"
 jq '.plugins.mlflow' /tmp/kfp-config.json
 
+# --- Deploy plugin config and wire CA trust on the API server + compiler ---
 kubectl create configmap kfp-mlflow-config -n "$KFP_NAMESPACE" \
   --from-file=config.json=/tmp/kfp-config.json --dry-run=client -o yaml | kubectl apply -f -
 kubectl patch deployment ml-pipeline -n "$KFP_NAMESPACE" --type=strategic -p \
-  '{"spec":{"template":{"spec":{"volumes":[{"name":"mlflow-cfg","configMap":{"name":"kfp-mlflow-config"}}],"containers":[{"name":"ml-pipeline-api-server","volumeMounts":[{"name":"mlflow-cfg","mountPath":"/config/config.json","subPath":"config.json"}]}]}}}}'
+  '{"spec":{"template":{"spec":{"volumes":[{"name":"mlflow-cfg","configMap":{"name":"kfp-mlflow-config"}},{"name":"mlflow-ca","configMap":{"name":"'"$MLFLOW_CA_CONFIGMAP"'"}}],"containers":[{"name":"ml-pipeline-api-server","env":[{"name":"CABUNDLE_CONFIGMAP_NAME","value":"'"$MLFLOW_CA_CONFIGMAP"'"}],"volumeMounts":[{"name":"mlflow-cfg","mountPath":"/config/config.json","subPath":"config.json"},{"name":"mlflow-ca","mountPath":"'"$KFP_CA_BUNDLE_DIR"'","readOnly":true}]}]}}}}'
 kubectl rollout status deployment/ml-pipeline -n "$KFP_NAMESPACE" --timeout=180s
 
 pkill -f "kubectl port-forward.*ml-pipeline.*8888" || true
@@ -87,12 +124,13 @@ if [ -n "${GITHUB_ENV:-}" ]; then
   echo "MLFLOW_PORT_FORWARD_NS=$MLFLOW_NAMESPACE" >> "$GITHUB_ENV"
   echo "MLFLOW_PORT_FORWARD_SVC=$MLFLOW_SVC" >> "$GITHUB_ENV"
   echo "MLFLOW_PORT_FORWARD_REMOTE_PORT=$MLFLOW_PORT" >> "$GITHUB_ENV"
+  echo "MLFLOW_CA_BUNDLE_PATH=$CA_CERT_FILE" >> "$GITHUB_ENV"
   if [ -n "$SA_TOKEN" ]; then
     echo "MLFLOW_BEARER_TOKEN=$SA_TOKEN" >> "$GITHUB_ENV"
-    echo "Exported MLFLOW_BEARER_TOKEN and MLFLOW_WORKSPACE for test helpers"
+    echo "Exported MLFLOW_BEARER_TOKEN, MLFLOW_WORKSPACE, and MLFLOW_CA_BUNDLE_PATH for test helpers"
   else
     echo "WARNING: Could not create SA token; MLflow requests may be unauthenticated"
-    echo "Exported MLFLOW_WORKSPACE only"
+    echo "Exported MLFLOW_WORKSPACE and MLFLOW_CA_BUNDLE_PATH only"
   fi
 fi
 
@@ -105,7 +143,7 @@ CURL_HEADERS=(-H "X-MLflow-Workspace: $KFP_NAMESPACE")
 
 STATUS=000
 for i in $(seq 1 30); do
-  STATUS=$(curl -sk -o /dev/null -w '%{http_code}' --connect-timeout 5 --max-time 10 \
+  STATUS=$(curl -s --cacert "$CA_CERT_FILE" -o /dev/null -w '%{http_code}' --connect-timeout 5 --max-time 10 \
     "${CURL_HEADERS[@]}" "$HEALTH_URL" 2>/dev/null || echo "000")
   if [ "$STATUS" != "000" ] && [ "$STATUS" -lt 500 ] 2>/dev/null; then
     echo "MLflow backend is healthy on localhost:8080 (HTTPS, status=$STATUS)"

.github/workflows/e2e-test.yml

@@ -12,7 +12,6 @@ env:
   AWS_SECRET_ACCESS_KEY: 'minio123'
   AWS_S3_BUCKET: 'mlpipeline'
   MLFLOW_TRACKING_URI: "https://localhost:8080/mlflow"
-  MLFLOW_TRACKING_INSECURE_TLS: "true"
 on:
   push:
     branches:
@@ -357,12 +356,12 @@ jobs:
 
       - name: Deploy MLflow
         id: deploy-mlflow        
-        uses: opendatahub-io/mlflow-operator/.github/actions/deploy@b55bcb6aa528ccecf8904745bb0218350b1ff453
+        uses: opendatahub-io/mlflow-operator/.github/actions/deploy@ec59b7c996ef4d14c448254132e21bd85fd1c28f
         if: ${{ steps.create-cluster.outcome == 'success' }}
         with:
           namespace: 'opendatahub'
-          mlflow_image: quay.io/${{ github.repository_owner == 'red-hat-data-services' && 'rhoai' || 'opendatahub' }}/mlflow:odh-stable
-          mlflow_operator_image: quay.io/${{ github.repository_owner == 'red-hat-data-services' && 'rhoai' || 'opendatahub' }}/mlflow-operator:odh-stable
+          mlflow_image: quay.io/${{ github.repository_owner == 'red-hat-data-services' && 'rhoai' || 'opendatahub' }}/mlflow:odh-stable-490cd77b5b697a8ad0c5a84272983f668b961524
+          mlflow_operator_image: quay.io/${{ github.repository_owner == 'red-hat-data-services' && 'rhoai' || 'opendatahub' }}/mlflow-operator:ec59b7c996ef4d14c448254132e21bd85fd1c28f
           backend_store: ${{ matrix.backend_store }}
           artifact_storage: ${{ matrix.artifact_storage }}
           registry_store: ${{ matrix.registry_store }}

.github/workflows/upgrade-test.yml

@@ -122,9 +122,47 @@ jobs:
           skip_operator_deployment: ${{ env.SKIP_OPERATOR_DEPLOYMENT }}
           pod_to_pod_tls_enabled: ${{ matrix.tls_enabled }}
 
+      - name: Wait for upgrade rollout and re-establish port-forwarding
+        if: ${{ steps.deploy.outcome == 'success' }}
+        id: post-upgrade-port-forward
+        shell: bash
+        run: |
+          DEPLOYMENT_NAME="${{ steps.deploy.outputs.deployment_name }}"
+          NAMESPACE="${{ env.NAMESPACE }}"
+
+          OLD_POD_UID=$(kubectl get pod -l app=$DEPLOYMENT_NAME -n $NAMESPACE -o jsonpath='{.items[0].metadata.uid}')
+          echo "Current pod UID: $OLD_POD_UID"
+
+          echo "Waiting for operator to roll the deployment..."
+          for i in $(seq 1 36); do
+            NEW_POD_UID=$(kubectl get pod -l app=$DEPLOYMENT_NAME -n $NAMESPACE -o jsonpath='{.items[0].metadata.uid}' 2>/dev/null || echo "")
+            if [ -n "$NEW_POD_UID" ] && [ "$NEW_POD_UID" != "$OLD_POD_UID" ]; then
+              echo "New pod detected (UID: $NEW_POD_UID) after $((i * 5))s"
+              break
+            fi
+            if [ $i -eq 36 ]; then
+              echo "ERROR: Pod UID did not change after 180s — operator did not roll the deployment"
+              exit 1
+            fi
+            sleep 5
+          done
+
+          echo "Waiting for rollout to complete..."
+          kubectl rollout status deployment/$DEPLOYMENT_NAME -n $NAMESPACE --timeout=300s
+
+          echo "Waiting for pod to be ready..."
+          kubectl wait --for=condition=ready pod -l app=$DEPLOYMENT_NAME -n $NAMESPACE --timeout=120s --field-selector=status.phase=Running
+
+          echo "Killing stale port-forward processes..."
+          pkill -f "kubectl port-forward.*8888" || true
+          sleep 3
+
+          echo "Re-establishing port-forward..."
+          ./.github/resources/scripts/forward-port.sh "$NAMESPACE" "$DEPLOYMENT_NAME" 8888 8888
+
       - name: Verify Upgrade
         uses: ./.github/actions/test-and-report
-        if: ${{ steps.deploy.outcome == 'success' }}
+        if: ${{ steps.post-upgrade-port-forward.outcome == 'success' }}
         with:
           test_directory: ${{ env.TESTS_DIR }}
           test_label: "UpgradeVerification"

AGENTS.md

@@ -243,7 +243,7 @@ ginkgo -v --label-filter="Smoke" ./backend/test/v2/api
 ginkgo -v ./backend/test/end2end -- -namespace=kubeflow -isDebugMode=true
 ```
 
-Without `--label-filter`, `gpu`-labeled E2E tests run as well (they request accelerators) and may pend or fail on CPU-only clusters; pass `--label-filter` to limit what runs (for example `Smoke`, or `gpu` on GPU-capable clusters). NVIDIA vs AMD fixtures: set `KFP_E2E_GPU_VENDOR` to `nvidia` (default if unset), `amd`, or `both` so only matching IRs run (`pytorch_nvidia_gpu_availability.yaml` / `pytorch_amd_gpu_availability.yaml`). Any value other than `amd`/`both`/`all` falls back to `nvidia`.
+Without `--label-filter`, `gpu`-labeled E2E tests run as well (they request accelerators) and may pend or fail on CPU-only clusters; pass `--label-filter` to limit what runs (for example `Smoke`, or `gpu` on GPU-capable clusters). NVIDIA vs AMD fixtures: pass `-gpuVendor` flag to `nvidia` (default), `amd`, or `both` so only matching IRs run (`pytorch_nvidia_gpu_availability.yaml` / `pytorch_amd_gpu_availability.yaml`). Any value other than `amd`/`both`/`all` falls back to `nvidia`.
 
 Test data is centralized under:
 

backend/src/apiserver/config/config.go

@@ -261,7 +261,8 @@ func LoadSamples(resourceManager *resource.ResourceManager, sampleConfigPath str
 		}
 
 		// Create pipeline if it does not already exist
-		p, fetchErr := resourceManager.GetPipelineByNameAndNamespace(cfg.Name, common.GetPodNamespace())
+		namespace := resourceManager.ReplaceNamespace(common.GetPodNamespace())
+		p, fetchErr := resourceManager.GetPipelineByNameAndNamespace(cfg.Name, namespace)
 		if fetchErr != nil {
 			if util.IsUserErrorCodeMatch(fetchErr, codes.NotFound) {
 				p, configErr = resourceManager.CreatePipeline(&model.Pipeline{
@@ -271,12 +272,21 @@ func LoadSamples(resourceManager *resource.ResourceManager, sampleConfigPath str
 					Tags:        tags,
 				})
 				if configErr != nil {
-					// Log the error but not fail. The API Server pod can restart and it could potentially cause
-					// name collision. In the future, we might consider loading samples during deployment, instead
-					// of when API server starts.
-					glog.Warningf(fmt.Sprintf(
-						"Failed to create pipeline for %s. Error: %v", cfg.Name, configErr))
-					continue
+					if util.IsUserErrorCodeMatch(configErr, codes.AlreadyExists) {
+						// Retry without namespace filter. In standalone mode this
+						// finds the empty-namespace pipeline; safe because sample
+						// names are deterministic and collisions across namespaces
+						// are not expected for managed pipelines.
+						p, fetchErr = resourceManager.GetPipelineByNameAndNamespace(cfg.Name, "")
+						if fetchErr != nil {
+							glog.Warningf("Failed to create or find pipeline %s: create=%v, fetch=%v", cfg.Name, configErr, fetchErr)
+							continue
+						}
+						glog.Infof("Pipeline %s already exists (id=%s), proceeding to version check.", cfg.Name, p.UUID)
+					} else {
+						glog.Warningf("Failed to create pipeline for %s. Error: %v", cfg.Name, configErr)
+						continue
+					}
 				} else {
 					glog.Info(fmt.Sprintf("Successfully uploaded Pipeline %s.", cfg.Name))
 				}
@@ -311,7 +321,7 @@ func LoadSamples(resourceManager *resource.ResourceManager, sampleConfigPath str
 		// If the Pipeline Version exists, do nothing
 		// Otherwise upload new Pipeline Version for
 		// this pipeline.
-		_, fetchErr = resourceManager.GetPipelineVersionByName(pvName)
+		_, fetchErr = resourceManager.GetPipelineVersionByName(p.UUID, pvName)
 		if fetchErr != nil {
 			if util.IsUserErrorCodeMatch(fetchErr, codes.NotFound) {
 				_, configErr = resourceManager.CreatePipelineVersion(

backend/src/apiserver/config/config.json

@@ -24,18 +24,5 @@
   "CacheEnabled": "true",
   "CRON_SCHEDULE_TIMEZONE": "UTC",
   "CACHE_IMAGE": "ghcr.io/containerd/busybox",
-  "CACHE_NODE_RESTRICTIONS": "false",
-  "plugins": {
-    "mlflow": {
-      "endpoint": "https://mlflow.opendatahub.svc.cluster.local:8443",
-      "timeout": "60s",
-      "tls": {
-        "insecureSkipVerify": false,
-        "caBundlePath": "/etc/mlflow-tracking-ca/ca.crt"
-      },
-      "settings": {
-        "workspacesEnabled": true
-      }
-    }
-  }
+  "CACHE_NODE_RESTRICTIONS": "false"
 }