UPSTREAM: <carry>: fixing removal of InsecureSkipverify

Signed-off-by: Nelesh Singla <117123879+nsingla@users.noreply.github.com>

Author: nsingla

.github/resources/scripts/configure-mlflow.sh

@@ -180,12 +180,13 @@ if [ -n "${GITHUB_ENV:-}" ]; then
   echo "MLFLOW_PORT_FORWARD_NS=$MLFLOW_NAMESPACE" >> "$GITHUB_ENV"
   echo "MLFLOW_PORT_FORWARD_SVC=$MLFLOW_SVC" >> "$GITHUB_ENV"
   echo "MLFLOW_PORT_FORWARD_REMOTE_PORT=$MLFLOW_PORT" >> "$GITHUB_ENV"
+  echo "MLFLOW_CA_BUNDLE_PATH=$CA_CERT_FILE" >> "$GITHUB_ENV"
   if [ -n "$SA_TOKEN" ]; then
     echo "MLFLOW_BEARER_TOKEN=$SA_TOKEN" >> "$GITHUB_ENV"
-    echo "Exported MLFLOW_BEARER_TOKEN and MLFLOW_WORKSPACE for test helpers"
+    echo "Exported MLFLOW_BEARER_TOKEN, MLFLOW_WORKSPACE, and MLFLOW_CA_BUNDLE_PATH for test helpers"
   else
     echo "WARNING: Could not create SA token; MLflow requests may be unauthenticated"
-    echo "Exported MLFLOW_WORKSPACE only"
+    echo "Exported MLFLOW_WORKSPACE and MLFLOW_CA_BUNDLE_PATH only"
   fi
 fi
 
@@ -198,7 +199,7 @@ CURL_HEADERS=(-H "X-MLflow-Workspace: $KFP_NAMESPACE")
 
 STATUS=000
 for i in $(seq 1 30); do
-  STATUS=$(curl -sk -o /dev/null -w '%{http_code}' --connect-timeout 5 --max-time 10 \
+  STATUS=$(curl -s --cacert "$CA_CERT_FILE" -o /dev/null -w '%{http_code}' --connect-timeout 5 --max-time 10 \
     "${CURL_HEADERS[@]}" "$HEALTH_URL" 2>/dev/null || echo "000")
   if [ "$STATUS" != "000" ] && [ "$STATUS" -lt 500 ] 2>/dev/null; then
     echo "MLflow backend is healthy on localhost:8080 (HTTPS, status=$STATUS)"

.github/workflows/e2e-test.yml

@@ -12,7 +12,6 @@ env:
   AWS_SECRET_ACCESS_KEY: 'minio123'
   AWS_S3_BUCKET: 'mlpipeline'
   MLFLOW_TRACKING_URI: "https://localhost:8080/mlflow"
-  MLFLOW_TRACKING_INSECURE_TLS: "true"
 on:
   push:
     branches:
@@ -357,12 +356,12 @@ jobs:
 
       - name: Deploy MLflow
         id: deploy-mlflow        
-        uses: opendatahub-io/mlflow-operator/.github/actions/deploy@b55bcb6aa528ccecf8904745bb0218350b1ff453
+        uses: opendatahub-io/mlflow-operator/.github/actions/deploy@ec59b7c996ef4d14c448254132e21bd85fd1c28f
         if: ${{ steps.create-cluster.outcome == 'success' }}
         with:
           namespace: 'opendatahub'
-          mlflow_image: quay.io/${{ github.repository_owner == 'red-hat-data-services' && 'rhoai' || 'opendatahub' }}/mlflow:odh-stable
-          mlflow_operator_image: quay.io/${{ github.repository_owner == 'red-hat-data-services' && 'rhoai' || 'opendatahub' }}/mlflow-operator:odh-stable
+          mlflow_image: quay.io/${{ github.repository_owner == 'red-hat-data-services' && 'rhoai' || 'opendatahub' }}/mlflow:odh-stable-490cd77b5b697a8ad0c5a84272983f668b961524
+          mlflow_operator_image: quay.io/${{ github.repository_owner == 'red-hat-data-services' && 'rhoai' || 'opendatahub' }}/mlflow-operator:ec59b7c996ef4d14c448254132e21bd85fd1c28f
           backend_store: ${{ matrix.backend_store }}
           artifact_storage: ${{ matrix.artifact_storage }}
           registry_store: ${{ matrix.registry_store }}

backend/test/end2end/utils/mlflow_utils.go

@@ -17,6 +17,7 @@ package utils
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -38,24 +39,37 @@ import (
 )
 
 const (
-	mlflowEndpointEnv    = "MLFLOW_TRACKING_URI"
-	mlflowInsecureTLSEnv = "MLFLOW_TRACKING_INSECURE_TLS"
-	mlflowBearerTokenEnv = "MLFLOW_BEARER_TOKEN"
-	mlflowWorkspaceEnv   = "MLFLOW_WORKSPACE"
-	mlflowPluginKey      = "MLflow"
+	mlflowEndpointEnv      = "MLFLOW_TRACKING_URI"
+	mlflowCABundlePathEnv  = "MLFLOW_CA_BUNDLE_PATH"
+	mlflowBearerTokenEnv   = "MLFLOW_BEARER_TOKEN"
+	mlflowWorkspaceEnv     = "MLFLOW_WORKSPACE"
+	mlflowPluginKey        = "MLflow"
 )
 
 func getMLflowClient(endpoint string) (*mlflowclient.Client, error) {
-	insecure := strings.EqualFold(os.Getenv(mlflowInsecureTLSEnv), "true")
 	workspace := os.Getenv(mlflowWorkspaceEnv)
 	bearerToken := os.Getenv(mlflowBearerTokenEnv)
+
+	tlsConfig := &tls.Config{}
+	if caBundlePath := os.Getenv(mlflowCABundlePathEnv); caBundlePath != "" {
+		caCert, err := os.ReadFile(caBundlePath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read CA bundle from %s: %w", caBundlePath, err)
+		}
+		pool, err := x509.SystemCertPool()
+		if err != nil {
+			pool = x509.NewCertPool()
+		}
+		if !pool.AppendCertsFromPEM(caCert) {
+			return nil, fmt.Errorf("failed to parse CA certificate from %s", caBundlePath)
+		}
+		tlsConfig.RootCAs = pool
+		logger.Log("MLflow client initialized with CA bundle from %s", caBundlePath)
+	}
+
 	httpClient := &http.Client{
-		Timeout: 30 * time.Second,
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: insecure, //nolint:gosec
-			},
-		},
+		Timeout:   30 * time.Second,
+		Transport: &http.Transport{TLSClientConfig: tlsConfig},
 	}
 	client, err := mlflowclient.NewClient(mlflowclient.Config{
 		Endpoint:          endpoint,
@@ -73,9 +87,6 @@ func getMLflowClient(endpoint string) (*mlflowclient.Client, error) {
 	if workspace != "" {
 		logger.Log("MLflow client initialized with workspace header: %s", workspace)
 	}
-	if insecure {
-		logger.Log("MLflow client initialized with InsecureSkipVerify=true")
-	}
 	return client, nil
 }