fix

Signed-off-by: Alyssa Goins <agoins@redhat.com>

Author: alyssacgoins

.github/resources/scripts/configure-mlflow.sh

@@ -42,9 +42,101 @@ MLFLOW_STATIC_PREFIX="/mlflow"
 MLFLOW_ENDPOINT="https://${MLFLOW_HOST}:${MLFLOW_PORT}${MLFLOW_STATIC_PREFIX}"
 echo "MLflow service: $MLFLOW_SVC port=$MLFLOW_PORT endpoint=$MLFLOW_ENDPOINT"
 
-MLFLOW_PATCH=$(jq -n --arg endpoint "$MLFLOW_ENDPOINT" '{
+# --- Extract CA certificate from the MLflow TLS secret ---
+CA_MOUNT_PATH="/etc/mlflow/tls/ca.crt"
+CA_CERT_FILE="/tmp/mlflow-ca.crt"
+
+echo "Extracting MLflow CA certificate..."
+TLS_SECRET=$(kubectl get secret -n "$MLFLOW_NAMESPACE" -o custom-columns=":metadata.name" --no-headers | grep -i "mlflow.*tls\|tls.*mlflow" | head -1)
+if [ -z "$TLS_SECRET" ]; then
+  TLS_SECRET=$(kubectl get secret -n "$MLFLOW_NAMESPACE" --field-selector type=kubernetes.io/tls -o custom-columns=":metadata.name" --no-headers | head -1)
+fi
+if [ -z "$TLS_SECRET" ]; then
+  echo "ERROR: No TLS secret found in namespace $MLFLOW_NAMESPACE"
+  exit 1
+fi
+echo "Found TLS secret: $TLS_SECRET"
+
+# cert-manager stores the CA in the ca.crt key; fall back to tls.crt if ca.crt is absent
+CA_DATA=$(kubectl get secret -n "$MLFLOW_NAMESPACE" "$TLS_SECRET" -o jsonpath='{.data.ca\.crt}' 2>/dev/null)
+if [ -z "$CA_DATA" ]; then
+  echo "ca.crt not found in secret, falling back to tls.crt"
+  CA_DATA=$(kubectl get secret -n "$MLFLOW_NAMESPACE" "$TLS_SECRET" -o jsonpath='{.data.tls\.crt}')
+fi
+if [ -z "$CA_DATA" ]; then
+  echo "ERROR: Could not extract CA certificate from secret $TLS_SECRET"
+  exit 1
+fi
+echo "$CA_DATA" | base64 -d > "$CA_CERT_FILE"
+echo "CA certificate extracted to $CA_CERT_FILE ($(wc -l < "$CA_CERT_FILE") lines)"
+
+# Create a ConfigMap with the CA cert in the KFP namespace for workflow pods
+kubectl create configmap mlflow-ca-cert -n "$KFP_NAMESPACE" \
+  --from-file=ca.crt="$CA_CERT_FILE" --dry-run=client -o yaml | kubectl apply -f -
+
+# --- Patch Argo workflow-controller-configmap to mount the CA cert into all workflow pods ---
+echo "Patching workflow-controller-configmap to inject MLflow CA into workflow pods..."
+
+EXISTING_MAIN_CONTAINER=$(kubectl get configmap workflow-controller-configmap -n "$KFP_NAMESPACE" \
+  -o jsonpath='{.data.mainContainer}' 2>/dev/null || echo "")
+if [ -n "$EXISTING_MAIN_CONTAINER" ]; then
+  MAIN_CONTAINER_PATCH=$(echo "$EXISTING_MAIN_CONTAINER" | \
+    python3 -c "
+import sys, json
+raw = sys.stdin.read().strip()
+try:
+    cfg = json.loads(raw)
+except json.JSONDecodeError:
+    import yaml
+    cfg = yaml.safe_load(raw) or {}
+vms = cfg.get('volumeMounts', [])
+vms = [vm for vm in vms if vm.get('name') != 'mlflow-ca']
+vms.append({'name': 'mlflow-ca', 'mountPath': '/etc/mlflow/tls', 'readOnly': True})
+cfg['volumeMounts'] = vms
+print(json.dumps(cfg))
+")
+else
+  MAIN_CONTAINER_PATCH='{"volumeMounts":[{"name":"mlflow-ca","mountPath":"/etc/mlflow/tls","readOnly":true}]}'
+fi
+
+EXISTING_WF_DEFAULTS=$(kubectl get configmap workflow-controller-configmap -n "$KFP_NAMESPACE" \
+  -o jsonpath='{.data.workflowDefaults}' 2>/dev/null || echo "")
+if [ -n "$EXISTING_WF_DEFAULTS" ]; then
+  WF_DEFAULTS_PATCH=$(echo "$EXISTING_WF_DEFAULTS" | \
+    python3 -c "
+import sys, yaml
+cfg = yaml.safe_load(sys.stdin) or {}
+spec = cfg.setdefault('spec', {})
+volumes = spec.setdefault('volumes', [])
+volumes = [v for v in volumes if v.get('name') != 'mlflow-ca']
+volumes.append({'name': 'mlflow-ca', 'configMap': {'name': 'mlflow-ca-cert'}})
+spec['volumes'] = volumes
+cfg['spec'] = spec
+print(yaml.dump(cfg, default_flow_style=False))
+")
+else
+  WF_DEFAULTS_PATCH=$(cat <<'YAML'
+spec:
+  volumes:
+  - name: mlflow-ca
+    configMap:
+      name: mlflow-ca-cert
+YAML
+)
+fi
+
+kubectl patch configmap workflow-controller-configmap -n "$KFP_NAMESPACE" --type=merge \
+  -p "$(jq -n --arg mc "$MAIN_CONTAINER_PATCH" --arg wd "$WF_DEFAULTS_PATCH" \
+    '{"data":{"mainContainer":$mc,"workflowDefaults":$wd}}')"
+
+# Restart workflow controller to pick up configmap changes
+kubectl rollout restart deployment/workflow-controller -n "$KFP_NAMESPACE"
+kubectl rollout status deployment/workflow-controller -n "$KFP_NAMESPACE" --timeout=120s
+
+# --- Build the MLflow plugin config with caBundlePath ---
+MLFLOW_PATCH=$(jq -n --arg endpoint "$MLFLOW_ENDPOINT" --arg caBundlePath "$CA_MOUNT_PATH" '{
   endpoint: $endpoint,
-  tls: { insecureSkipVerify: true },
+  tls: { caBundlePath: $caBundlePath },
   settings: { workspacesEnabled: true }
 }')
 
@@ -54,10 +146,11 @@ jq --argjson mlflow "$MLFLOW_PATCH" '. + { plugins: { mlflow: $mlflow } }' \
 echo "Patched config.json plugins.mlflow:"
 jq '.plugins.mlflow' /tmp/kfp-config.json
 
+# --- Deploy the config and mount the CA cert into the API server ---
 kubectl create configmap kfp-mlflow-config -n "$KFP_NAMESPACE" \
   --from-file=config.json=/tmp/kfp-config.json --dry-run=client -o yaml | kubectl apply -f -
 kubectl patch deployment ml-pipeline -n "$KFP_NAMESPACE" --type=strategic -p \
-  '{"spec":{"template":{"spec":{"volumes":[{"name":"mlflow-cfg","configMap":{"name":"kfp-mlflow-config"}}],"containers":[{"name":"ml-pipeline-api-server","volumeMounts":[{"name":"mlflow-cfg","mountPath":"/config/config.json","subPath":"config.json"}]}]}}}}'
+  '{"spec":{"template":{"spec":{"volumes":[{"name":"mlflow-cfg","configMap":{"name":"kfp-mlflow-config"}},{"name":"mlflow-ca","configMap":{"name":"mlflow-ca-cert"}}],"containers":[{"name":"ml-pipeline-api-server","volumeMounts":[{"name":"mlflow-cfg","mountPath":"/config/config.json","subPath":"config.json"},{"name":"mlflow-ca","mountPath":"/etc/mlflow/tls","readOnly":true}]}]}}}}'
 kubectl rollout status deployment/ml-pipeline -n "$KFP_NAMESPACE" --timeout=180s
 
 pkill -f "kubectl port-forward.*ml-pipeline.*8888" || true