|
| 1 | +#!/bin/bash |
| 2 | +# Copyright 2026 The Kubeflow Authors |
| 3 | +# |
| 4 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | +# you may not use this file except in compliance with the License. |
| 6 | +# You may obtain a copy of the License at |
| 7 | +# |
| 8 | +# https://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +# |
| 10 | +# Unless required by applicable law or agreed to in writing, software |
| 11 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +# See the License for the specific language governing permissions and |
| 14 | +# limitations under the License. |
| 15 | + |
| 16 | +# Purpose: |
| 17 | +# This script configures KFP to use an already-deployed MLflow instance for |
| 18 | +# MLflow E2E tests. |
| 19 | +# |
| 20 | +# CI helper: patch the KFP API server with plugins.mlflow, roll it out, and |
| 21 | +# port-forward the API server and MLflow so E2E tests can reach both. |
| 22 | +# It also exports workspace/auth variables used by MLflow test helpers. |
| 23 | +# |
| 24 | +# Usage: configure-mlflow.sh <KFP_NAMESPACE> <MLFLOW_NAMESPACE> <CONFIG_JSON_PATH> |
| 25 | + |
| 26 | +set -e |
| 27 | + |
| 28 | +KFP_NAMESPACE="${1:?KFP namespace required}" |
| 29 | +MLFLOW_NAMESPACE="${2:?MLflow namespace required}" |
| 30 | +CONFIG_JSON_PATH="${3:?Path to source config.json required}" |
| 31 | + |
| 32 | +echo "Services in ${MLFLOW_NAMESPACE} namespace:" |
| 33 | +kubectl get svc -n "$MLFLOW_NAMESPACE" --no-headers |
| 34 | +MLFLOW_SVC=$(kubectl get svc -n "$MLFLOW_NAMESPACE" --no-headers -o custom-columns=":metadata.name" | grep -i mlflow | head -1) |
| 35 | +if [ -z "$MLFLOW_SVC" ]; then |
| 36 | + echo "ERROR: No service matching 'mlflow' found in namespace $MLFLOW_NAMESPACE" |
| 37 | + exit 1 |
| 38 | +fi |
| 39 | +MLFLOW_PORT=$(kubectl get svc -n "$MLFLOW_NAMESPACE" "$MLFLOW_SVC" -o jsonpath='{.spec.ports[0].port}') |
| 40 | +MLFLOW_HOST="${MLFLOW_SVC}.${MLFLOW_NAMESPACE}.svc.cluster.local" |
| 41 | +MLFLOW_STATIC_PREFIX="/mlflow" |
| 42 | +MLFLOW_ENDPOINT="https://${MLFLOW_HOST}:${MLFLOW_PORT}${MLFLOW_STATIC_PREFIX}" |
| 43 | +echo "MLflow service: $MLFLOW_SVC port=$MLFLOW_PORT endpoint=$MLFLOW_ENDPOINT" |
| 44 | + |
| 45 | +MLFLOW_PATCH=$(jq -n --arg endpoint "$MLFLOW_ENDPOINT" '{ |
| 46 | + endpoint: $endpoint, |
| 47 | + tls: { insecureSkipVerify: true }, |
| 48 | + settings: { workspacesEnabled: true } |
| 49 | +}') |
| 50 | + |
| 51 | +jq --argjson mlflow "$MLFLOW_PATCH" '. + { plugins: { mlflow: $mlflow } }' \ |
| 52 | + "$CONFIG_JSON_PATH" > /tmp/kfp-config.json |
| 53 | + |
| 54 | +echo "Patched config.json plugins.mlflow:" |
| 55 | +jq '.plugins.mlflow' /tmp/kfp-config.json |
| 56 | + |
| 57 | +kubectl create configmap kfp-mlflow-config -n "$KFP_NAMESPACE" \ |
| 58 | + --from-file=config.json=/tmp/kfp-config.json --dry-run=client -o yaml | kubectl apply -f - |
| 59 | +kubectl patch deployment ml-pipeline -n "$KFP_NAMESPACE" --type=strategic -p \ |
| 60 | + '{"spec":{"template":{"spec":{"volumes":[{"name":"mlflow-cfg","configMap":{"name":"kfp-mlflow-config"}}],"containers":[{"name":"ml-pipeline-api-server","volumeMounts":[{"name":"mlflow-cfg","mountPath":"/config/config.json","subPath":"config.json"}]}]}}}}' |
| 61 | +kubectl rollout status deployment/ml-pipeline -n "$KFP_NAMESPACE" --timeout=180s |
| 62 | + |
| 63 | +pkill -f "kubectl port-forward.*ml-pipeline.*8888" || true |
| 64 | +sleep 2 |
| 65 | + |
| 66 | +C_DIR="${BASH_SOURCE%/*}" |
| 67 | +"${C_DIR}/forward-port.sh" "$KFP_NAMESPACE" ml-pipeline 8888 8888 |
| 68 | + |
| 69 | +for i in $(seq 1 12); do |
| 70 | + if curl -sf http://localhost:8888/apis/v1beta1/healthz > /dev/null 2>&1; then |
| 71 | + echo "API server is healthy on localhost:8888" |
| 72 | + break |
| 73 | + fi |
| 74 | + echo "Waiting for API server to become healthy... ($i/12)" |
| 75 | + sleep 5 |
| 76 | +done |
| 77 | +curl -sf http://localhost:8888/apis/v1beta1/healthz > /dev/null 2>&1 || { |
| 78 | + echo "ERROR: API server not reachable at localhost:8888" |
| 79 | + exit 1 |
| 80 | +} |
| 81 | + |
| 82 | +SA_TOKEN=$(kubectl create token ml-pipeline -n "$KFP_NAMESPACE" --duration=1h 2>/dev/null || true) |
| 83 | +if [ -n "${GITHUB_ENV:-}" ]; then |
| 84 | + echo "MLFLOW_WORKSPACE=$KFP_NAMESPACE" >> "$GITHUB_ENV" |
| 85 | + # Later workflow steps need these to re-establish port-forward: background jobs from this step |
| 86 | + # are terminated when the step exits, so test-and-report starts kubectl port-forward again. |
| 87 | + echo "MLFLOW_PORT_FORWARD_NS=$MLFLOW_NAMESPACE" >> "$GITHUB_ENV" |
| 88 | + echo "MLFLOW_PORT_FORWARD_SVC=$MLFLOW_SVC" >> "$GITHUB_ENV" |
| 89 | + echo "MLFLOW_PORT_FORWARD_REMOTE_PORT=$MLFLOW_PORT" >> "$GITHUB_ENV" |
| 90 | + if [ -n "$SA_TOKEN" ]; then |
| 91 | + echo "MLFLOW_BEARER_TOKEN=$SA_TOKEN" >> "$GITHUB_ENV" |
| 92 | + echo "Exported MLFLOW_BEARER_TOKEN and MLFLOW_WORKSPACE for test helpers" |
| 93 | + else |
| 94 | + echo "WARNING: Could not create SA token; MLflow requests may be unauthenticated" |
| 95 | + echo "Exported MLFLOW_WORKSPACE only" |
| 96 | + fi |
| 97 | +fi |
| 98 | + |
| 99 | +kubectl port-forward -n "$MLFLOW_NAMESPACE" "svc/$MLFLOW_SVC" "8080:$MLFLOW_PORT" & |
| 100 | +sleep 3 |
| 101 | + |
| 102 | +HEALTH_URL="https://localhost:8080${MLFLOW_STATIC_PREFIX}/health" |
| 103 | +CURL_HEADERS=(-H "X-MLflow-Workspace: $KFP_NAMESPACE") |
| 104 | +[ -n "$SA_TOKEN" ] && CURL_HEADERS+=(-H "Authorization: Bearer $SA_TOKEN") |
| 105 | + |
| 106 | +STATUS=000 |
| 107 | +for i in $(seq 1 30); do |
| 108 | + STATUS=$(curl -sk -o /dev/null -w '%{http_code}' --connect-timeout 5 --max-time 10 \ |
| 109 | + "${CURL_HEADERS[@]}" "$HEALTH_URL" 2>/dev/null || echo "000") |
| 110 | + if [ "$STATUS" != "000" ] && [ "$STATUS" -lt 500 ] 2>/dev/null; then |
| 111 | + echo "MLflow backend is healthy on localhost:8080 (HTTPS, status=$STATUS)" |
| 112 | + break |
| 113 | + fi |
| 114 | + echo "Waiting for MLflow backend... ($i/30, status=$STATUS)" |
| 115 | + sleep 5 |
| 116 | +done |
| 117 | +if [ "$STATUS" = "000" ] || { [ "$STATUS" -ge 500 ] 2>/dev/null; }; then |
| 118 | + echo "ERROR: MLflow backend not healthy after 30 attempts (last status=$STATUS)" |
| 119 | + exit 1 |
| 120 | +fi |
0 commit comments