|
| 1 | +// Copyright 2026 The Kubeflow Authors |
| 2 | +// |
| 3 | +// Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +// you may not use this file except in compliance with the License. |
| 5 | +// You may obtain a copy of the License at |
| 6 | +// |
| 7 | +// https://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +// |
| 9 | +// Unless required by applicable law or agreed to in writing, software |
| 10 | +// distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +// See the License for the specific language governing permissions and |
| 13 | +// limitations under the License. |
| 14 | + |
| 15 | +package mlflow |
| 16 | + |
| 17 | +import ( |
| 18 | + "bytes" |
| 19 | + "context" |
| 20 | + "encoding/json" |
| 21 | + "fmt" |
| 22 | + "io" |
| 23 | + "net/url" |
| 24 | + "os" |
| 25 | + "strings" |
| 26 | + "time" |
| 27 | + |
| 28 | + apiserverPlugins "github.com/kubeflow/pipelines/backend/src/apiserver/plugins" |
| 29 | + commonmlflow "github.com/kubeflow/pipelines/backend/src/common/mlflow" |
| 30 | + "github.com/kubeflow/pipelines/backend/src/common/util" |
| 31 | + "github.com/pkg/errors" |
| 32 | + "github.com/spf13/viper" |
| 33 | + apierrors "k8s.io/apimachinery/pkg/api/errors" |
| 34 | + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 35 | + "k8s.io/client-go/kubernetes" |
| 36 | +) |
| 37 | + |
| 38 | +const ( |
| 39 | + // DefaultExperimentName is the MLflow experiment name used when the user |
| 40 | + // and admin configuration do not specify one. |
| 41 | + DefaultExperimentName = "KFP-Default" |
| 42 | + // DefaultTimeout is the default HTTP request timeout for the MLflow client. |
| 43 | + DefaultTimeout = "30s" |
| 44 | +) |
| 45 | + |
| 46 | +// ApplySettingsDefaults applies default values to a parsed MLflowPluginSettings. |
| 47 | +func ApplySettingsDefaults(settings *commonmlflow.MLflowPluginSettings) *commonmlflow.MLflowPluginSettings { |
| 48 | + if settings == nil { |
| 49 | + settings = &commonmlflow.MLflowPluginSettings{} |
| 50 | + } |
| 51 | + if settings.WorkspacesEnabled == nil { |
| 52 | + defaultEnabled := true |
| 53 | + settings.WorkspacesEnabled = &defaultEnabled |
| 54 | + } |
| 55 | + if settings.DefaultExperimentName == "" { |
| 56 | + settings.DefaultExperimentName = DefaultExperimentName |
| 57 | + } |
| 58 | + if settings.ExperimentDescription == nil { |
| 59 | + d := DefaultExperimentDescription |
| 60 | + settings.ExperimentDescription = &d |
| 61 | + } |
| 62 | + return settings |
| 63 | +} |
| 64 | + |
| 65 | +// ResolvedConfig bundles the merged plugin configuration and its parsed settings. |
| 66 | +type ResolvedConfig struct { |
| 67 | + Settings *commonmlflow.MLflowPluginSettings |
| 68 | + Config *commonmlflow.PluginConfig |
| 69 | +} |
| 70 | + |
| 71 | +// MLflowPluginInput represents the user-facing plugins_input.mlflow schema. |
| 72 | +type MLflowPluginInput struct { |
| 73 | + ExperimentName string `json:"experiment_name,omitempty"` |
| 74 | + ExperimentID string `json:"experiment_id,omitempty"` |
| 75 | + Disabled bool `json:"disabled,omitempty"` |
| 76 | +} |
| 77 | + |
| 78 | +// KubeClientProvider abstracts Kubernetes clientset access. |
| 79 | +type KubeClientProvider interface { |
| 80 | + GetClientSet() kubernetes.Interface |
| 81 | +} |
| 82 | + |
| 83 | +// IsEnabled reports whether the global plugins.mlflow configuration is present, |
| 84 | +// indicating the API server has opted in to MLflow integration. |
| 85 | +func IsEnabled() bool { |
| 86 | + return viper.IsSet("plugins.mlflow") |
| 87 | +} |
| 88 | + |
| 89 | +// GetGlobalMLflowConfig reads the global plugins.mlflow configuration |
| 90 | +func GetGlobalMLflowConfig() (commonmlflow.PluginConfig, bool, error) { |
| 91 | + if !viper.IsSet("plugins.mlflow") { |
| 92 | + return commonmlflow.PluginConfig{}, false, nil |
| 93 | + } |
| 94 | + raw := viper.Get("plugins.mlflow") |
| 95 | + data, err := json.Marshal(raw) |
| 96 | + if err != nil { |
| 97 | + return commonmlflow.PluginConfig{}, false, util.NewInvalidInputError("failed to marshal global plugins.mlflow config: %v", err) |
| 98 | + } |
| 99 | + var cfg commonmlflow.PluginConfig |
| 100 | + if err := json.Unmarshal(data, &cfg); err != nil { |
| 101 | + return commonmlflow.PluginConfig{}, false, util.NewInvalidInputError("failed to parse global plugins.mlflow config: %v", err) |
| 102 | + } |
| 103 | + return cfg, true, nil |
| 104 | +} |
| 105 | + |
| 106 | +// GetNamespaceMLflowConfig reads the namespace-level MLflow configuration |
| 107 | +// from the kfp-launcher ConfigMap. Returns nil (no error) when the ConfigMap |
| 108 | +// or key is absent. |
| 109 | +func GetNamespaceMLflowConfig(ctx context.Context, clientSet kubernetes.Interface, namespace string) (*commonmlflow.PluginConfig, error) { |
| 110 | + if namespace == "" { |
| 111 | + return nil, util.NewInternalServerError(fmt.Errorf("namespace is empty"), "namespace must be specified when reading MLflow config") |
| 112 | + } |
| 113 | + if clientSet == nil { |
| 114 | + return nil, util.NewInternalServerError(fmt.Errorf("clientSet is nil"), "Kubernetes clientset must be provided when reading MLflow namespace config") |
| 115 | + } |
| 116 | + cm, err := clientSet.CoreV1().ConfigMaps(namespace).Get(ctx, commonmlflow.LauncherConfigMapName, v1.GetOptions{}) |
| 117 | + if err != nil { |
| 118 | + if apierrors.IsNotFound(err) { |
| 119 | + return nil, nil |
| 120 | + } |
| 121 | + return nil, util.NewInternalServerError(err, "failed to read MLflow namespace config from configmap %q in namespace %q", commonmlflow.LauncherConfigMapName, namespace) |
| 122 | + } |
| 123 | + raw, ok := cm.Data[commonmlflow.LauncherConfigKey] |
| 124 | + if !ok || raw == "" { |
| 125 | + return nil, nil |
| 126 | + } |
| 127 | + var cfg commonmlflow.PluginConfig |
| 128 | + if err := json.Unmarshal([]byte(raw), &cfg); err != nil { |
| 129 | + return nil, util.NewInternalServerError(err, "failed to parse MLflow config from key %q in configmap %q/%q", commonmlflow.LauncherConfigKey, namespace, commonmlflow.LauncherConfigMapName) |
| 130 | + } |
| 131 | + return &cfg, nil |
| 132 | +} |
| 133 | + |
| 134 | +// ResolveMLflowRequestConfig builds a merged and validated ResolvedConfig for the |
| 135 | +// given namespace, combining global and namespace-level configuration. |
| 136 | +func ResolveMLflowRequestConfig(ctx context.Context, kubeClients KubeClientProvider, namespace string) (*ResolvedConfig, error) { |
| 137 | + globalCfg, hasGlobal, err := GetGlobalMLflowConfig() |
| 138 | + if err != nil { |
| 139 | + return nil, err |
| 140 | + } |
| 141 | + if !hasGlobal { |
| 142 | + return nil, nil |
| 143 | + } |
| 144 | + |
| 145 | + namespaceCfg, err := GetNamespaceMLflowConfig(ctx, kubeClients.GetClientSet(), namespace) |
| 146 | + if err != nil { |
| 147 | + return nil, err |
| 148 | + } |
| 149 | + mergedCfg := commonmlflow.MergePluginConfig(globalCfg, namespaceCfg) |
| 150 | + if mergedCfg.Timeout == "" { |
| 151 | + mergedCfg.Timeout = DefaultTimeout |
| 152 | + } |
| 153 | + settings := ApplySettingsDefaults(mergedCfg.Settings) |
| 154 | + return &ResolvedConfig{ |
| 155 | + Settings: settings, |
| 156 | + Config: &mergedCfg, |
| 157 | + }, nil |
| 158 | +} |
| 159 | + |
| 160 | +// ResolveMLflowCredentials resolves the Kubernetes service account token used |
| 161 | +// to authenticate with the MLflow endpoint. |
| 162 | +func ResolveMLflowCredentials() (commonmlflow.MLflowCredentials, error) { |
| 163 | + restConfig, err := util.GetKubernetesConfig() |
| 164 | + if err != nil { |
| 165 | + return commonmlflow.MLflowCredentials{}, util.NewInternalServerError(err, "failed to get Kubernetes config for MLflow auth") |
| 166 | + } |
| 167 | + token := restConfig.BearerToken |
| 168 | + if token == "" && restConfig.BearerTokenFile != "" { |
| 169 | + tokenBytes, err := os.ReadFile(restConfig.BearerTokenFile) |
| 170 | + if err != nil { |
| 171 | + return commonmlflow.MLflowCredentials{}, util.NewInternalServerError(err, "failed to read bearer token file %q for MLflow auth", restConfig.BearerTokenFile) |
| 172 | + } |
| 173 | + token = strings.TrimSpace(string(tokenBytes)) |
| 174 | + } |
| 175 | + if token == "" { |
| 176 | + return commonmlflow.MLflowCredentials{}, util.NewInvalidInputError("Kubernetes bearer token is empty for MLflow auth") |
| 177 | + } |
| 178 | + return commonmlflow.MLflowCredentials{ |
| 179 | + AuthType: commonmlflow.AuthTypeKubernetes, |
| 180 | + BearerToken: token, |
| 181 | + }, nil |
| 182 | +} |
| 183 | + |
| 184 | +// BuildMLflowRequestContext constructs a fully initialized RequestContext including |
| 185 | +// the underlying MLflow HTTP client with authentication. |
| 186 | +func BuildMLflowRequestContext(ctx context.Context, namespace string, requestCfg *ResolvedConfig) (*commonmlflow.RequestContext, error) { |
| 187 | + if requestCfg == nil || requestCfg.Config == nil { |
| 188 | + return nil, util.NewInternalServerError(errors.New("MLflow config is nil"), "cannot build MLflow request context without a resolved config") |
| 189 | + } |
| 190 | + if requestCfg.Config.Endpoint == "" { |
| 191 | + return nil, util.NewInvalidInputError("plugins.mlflow endpoint must be set") |
| 192 | + } |
| 193 | + baseURL, err := url.Parse(requestCfg.Config.Endpoint) |
| 194 | + if err != nil || baseURL.Scheme == "" || baseURL.Host == "" { |
| 195 | + return nil, util.NewInvalidInputError("invalid plugins.mlflow endpoint %q", requestCfg.Config.Endpoint) |
| 196 | + } |
| 197 | + settings := requestCfg.Settings |
| 198 | + if settings == nil { |
| 199 | + return nil, util.NewInternalServerError(errors.New("MLflow plugin settings are nil"), "BuildMLflowRequestContext requires resolved settings") |
| 200 | + } |
| 201 | + timeout, err := time.ParseDuration(requestCfg.Config.Timeout) |
| 202 | + if err != nil { |
| 203 | + return nil, util.NewInvalidInputError("invalid plugins.mlflow timeout %q: %v", requestCfg.Config.Timeout, err) |
| 204 | + } |
| 205 | + if timeout <= 0 { |
| 206 | + return nil, util.NewInvalidInputError("plugins.mlflow timeout must be > 0") |
| 207 | + } |
| 208 | + authMaterial, err := ResolveMLflowCredentials() |
| 209 | + if err != nil { |
| 210 | + return nil, err |
| 211 | + } |
| 212 | + httpClient, err := commonmlflow.BuildHTTPClient(timeout, requestCfg.Config.TLS) |
| 213 | + if err != nil { |
| 214 | + return nil, err |
| 215 | + } |
| 216 | + workspacesEnabled := settings.WorkspacesEnabled != nil && *settings.WorkspacesEnabled |
| 217 | + retrySettings := commonmlflow.RetryPolicy{ |
| 218 | + InitialInterval: commonmlflow.DefaultRetryInitial, |
| 219 | + MaxInterval: commonmlflow.DefaultRetryMax, |
| 220 | + MaxElapsedTime: commonmlflow.DefaultRetryElapsed, |
| 221 | + Multiplier: 2.0, |
| 222 | + } |
| 223 | + sharedClient, err := commonmlflow.NewClient(commonmlflow.Config{ |
| 224 | + Endpoint: requestCfg.Config.Endpoint, |
| 225 | + HTTPClient: httpClient, |
| 226 | + BearerToken: authMaterial.BearerToken, |
| 227 | + WorkspacesEnabled: workspacesEnabled, |
| 228 | + Workspace: namespace, |
| 229 | + Retry: retrySettings, |
| 230 | + }) |
| 231 | + if err != nil { |
| 232 | + return nil, util.NewInvalidInputError("failed to build MLflow client: %v", err) |
| 233 | + } |
| 234 | + return &commonmlflow.RequestContext{ |
| 235 | + BaseURL: baseURL, |
| 236 | + Workspace: namespace, |
| 237 | + WorkspacesEnabled: workspacesEnabled, |
| 238 | + Client: sharedClient, |
| 239 | + }, nil |
| 240 | +} |
| 241 | + |
| 242 | +// ResolveMLflowPluginInput parses the plugins_input.mlflow JSON from a run model, |
| 243 | +// validates it against the MLflowPluginInput schema, and applies defaults. |
| 244 | +func ResolveMLflowPluginInput(pluginsInputString *string) (*MLflowPluginInput, error) { |
| 245 | + if pluginsInputString == nil || *pluginsInputString == "" { |
| 246 | + return &MLflowPluginInput{ExperimentName: DefaultExperimentName}, nil |
| 247 | + } |
| 248 | + |
| 249 | + var pluginInputs apiserverPlugins.PluginsInputMap |
| 250 | + if err := json.Unmarshal([]byte(*pluginsInputString), &pluginInputs); err != nil { |
| 251 | + return nil, util.NewInvalidInputError("plugins_input must be a valid JSON object: %v", err) |
| 252 | + } |
| 253 | + mlflowRaw, ok := pluginInputs["mlflow"] |
| 254 | + if !ok || len(mlflowRaw) == 0 { |
| 255 | + return &MLflowPluginInput{ExperimentName: DefaultExperimentName}, nil |
| 256 | + } |
| 257 | + |
| 258 | + decoder := json.NewDecoder(bytes.NewReader(mlflowRaw)) |
| 259 | + decoder.DisallowUnknownFields() |
| 260 | + input := &MLflowPluginInput{} |
| 261 | + if err := decoder.Decode(input); err != nil { |
| 262 | + return nil, util.NewInvalidInputError("plugins_input.mlflow must follow schema {experiment_name?: string, experiment_id?: string, disabled?: bool}: %v", err) |
| 263 | + } |
| 264 | + var trailing json.RawMessage |
| 265 | + if err := decoder.Decode(&trailing); err != io.EOF { |
| 266 | + return nil, util.NewInvalidInputError("plugins_input.mlflow must be a single JSON object") |
| 267 | + } |
| 268 | + |
| 269 | + if input.Disabled { |
| 270 | + return input, nil |
| 271 | + } |
| 272 | + if input.ExperimentID != "" { |
| 273 | + return input, nil |
| 274 | + } |
| 275 | + if input.ExperimentName == "" { |
| 276 | + input.ExperimentName = DefaultExperimentName |
| 277 | + } |
| 278 | + return input, nil |
| 279 | +} |
| 280 | + |
| 281 | +// SelectMLflowExperiment chooses the selector used for MLflow experiment resolution. |
| 282 | +// Priority: user-provided experiment_id > user-provided experiment_name > |
| 283 | +// admin-configured defaultExperimentName > hardcoded "KFP-Default". |
| 284 | +func SelectMLflowExperiment(input *MLflowPluginInput, settings *commonmlflow.MLflowPluginSettings) (experimentID string, experimentName string) { |
| 285 | + if input != nil { |
| 286 | + if input.ExperimentID != "" { |
| 287 | + return input.ExperimentID, "" |
| 288 | + } |
| 289 | + if input.ExperimentName != "" { |
| 290 | + return "", input.ExperimentName |
| 291 | + } |
| 292 | + } |
| 293 | + if settings != nil && settings.DefaultExperimentName != "" { |
| 294 | + return "", settings.DefaultExperimentName |
| 295 | + } |
| 296 | + return "", DefaultExperimentName |
| 297 | +} |
| 298 | + |
| 299 | +// InjectMLflowRuntimeEnv sets KFP_MLFLOW_CONFIG on driver and launcher |
| 300 | +// containers. |
| 301 | +func InjectMLflowRuntimeEnv(executionSpec util.ExecutionSpec, env map[string]string) error { |
| 302 | + if len(env) == 0 || executionSpec == nil { |
| 303 | + return nil |
| 304 | + } |
| 305 | + return executionSpec.UpsertRuntimeEnvVars(env, |
| 306 | + util.ExecutionRuntimeRoleDriver, |
| 307 | + util.ExecutionRuntimeRoleLauncher, |
| 308 | + ) |
| 309 | +} |
0 commit comments