Skip to content

Commit 5c5ccb9

Browse files
committed
feat(backend): Add pipeline run lifecycle hooks
Signed-off-by: Vani Haripriya Mudadla <vmudadla@redhat.com>
1 parent 6968090 commit 5c5ccb9

165 files changed

Lines changed: 6321 additions & 427 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

backend/src/apiserver/client/kubernetes_core_fake.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
policyv1 "k8s.io/api/policy/v1"
2222
policyv1beta1 "k8s.io/api/policy/v1beta1"
2323
"k8s.io/client-go/kubernetes"
24+
k8sfake "k8s.io/client-go/kubernetes/fake"
2425
v1 "k8s.io/client-go/kubernetes/typed/core/v1"
2526
)
2627

@@ -37,8 +38,7 @@ func (c *FakeKuberneteCoreClient) PodClient(namespace string) v1.PodInterface {
3738
}
3839

3940
func (c *FakeKuberneteCoreClient) GetClientSet() kubernetes.Interface {
40-
// Return nil for fake implementation - tests that need this should use a mock
41-
return nil
41+
return k8sfake.NewClientset()
4242
}
4343

4444
func (c *FakeKuberneteCoreClient) SecretClient(namespace string) v1.SecretInterface {
@@ -71,8 +71,7 @@ func (c *FakeKubernetesCoreClientWithBadPodClient) PodClient(namespace string) v
7171
}
7272

7373
func (c *FakeKubernetesCoreClientWithBadPodClient) GetClientSet() kubernetes.Interface {
74-
// Return nil for fake implementation
75-
return nil
74+
return k8sfake.NewClientset()
7675
}
7776

7877
func (c *FakeKubernetesCoreClientWithBadPodClient) SecretClient(namespace string) v1.SecretInterface {
Lines changed: 309 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,309 @@
1+
// Copyright 2026 The Kubeflow Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package mlflow
16+
17+
import (
18+
"bytes"
19+
"context"
20+
"encoding/json"
21+
"fmt"
22+
"io"
23+
"net/url"
24+
"os"
25+
"strings"
26+
"time"
27+
28+
apiserverPlugins "github.com/kubeflow/pipelines/backend/src/apiserver/plugins"
29+
commonmlflow "github.com/kubeflow/pipelines/backend/src/common/mlflow"
30+
"github.com/kubeflow/pipelines/backend/src/common/util"
31+
"github.com/pkg/errors"
32+
"github.com/spf13/viper"
33+
apierrors "k8s.io/apimachinery/pkg/api/errors"
34+
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
35+
"k8s.io/client-go/kubernetes"
36+
)
37+
38+
const (
39+
// DefaultExperimentName is the MLflow experiment name used when the user
40+
// and admin configuration do not specify one.
41+
DefaultExperimentName = "KFP-Default"
42+
// DefaultTimeout is the default HTTP request timeout for the MLflow client.
43+
DefaultTimeout = "30s"
44+
)
45+
46+
// ApplySettingsDefaults applies default values to a parsed MLflowPluginSettings.
47+
func ApplySettingsDefaults(settings *commonmlflow.MLflowPluginSettings) *commonmlflow.MLflowPluginSettings {
48+
if settings == nil {
49+
settings = &commonmlflow.MLflowPluginSettings{}
50+
}
51+
if settings.WorkspacesEnabled == nil {
52+
defaultEnabled := true
53+
settings.WorkspacesEnabled = &defaultEnabled
54+
}
55+
if settings.DefaultExperimentName == "" {
56+
settings.DefaultExperimentName = DefaultExperimentName
57+
}
58+
if settings.ExperimentDescription == nil {
59+
d := DefaultExperimentDescription
60+
settings.ExperimentDescription = &d
61+
}
62+
return settings
63+
}
64+
65+
// ResolvedConfig bundles the merged plugin configuration and its parsed settings.
66+
type ResolvedConfig struct {
67+
Settings *commonmlflow.MLflowPluginSettings
68+
Config *commonmlflow.PluginConfig
69+
}
70+
71+
// MLflowPluginInput represents the user-facing plugins_input.mlflow schema.
72+
type MLflowPluginInput struct {
73+
ExperimentName string `json:"experiment_name,omitempty"`
74+
ExperimentID string `json:"experiment_id,omitempty"`
75+
Disabled bool `json:"disabled,omitempty"`
76+
}
77+
78+
// KubeClientProvider abstracts Kubernetes clientset access.
79+
type KubeClientProvider interface {
80+
GetClientSet() kubernetes.Interface
81+
}
82+
83+
// IsEnabled reports whether the global plugins.mlflow configuration is present,
84+
// indicating the API server has opted in to MLflow integration.
85+
func IsEnabled() bool {
86+
return viper.IsSet("plugins.mlflow")
87+
}
88+
89+
// GetGlobalMLflowConfig reads the global plugins.mlflow configuration
90+
func GetGlobalMLflowConfig() (commonmlflow.PluginConfig, bool, error) {
91+
if !viper.IsSet("plugins.mlflow") {
92+
return commonmlflow.PluginConfig{}, false, nil
93+
}
94+
raw := viper.Get("plugins.mlflow")
95+
data, err := json.Marshal(raw)
96+
if err != nil {
97+
return commonmlflow.PluginConfig{}, false, util.NewInvalidInputError("failed to marshal global plugins.mlflow config: %v", err)
98+
}
99+
var cfg commonmlflow.PluginConfig
100+
if err := json.Unmarshal(data, &cfg); err != nil {
101+
return commonmlflow.PluginConfig{}, false, util.NewInvalidInputError("failed to parse global plugins.mlflow config: %v", err)
102+
}
103+
return cfg, true, nil
104+
}
105+
106+
// GetNamespaceMLflowConfig reads the namespace-level MLflow configuration
107+
// from the kfp-launcher ConfigMap. Returns nil (no error) when the ConfigMap
108+
// or key is absent.
109+
func GetNamespaceMLflowConfig(ctx context.Context, clientSet kubernetes.Interface, namespace string) (*commonmlflow.PluginConfig, error) {
110+
if namespace == "" {
111+
return nil, util.NewInternalServerError(fmt.Errorf("namespace is empty"), "namespace must be specified when reading MLflow config")
112+
}
113+
if clientSet == nil {
114+
return nil, util.NewInternalServerError(fmt.Errorf("clientSet is nil"), "Kubernetes clientset must be provided when reading MLflow namespace config")
115+
}
116+
cm, err := clientSet.CoreV1().ConfigMaps(namespace).Get(ctx, commonmlflow.LauncherConfigMapName, v1.GetOptions{})
117+
if err != nil {
118+
if apierrors.IsNotFound(err) {
119+
return nil, nil
120+
}
121+
return nil, util.NewInternalServerError(err, "failed to read MLflow namespace config from configmap %q in namespace %q", commonmlflow.LauncherConfigMapName, namespace)
122+
}
123+
raw, ok := cm.Data[commonmlflow.LauncherConfigKey]
124+
if !ok || raw == "" {
125+
return nil, nil
126+
}
127+
var cfg commonmlflow.PluginConfig
128+
if err := json.Unmarshal([]byte(raw), &cfg); err != nil {
129+
return nil, util.NewInternalServerError(err, "failed to parse MLflow config from key %q in configmap %q/%q", commonmlflow.LauncherConfigKey, namespace, commonmlflow.LauncherConfigMapName)
130+
}
131+
return &cfg, nil
132+
}
133+
134+
// ResolveMLflowRequestConfig builds a merged and validated ResolvedConfig for the
135+
// given namespace, combining global and namespace-level configuration.
136+
func ResolveMLflowRequestConfig(ctx context.Context, kubeClients KubeClientProvider, namespace string) (*ResolvedConfig, error) {
137+
globalCfg, hasGlobal, err := GetGlobalMLflowConfig()
138+
if err != nil {
139+
return nil, err
140+
}
141+
if !hasGlobal {
142+
return nil, nil
143+
}
144+
145+
namespaceCfg, err := GetNamespaceMLflowConfig(ctx, kubeClients.GetClientSet(), namespace)
146+
if err != nil {
147+
return nil, err
148+
}
149+
mergedCfg := commonmlflow.MergePluginConfig(globalCfg, namespaceCfg)
150+
if mergedCfg.Timeout == "" {
151+
mergedCfg.Timeout = DefaultTimeout
152+
}
153+
settings := ApplySettingsDefaults(mergedCfg.Settings)
154+
return &ResolvedConfig{
155+
Settings: settings,
156+
Config: &mergedCfg,
157+
}, nil
158+
}
159+
160+
// ResolveMLflowCredentials resolves the Kubernetes service account token used
161+
// to authenticate with the MLflow endpoint.
162+
func ResolveMLflowCredentials() (commonmlflow.MLflowCredentials, error) {
163+
restConfig, err := util.GetKubernetesConfig()
164+
if err != nil {
165+
return commonmlflow.MLflowCredentials{}, util.NewInternalServerError(err, "failed to get Kubernetes config for MLflow auth")
166+
}
167+
token := restConfig.BearerToken
168+
if token == "" && restConfig.BearerTokenFile != "" {
169+
tokenBytes, err := os.ReadFile(restConfig.BearerTokenFile)
170+
if err != nil {
171+
return commonmlflow.MLflowCredentials{}, util.NewInternalServerError(err, "failed to read bearer token file %q for MLflow auth", restConfig.BearerTokenFile)
172+
}
173+
token = strings.TrimSpace(string(tokenBytes))
174+
}
175+
if token == "" {
176+
return commonmlflow.MLflowCredentials{}, util.NewInvalidInputError("Kubernetes bearer token is empty for MLflow auth")
177+
}
178+
return commonmlflow.MLflowCredentials{
179+
AuthType: commonmlflow.AuthTypeKubernetes,
180+
BearerToken: token,
181+
}, nil
182+
}
183+
184+
// BuildMLflowRequestContext constructs a fully initialized RequestContext including
185+
// the underlying MLflow HTTP client with authentication.
186+
func BuildMLflowRequestContext(ctx context.Context, namespace string, requestCfg *ResolvedConfig) (*commonmlflow.RequestContext, error) {
187+
if requestCfg == nil || requestCfg.Config == nil {
188+
return nil, util.NewInternalServerError(errors.New("MLflow config is nil"), "cannot build MLflow request context without a resolved config")
189+
}
190+
if requestCfg.Config.Endpoint == "" {
191+
return nil, util.NewInvalidInputError("plugins.mlflow endpoint must be set")
192+
}
193+
baseURL, err := url.Parse(requestCfg.Config.Endpoint)
194+
if err != nil || baseURL.Scheme == "" || baseURL.Host == "" {
195+
return nil, util.NewInvalidInputError("invalid plugins.mlflow endpoint %q", requestCfg.Config.Endpoint)
196+
}
197+
settings := requestCfg.Settings
198+
if settings == nil {
199+
return nil, util.NewInternalServerError(errors.New("MLflow plugin settings are nil"), "BuildMLflowRequestContext requires resolved settings")
200+
}
201+
timeout, err := time.ParseDuration(requestCfg.Config.Timeout)
202+
if err != nil {
203+
return nil, util.NewInvalidInputError("invalid plugins.mlflow timeout %q: %v", requestCfg.Config.Timeout, err)
204+
}
205+
if timeout <= 0 {
206+
return nil, util.NewInvalidInputError("plugins.mlflow timeout must be > 0")
207+
}
208+
authMaterial, err := ResolveMLflowCredentials()
209+
if err != nil {
210+
return nil, err
211+
}
212+
httpClient, err := commonmlflow.BuildHTTPClient(timeout, requestCfg.Config.TLS)
213+
if err != nil {
214+
return nil, err
215+
}
216+
workspacesEnabled := settings.WorkspacesEnabled != nil && *settings.WorkspacesEnabled
217+
retrySettings := commonmlflow.RetryPolicy{
218+
InitialInterval: commonmlflow.DefaultRetryInitial,
219+
MaxInterval: commonmlflow.DefaultRetryMax,
220+
MaxElapsedTime: commonmlflow.DefaultRetryElapsed,
221+
Multiplier: 2.0,
222+
}
223+
sharedClient, err := commonmlflow.NewClient(commonmlflow.Config{
224+
Endpoint: requestCfg.Config.Endpoint,
225+
HTTPClient: httpClient,
226+
BearerToken: authMaterial.BearerToken,
227+
WorkspacesEnabled: workspacesEnabled,
228+
Workspace: namespace,
229+
Retry: retrySettings,
230+
})
231+
if err != nil {
232+
return nil, util.NewInvalidInputError("failed to build MLflow client: %v", err)
233+
}
234+
return &commonmlflow.RequestContext{
235+
BaseURL: baseURL,
236+
Workspace: namespace,
237+
WorkspacesEnabled: workspacesEnabled,
238+
Client: sharedClient,
239+
}, nil
240+
}
241+
242+
// ResolveMLflowPluginInput parses the plugins_input.mlflow JSON from a run model,
243+
// validates it against the MLflowPluginInput schema, and applies defaults.
244+
func ResolveMLflowPluginInput(pluginsInputString *string) (*MLflowPluginInput, error) {
245+
if pluginsInputString == nil || *pluginsInputString == "" {
246+
return &MLflowPluginInput{ExperimentName: DefaultExperimentName}, nil
247+
}
248+
249+
var pluginInputs apiserverPlugins.PluginsInputMap
250+
if err := json.Unmarshal([]byte(*pluginsInputString), &pluginInputs); err != nil {
251+
return nil, util.NewInvalidInputError("plugins_input must be a valid JSON object: %v", err)
252+
}
253+
mlflowRaw, ok := pluginInputs["mlflow"]
254+
if !ok || len(mlflowRaw) == 0 {
255+
return &MLflowPluginInput{ExperimentName: DefaultExperimentName}, nil
256+
}
257+
258+
decoder := json.NewDecoder(bytes.NewReader(mlflowRaw))
259+
decoder.DisallowUnknownFields()
260+
input := &MLflowPluginInput{}
261+
if err := decoder.Decode(input); err != nil {
262+
return nil, util.NewInvalidInputError("plugins_input.mlflow must follow schema {experiment_name?: string, experiment_id?: string, disabled?: bool}: %v", err)
263+
}
264+
var trailing json.RawMessage
265+
if err := decoder.Decode(&trailing); err != io.EOF {
266+
return nil, util.NewInvalidInputError("plugins_input.mlflow must be a single JSON object")
267+
}
268+
269+
if input.Disabled {
270+
return input, nil
271+
}
272+
if input.ExperimentID != "" {
273+
return input, nil
274+
}
275+
if input.ExperimentName == "" {
276+
input.ExperimentName = DefaultExperimentName
277+
}
278+
return input, nil
279+
}
280+
281+
// SelectMLflowExperiment chooses the selector used for MLflow experiment resolution.
282+
// Priority: user-provided experiment_id > user-provided experiment_name >
283+
// admin-configured defaultExperimentName > hardcoded "KFP-Default".
284+
func SelectMLflowExperiment(input *MLflowPluginInput, settings *commonmlflow.MLflowPluginSettings) (experimentID string, experimentName string) {
285+
if input != nil {
286+
if input.ExperimentID != "" {
287+
return input.ExperimentID, ""
288+
}
289+
if input.ExperimentName != "" {
290+
return "", input.ExperimentName
291+
}
292+
}
293+
if settings != nil && settings.DefaultExperimentName != "" {
294+
return "", settings.DefaultExperimentName
295+
}
296+
return "", DefaultExperimentName
297+
}
298+
299+
// InjectMLflowRuntimeEnv sets KFP_MLFLOW_CONFIG on driver and launcher
300+
// containers.
301+
func InjectMLflowRuntimeEnv(executionSpec util.ExecutionSpec, env map[string]string) error {
302+
if len(env) == 0 || executionSpec == nil {
303+
return nil
304+
}
305+
return executionSpec.UpsertRuntimeEnvVars(env,
306+
util.ExecutionRuntimeRoleDriver,
307+
util.ExecutionRuntimeRoleLauncher,
308+
)
309+
}

0 commit comments

Comments
 (0)