diff --git a/api/fma/v1alpha1/launcherconfig_types.go b/api/fma/v1alpha1/launcherconfig_types.go index 5236d229..a5901cb1 100644 --- a/api/fma/v1alpha1/launcherconfig_types.go +++ b/api/fma/v1alpha1/launcherconfig_types.go @@ -21,11 +21,34 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// EmbeddedObjectMeta holds the subset of metav1.ObjectMeta fields that +// we want in the CRD schema, so that strict decoding accepts them. +type EmbeddedObjectMeta struct { + // Labels for organizing and categorizing objects. + // +optional + Labels map[string]string `json:"labels,omitempty"` + + // Annotations for storing arbitrary non-identifying metadata. + // +optional + Annotations map[string]string `json:"annotations,omitempty"` +} + +// EmbeddedPodTemplateSpec is a PodTemplateSpec whose metadata fields +// are explicitly declared so that the CRD schema admits them. +type EmbeddedPodTemplateSpec struct { + // +optional + Metadata EmbeddedObjectMeta `json:"metadata,omitempty"` + + // Spec defines the behavior of pods created from this template. + // +optional + Spec corev1.PodSpec `json:"spec,omitempty"` +} + // LauncherConfigSpec defines the configuration to manage the nominal server-providing pod definition. type LauncherConfigSpec struct { // PodTemplate defines the pod specification for the server-providing pod. // +optional - PodTemplate corev1.PodTemplateSpec `json:"podTemplate,omitempty"` + PodTemplate EmbeddedPodTemplateSpec `json:"podTemplate,omitempty"` // MaxSleepingInstances is the maximum number of sleeping inference engine instances allowed per launcher pod. // +kubebuilder:validation:Required diff --git a/api/fma/v1alpha1/zz_generated.deepcopy.go b/api/fma/v1alpha1/zz_generated.deepcopy.go index aaffe9af..ab5ceec6 100644 --- a/api/fma/v1alpha1/zz_generated.deepcopy.go +++ b/api/fma/v1alpha1/zz_generated.deepcopy.go @@ -39,6 +39,52 @@ func (in *CountForLauncher) DeepCopy() *CountForLauncher { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EmbeddedObjectMeta) DeepCopyInto(out *EmbeddedObjectMeta) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EmbeddedObjectMeta. +func (in *EmbeddedObjectMeta) DeepCopy() *EmbeddedObjectMeta { + if in == nil { + return nil + } + out := new(EmbeddedObjectMeta) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EmbeddedPodTemplateSpec) DeepCopyInto(out *EmbeddedPodTemplateSpec) { + *out = *in + in.Metadata.DeepCopyInto(&out.Metadata) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EmbeddedPodTemplateSpec. +func (in *EmbeddedPodTemplateSpec) DeepCopy() *EmbeddedPodTemplateSpec { + if in == nil { + return nil + } + out := new(EmbeddedPodTemplateSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EnhancedNodeSelector) DeepCopyInto(out *EnhancedNodeSelector) { *out = *in diff --git a/config/crd/fma.llm-d.ai_launcherconfigs.yaml b/config/crd/fma.llm-d.ai_launcherconfigs.yaml index 66bfb6a5..fb1eaff1 100644 --- a/config/crd/fma.llm-d.ai_launcherconfigs.yaml +++ b/config/crd/fma.llm-d.ai_launcherconfigs.yaml @@ -54,13 +54,24 @@ spec: properties: metadata: description: |- - Standard object's metadata. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + EmbeddedObjectMeta holds the subset of metav1.ObjectMeta fields that + we want in the CRD schema, so that strict decoding accepts them. + properties: + annotations: + additionalProperties: + type: string + description: Annotations for storing arbitrary non-identifying + metadata. + type: object + labels: + additionalProperties: + type: string + description: Labels for organizing and categorizing objects. + type: object type: object spec: - description: |- - Specification of the desired behavior of the pod. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + description: Spec defines the behavior of pods created from this + template. properties: activeDeadlineSeconds: description: |- diff --git a/docs/cluster-sharing.md b/docs/cluster-sharing.md index 76783190..f0d6fd18 100644 --- a/docs/cluster-sharing.md +++ b/docs/cluster-sharing.md @@ -29,10 +29,13 @@ object. (One design goal is to minimize chores for administrators of shared clusters.) -- As development progresses, we never change the definitions in an - existing version of the `fma.llm-d.ai` API group; we only add new +- As development progresses, we almost never change the definitions in + an existing version of the `fma.llm-d.ai` API group; we only add new versions. Old versions may be deleted only once we are sure there is no further dev/test activity using them. + We can make a change in an existing definition if the change does not + affect any current usage (e.g., add a field, change an optional thing + that does not appear in any existing YAML). - During development of a PR that adds a version of the API group, successive revisions of the PR's head branch can change the diff --git a/pkg/controller/utils/pod-helper.go b/pkg/controller/utils/pod-helper.go index e8dec125..37a9faaf 100644 --- a/pkg/controller/utils/pod-helper.go +++ b/pkg/controller/utils/pod-helper.go @@ -22,13 +22,16 @@ import ( "encoding/json" "errors" "fmt" + "maps" "regexp" "slices" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/intstr" + v1alpha1 "github.com/llm-d-incubation/llm-d-fast-model-actuation/api/fma/v1alpha1" "github.com/llm-d-incubation/llm-d-fast-model-actuation/pkg/api" "github.com/llm-d-incubation/llm-d-fast-model-actuation/pkg/controller/common" ) @@ -141,10 +144,13 @@ func IsPodReady(pod *corev1.Pod) bool { // BuildLauncherPodFromTemplate creates a launcher pod from a LauncherConfig object's // Spec.PodTemplate and assigns the built launcher pod to a node -func BuildLauncherPodFromTemplate(template corev1.PodTemplateSpec, ns, nodeName, launcherConfigName string) (*corev1.Pod, error) { +func BuildLauncherPodFromTemplate(template v1alpha1.EmbeddedPodTemplateSpec, ns, nodeName, launcherConfigName string) (*corev1.Pod, error) { pod := &corev1.Pod{ - ObjectMeta: template.ObjectMeta, - Spec: *DeIndividualize(template.Spec.DeepCopy()), + ObjectMeta: metav1.ObjectMeta{ + Labels: maps.Clone(template.Metadata.Labels), + Annotations: maps.Clone(template.Metadata.Annotations), + }, + Spec: *DeIndividualize(template.Spec.DeepCopy()), } pod.Namespace = ns pod.GenerateName = "launcher-" diff --git a/pkg/generated/applyconfiguration/fma/v1alpha1/embeddedobjectmeta.go b/pkg/generated/applyconfiguration/fma/v1alpha1/embeddedobjectmeta.go new file mode 100644 index 00000000..66c6a8b7 --- /dev/null +++ b/pkg/generated/applyconfiguration/fma/v1alpha1/embeddedobjectmeta.go @@ -0,0 +1,59 @@ +/* +Copyright 2025 The llm-d Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1alpha1 + +// EmbeddedObjectMetaApplyConfiguration represents a declarative configuration of the EmbeddedObjectMeta type for use +// with apply. +type EmbeddedObjectMetaApplyConfiguration struct { + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +// EmbeddedObjectMetaApplyConfiguration constructs a declarative configuration of the EmbeddedObjectMeta type for use with +// apply. +func EmbeddedObjectMeta() *EmbeddedObjectMetaApplyConfiguration { + return &EmbeddedObjectMetaApplyConfiguration{} +} + +// WithLabels puts the entries into the Labels field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Labels field, +// overwriting an existing map entries in Labels field with the same key. +func (b *EmbeddedObjectMetaApplyConfiguration) WithLabels(entries map[string]string) *EmbeddedObjectMetaApplyConfiguration { + if b.Labels == nil && len(entries) > 0 { + b.Labels = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.Labels[k] = v + } + return b +} + +// WithAnnotations puts the entries into the Annotations field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the Annotations field, +// overwriting an existing map entries in Annotations field with the same key. +func (b *EmbeddedObjectMetaApplyConfiguration) WithAnnotations(entries map[string]string) *EmbeddedObjectMetaApplyConfiguration { + if b.Annotations == nil && len(entries) > 0 { + b.Annotations = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.Annotations[k] = v + } + return b +} diff --git a/pkg/generated/applyconfiguration/fma/v1alpha1/embeddedpodtemplatespec.go b/pkg/generated/applyconfiguration/fma/v1alpha1/embeddedpodtemplatespec.go new file mode 100644 index 00000000..0c10fe14 --- /dev/null +++ b/pkg/generated/applyconfiguration/fma/v1alpha1/embeddedpodtemplatespec.go @@ -0,0 +1,51 @@ +/* +Copyright 2025 The llm-d Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1 "k8s.io/api/core/v1" +) + +// EmbeddedPodTemplateSpecApplyConfiguration represents a declarative configuration of the EmbeddedPodTemplateSpec type for use +// with apply. +type EmbeddedPodTemplateSpecApplyConfiguration struct { + Metadata *EmbeddedObjectMetaApplyConfiguration `json:"metadata,omitempty"` + Spec *v1.PodSpec `json:"spec,omitempty"` +} + +// EmbeddedPodTemplateSpecApplyConfiguration constructs a declarative configuration of the EmbeddedPodTemplateSpec type for use with +// apply. +func EmbeddedPodTemplateSpec() *EmbeddedPodTemplateSpecApplyConfiguration { + return &EmbeddedPodTemplateSpecApplyConfiguration{} +} + +// WithMetadata sets the Metadata field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Metadata field is set to the value of the last call. +func (b *EmbeddedPodTemplateSpecApplyConfiguration) WithMetadata(value *EmbeddedObjectMetaApplyConfiguration) *EmbeddedPodTemplateSpecApplyConfiguration { + b.Metadata = value + return b +} + +// WithSpec sets the Spec field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Spec field is set to the value of the last call. +func (b *EmbeddedPodTemplateSpecApplyConfiguration) WithSpec(value v1.PodSpec) *EmbeddedPodTemplateSpecApplyConfiguration { + b.Spec = &value + return b +} diff --git a/pkg/generated/applyconfiguration/fma/v1alpha1/launcherconfigspec.go b/pkg/generated/applyconfiguration/fma/v1alpha1/launcherconfigspec.go index 85d0d6e0..200e334d 100644 --- a/pkg/generated/applyconfiguration/fma/v1alpha1/launcherconfigspec.go +++ b/pkg/generated/applyconfiguration/fma/v1alpha1/launcherconfigspec.go @@ -17,15 +17,11 @@ limitations under the License. package v1alpha1 -import ( - v1 "k8s.io/api/core/v1" -) - // LauncherConfigSpecApplyConfiguration represents a declarative configuration of the LauncherConfigSpec type for use // with apply. type LauncherConfigSpecApplyConfiguration struct { - PodTemplate *v1.PodTemplateSpec `json:"podTemplate,omitempty"` - MaxSleepingInstances *int32 `json:"maxSleepingInstances,omitempty"` + PodTemplate *EmbeddedPodTemplateSpecApplyConfiguration `json:"podTemplate,omitempty"` + MaxSleepingInstances *int32 `json:"maxSleepingInstances,omitempty"` } // LauncherConfigSpecApplyConfiguration constructs a declarative configuration of the LauncherConfigSpec type for use with @@ -37,8 +33,8 @@ func LauncherConfigSpec() *LauncherConfigSpecApplyConfiguration { // WithPodTemplate sets the PodTemplate field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the PodTemplate field is set to the value of the last call. -func (b *LauncherConfigSpecApplyConfiguration) WithPodTemplate(value v1.PodTemplateSpec) *LauncherConfigSpecApplyConfiguration { - b.PodTemplate = &value +func (b *LauncherConfigSpecApplyConfiguration) WithPodTemplate(value *EmbeddedPodTemplateSpecApplyConfiguration) *LauncherConfigSpecApplyConfiguration { + b.PodTemplate = value return b } diff --git a/pkg/generated/applyconfiguration/utils.go b/pkg/generated/applyconfiguration/utils.go index 8d1dee3e..0db1709b 100644 --- a/pkg/generated/applyconfiguration/utils.go +++ b/pkg/generated/applyconfiguration/utils.go @@ -33,6 +33,10 @@ func ForKind(kind schema.GroupVersionKind) interface{} { // Group=fma.llm-d.ai, Version=v1alpha1 case v1alpha1.SchemeGroupVersion.WithKind("CountForLauncher"): return &fmav1alpha1.CountForLauncherApplyConfiguration{} + case v1alpha1.SchemeGroupVersion.WithKind("EmbeddedObjectMeta"): + return &fmav1alpha1.EmbeddedObjectMetaApplyConfiguration{} + case v1alpha1.SchemeGroupVersion.WithKind("EmbeddedPodTemplateSpec"): + return &fmav1alpha1.EmbeddedPodTemplateSpecApplyConfiguration{} case v1alpha1.SchemeGroupVersion.WithKind("EnhancedNodeSelector"): return &fmav1alpha1.EnhancedNodeSelectorApplyConfiguration{} case v1alpha1.SchemeGroupVersion.WithKind("InferenceServerConfig"): diff --git a/test/e2e/deploy_fma.sh b/test/e2e/deploy_fma.sh index fe6c90c6..85c6c421 100755 --- a/test/e2e/deploy_fma.sh +++ b/test/e2e/deploy_fma.sh @@ -89,8 +89,15 @@ CRD_NAMES="" for crd_file in config/crd/*.yaml; do crd_name=$(kubectl apply --dry-run=client -f "$crd_file" -o jsonpath='{.metadata.name}') CRD_NAMES="$CRD_NAMES $crd_name" - if kubectl get crd "$crd_name" &>/dev/null; then - echo " CRD $crd_name already exists, skipping" + live_spec=$(kubectl get crd "$crd_name" -o jsonpath='{.spec}' 2>/dev/null) || live_spec="" + if [ -n "$live_spec" ]; then + desired_spec=$(kubectl apply --dry-run=client -f "$crd_file" -o jsonpath='{.spec}') + if [ "$live_spec" = "$desired_spec" ]; then + echo " CRD $crd_name already exists with matching spec, skipping" + else + echo " CRD $crd_name exists but spec differs, updating" + kubectl apply --server-side -f "$crd_file" + fi else echo " Applying $crd_file ($crd_name)" kubectl apply --server-side -f "$crd_file" @@ -178,7 +185,7 @@ helm upgrade --install "$FMA_CHART_INSTANCE_NAME" charts/fma-controllers \ step "Wait for controllers to be ready" -kubectl wait --for=condition=available --timeout=120s \ +kubectl wait --for=condition=available --timeout=180s \ deployment "${FMA_CHART_INSTANCE_NAME}-dual-pods-controller" -n "$FMA_NAMESPACE" kubectl wait --for=condition=available --timeout=120s \ deployment "${FMA_CHART_INSTANCE_NAME}-launcher-populator" -n "$FMA_NAMESPACE"