Skip to content

Commit e544dc8

Browse files
mimowomimowo
authored
Use annotation rather than label for serving workloads (#3815)
1 parent 9770707 commit e544dc8

File tree

7 files changed

+38
-18
lines changed

7 files changed

+38
-18
lines changed

‎pkg/controller/jobs/deployment/deployment_webhook.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,13 @@ func (wh *Webhook) Default(ctx context.Context, obj runtime.Object) error {
7474
return err
7575
}
7676
if suspend {
77+
if deployment.Spec.Template.Annotations == nil {
78+
deployment.Spec.Template.Annotations = make(map[string]string, 1)
79+
}
80+
deployment.Spec.Template.Annotations[pod.SuspendedByParentAnnotation] = FrameworkName
7781
if deployment.Spec.Template.Labels == nil {
78-
deployment.Spec.Template.Labels = make(map[string]string, 2)
82+
deployment.Spec.Template.Labels = make(map[string]string, 1)
7983
}
80-
deployment.Spec.Template.Labels[pod.SuspendedByParentLabelKey] = FrameworkName
8184
queueName := jobframework.QueueNameForObject(deployment.Object())
8285
if queueName != "" {
8386
deployment.Spec.Template.Labels[constants.QueueLabel] = queueName

‎pkg/controller/jobs/deployment/deployment_webhook_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ func TestDefault(t *testing.T) {
5252
want: testingdeployment.MakeDeployment("test-pod", "").
5353
Queue("test-queue").
5454
PodTemplateSpecQueue("test-queue").
55-
PodTemplateSpecLabel(pod.SuspendedByParentLabelKey, FrameworkName).
55+
PodTemplateAnnotation(pod.SuspendedByParentAnnotation, FrameworkName).
5656
Obj(),
5757
},
5858
"deployment with queue and pod template spec queue": {
@@ -63,7 +63,7 @@ func TestDefault(t *testing.T) {
6363
want: testingdeployment.MakeDeployment("test-pod", "").
6464
Queue("new-test-queue").
6565
PodTemplateSpecQueue("new-test-queue").
66-
PodTemplateSpecLabel(pod.SuspendedByParentLabelKey, FrameworkName).
66+
PodTemplateAnnotation(pod.SuspendedByParentAnnotation, FrameworkName).
6767
Obj(),
6868
},
6969
"deployment without queue with pod template spec queue": {
@@ -77,7 +77,7 @@ func TestDefault(t *testing.T) {
7777
want: testingdeployment.MakeDeployment("test-pod", "default").
7878
Queue("default").
7979
PodTemplateSpecQueue("default").
80-
PodTemplateSpecLabel(pod.SuspendedByParentLabelKey, FrameworkName).
80+
PodTemplateAnnotation(pod.SuspendedByParentAnnotation, FrameworkName).
8181
Obj(),
8282
},
8383
"LocalQueueDefaulting enabled, default lq is created, job has queue label": {
@@ -87,7 +87,7 @@ func TestDefault(t *testing.T) {
8787
want: testingdeployment.MakeDeployment("test-pod", "").
8888
Queue("test-queue").
8989
PodTemplateSpecQueue("test-queue").
90-
PodTemplateSpecLabel(pod.SuspendedByParentLabelKey, FrameworkName).
90+
PodTemplateAnnotation(pod.SuspendedByParentAnnotation, FrameworkName).
9191
Obj(),
9292
},
9393
"LocalQueueDefaulting enabled, default lq isn't created, job doesn't have queue label": {

‎pkg/controller/jobs/pod/pod_webhook.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,11 +49,11 @@ const (
4949
ManagedLabelKey = constants.ManagedByKueueLabel
5050
ManagedLabelValue = "true"
5151
PodFinalizer = ManagedLabelKey
52-
SuspendedByParentLabelKey = "kueue.x-k8s.io/pod-suspending-parent"
5352
GroupNameLabel = "kueue.x-k8s.io/pod-group-name"
5453
GroupTotalCountAnnotation = "kueue.x-k8s.io/pod-group-total-count"
5554
GroupFastAdmissionAnnotation = "kueue.x-k8s.io/pod-group-fast-admission"
5655
GroupServingAnnotation = "kueue.x-k8s.io/pod-group-serving"
56+
SuspendedByParentAnnotation = "kueue.x-k8s.io/pod-suspending-parent"
5757
RoleHashAnnotation = "kueue.x-k8s.io/role-hash"
5858
RetriableInGroupAnnotation = "kueue.x-k8s.io/retriable-in-group"
5959
)
@@ -152,7 +152,7 @@ func (w *PodWebhook) Default(ctx context.Context, obj runtime.Object) error {
152152
log := ctrl.LoggerFrom(ctx).WithName("pod-webhook")
153153
log.V(5).Info("Applying defaults")
154154

155-
_, suspend := pod.pod.GetLabels()[SuspendedByParentLabelKey]
155+
_, suspend := pod.pod.GetAnnotations()[SuspendedByParentAnnotation]
156156
if !suspend {
157157
// Namespace filtering
158158
ns := corev1.Namespace{}

‎pkg/controller/jobs/statefulset/statefulset_webhook.go‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -75,18 +75,17 @@ func (wh *Webhook) Default(ctx context.Context, obj runtime.Object) error {
7575
return err
7676
}
7777
if suspend {
78+
if ss.Spec.Template.Annotations == nil {
79+
ss.Spec.Template.Annotations = make(map[string]string, 1)
80+
}
81+
ss.Spec.Template.Annotations[pod.SuspendedByParentAnnotation] = FrameworkName
7882
if ss.Spec.Template.Labels == nil {
79-
ss.Spec.Template.Labels = make(map[string]string, 3)
83+
ss.Spec.Template.Labels = make(map[string]string, 2)
8084
}
81-
ss.Spec.Template.Labels[pod.SuspendedByParentLabelKey] = FrameworkName
8285
queueName := jobframework.QueueNameForObject(ss.Object())
8386
if queueName != "" {
8487
ss.Spec.Template.Labels[constants.QueueLabel] = queueName
8588
ss.Spec.Template.Labels[pod.GroupNameLabel] = GetWorkloadName(ss.Name)
86-
87-
if ss.Spec.Template.Annotations == nil {
88-
ss.Spec.Template.Annotations = make(map[string]string, 4)
89-
}
9089
ss.Spec.Template.Annotations[pod.GroupTotalCountAnnotation] = fmt.Sprint(ptr.Deref(ss.Spec.Replicas, 1))
9190
ss.Spec.Template.Annotations[pod.GroupFastAdmissionAnnotation] = "true"
9291
ss.Spec.Template.Annotations[pod.GroupServingAnnotation] = "true"

‎pkg/controller/jobs/statefulset/statefulset_webhook_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ func TestDefault(t *testing.T) {
5858
Replicas(10).
5959
Queue("test-queue").
6060
PodTemplateSpecQueue("test-queue").
61-
PodTemplateSpecLabel(pod.SuspendedByParentLabelKey, FrameworkName).
61+
PodTemplateAnnotation(pod.SuspendedByParentAnnotation, FrameworkName).
6262
PodTemplateSpecPodGroupNameLabel("test-pod", "", gvk).
6363
PodTemplateSpecPodGroupTotalCountAnnotation(10).
6464
PodTemplateSpecPodGroupFastAdmissionAnnotation(true).
@@ -76,7 +76,7 @@ func TestDefault(t *testing.T) {
7676
PodTemplateSpecPodGroupNameLabel("test-pod", "", gvk).
7777
PodTemplateSpecPodGroupTotalCountAnnotation(1).
7878
PodTemplateSpecQueue("test-queue").
79-
PodTemplateSpecLabel(pod.SuspendedByParentLabelKey, FrameworkName).
79+
PodTemplateAnnotation(pod.SuspendedByParentAnnotation, FrameworkName).
8080
PodTemplateSpecPodGroupFastAdmissionAnnotation(true).
8181
PodTemplateSpecPodGroupServingAnnotation(true).
8282
PodTemplateSpecPodGroupPodIndexLabelAnnotation(appsv1.PodIndexLabel).
@@ -89,7 +89,7 @@ func TestDefault(t *testing.T) {
8989
want: testingstatefulset.MakeStatefulSet("test-pod", "default").
9090
Queue("default").
9191
PodTemplateSpecQueue("default").
92-
PodTemplateSpecLabel(pod.SuspendedByParentLabelKey, FrameworkName).
92+
PodTemplateAnnotation(pod.SuspendedByParentAnnotation, FrameworkName).
9393
PodTemplateSpecPodGroupNameLabel("test-pod", "", gvk).
9494
PodTemplateSpecPodGroupTotalCountAnnotation(1).
9595
PodTemplateSpecPodGroupFastAdmissionAnnotation(true).
@@ -104,7 +104,7 @@ func TestDefault(t *testing.T) {
104104
want: testingstatefulset.MakeStatefulSet("test-pod", "").
105105
Queue("test-queue").
106106
PodTemplateSpecQueue("test-queue").
107-
PodTemplateSpecLabel(pod.SuspendedByParentLabelKey, FrameworkName).
107+
PodTemplateAnnotation(pod.SuspendedByParentAnnotation, FrameworkName).
108108
PodTemplateSpecPodGroupNameLabel("test-pod", "", gvk).
109109
PodTemplateSpecPodGroupTotalCountAnnotation(1).
110110
PodTemplateSpecPodGroupFastAdmissionAnnotation(true).

‎pkg/util/testingjobs/deployment/wrappers.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,15 @@ func (d *DeploymentWrapper) PodTemplateSpecLabel(k, v string) *DeploymentWrapper
128128
return d
129129
}
130130

131+
// PodTemplateAnnotation sets the annotation of the pod template
132+
func (d *DeploymentWrapper) PodTemplateAnnotation(k, v string) *DeploymentWrapper {
133+
if d.Spec.Template.Annotations == nil {
134+
d.Spec.Template.Annotations = make(map[string]string, 1)
135+
}
136+
d.Spec.Template.Annotations[k] = v
137+
return d
138+
}
139+
131140
// PodTemplateSpecQueue updates the queue name of the pod template spec of the Deployment
132141
func (d *DeploymentWrapper) PodTemplateSpecQueue(q string) *DeploymentWrapper {
133142
return d.PodTemplateSpecLabel(constants.QueueLabel, q)

‎pkg/util/testingjobs/statefulset/wrappers.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,15 @@ func (ss *StatefulSetWrapper) PodTemplateSpecLabel(k, v string) *StatefulSetWrap
106106
return ss
107107
}
108108

109+
// PodTemplateAnnotation sets the annotation of the pod template
110+
func (ss *StatefulSetWrapper) PodTemplateAnnotation(k, v string) *StatefulSetWrapper {
111+
if ss.Spec.Template.Annotations == nil {
112+
ss.Spec.Template.Annotations = make(map[string]string, 1)
113+
}
114+
ss.Spec.Template.Annotations[k] = v
115+
return ss
116+
}
117+
109118
// PodTemplateSpecAnnotation sets the annotation of the pod template spec of the StatefulSet
110119
func (ss *StatefulSetWrapper) PodTemplateSpecAnnotation(k, v string) *StatefulSetWrapper {
111120
if ss.Spec.Template.Annotations == nil {

0 commit comments

Comments
 (0)