Skip to content

Commit 94c3cfc

Browse files
dhaifleydhaifley
authored
feat(deployer): enable zero downtime rollout, graceful shutdown by default (#12242)
Signed-off-by: David Haifley <[email protected]>
1 parent 488d33a commit 94c3cfc

File tree

29 files changed

+401
-232
lines changed

29 files changed

+401
-232
lines changed

‎.github/workflows/pr-kubernetes-tests.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ jobs:
5858
# August 29, 2025: ~9 minutes
5959
- cluster-name: 'cluster-six'
6060
go-test-args: '-v -timeout=25m'
61-
go-test-run-regex: '^TestInferenceExtension$$|^TestListenerSet$$|^TestKgateway$$/^TCPRouteServices$$|^TestKgatewayIstioAutoMtls$$'
61+
go-test-run-regex: '^TestInferenceExtension$$|^TestListenerSet$$|^TestKgateway$$/^TCPRouteServices$$|^TestKgatewayIstioAutoMtls$$|^TestZeroDowntimeRollout$$'
6262
localstack: 'false'
6363
# August 29, 2025: ~7 minutes
6464
- cluster-name: 'cluster-ai'

‎api/applyconfiguration/api/v1alpha1/pod.go‎

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/applyconfiguration/internal/internal.go‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/v1alpha1/kube_types.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,14 @@ type Pod struct {
273273
// +kubebuilder:validation:Maximum=31536000
274274
TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
275275

276+
// If specified, the pod's startup probe. A probe of container startup readiness.
277+
// Container will be only be added to service endpoints if the probe succeeds. See
278+
// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#probe-v1-core
279+
// for details.
280+
//
281+
// +optional
282+
StartupProbe *corev1.Probe `json:"startupProbe,omitempty"`
283+
276284
// If specified, the pod's readiness probe. Periodic probe of container service readiness.
277285
// Container will be removed from service endpoints if the probe fails. See
278286
// https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#probe-v1-core
@@ -353,6 +361,13 @@ func (in *Pod) GetTolerations() []corev1.Toleration {
353361
return in.Tolerations
354362
}
355363

364+
func (in *Pod) GetStartupProbe() *corev1.Probe {
365+
if in == nil {
366+
return nil
367+
}
368+
return in.StartupProbe
369+
}
370+
356371
func (in *Pod) GetReadinessProbe() *corev1.Probe {
357372
if in == nil {
358373
return nil

‎api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎install/helm/kgateway-crds/templates/gateway.kgateway.dev_gatewayparameters.yaml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3079,6 +3079,87 @@ spec:
30793079
type: string
30803080
type: object
30813081
type: object
3082+
startupProbe:
3083+
properties:
3084+
exec:
3085+
properties:
3086+
command:
3087+
items:
3088+
type: string
3089+
type: array
3090+
x-kubernetes-list-type: atomic
3091+
type: object
3092+
failureThreshold:
3093+
format: int32
3094+
type: integer
3095+
grpc:
3096+
properties:
3097+
port:
3098+
format: int32
3099+
type: integer
3100+
service:
3101+
default: ""
3102+
type: string
3103+
required:
3104+
- port
3105+
type: object
3106+
httpGet:
3107+
properties:
3108+
host:
3109+
type: string
3110+
httpHeaders:
3111+
items:
3112+
properties:
3113+
name:
3114+
type: string
3115+
value:
3116+
type: string
3117+
required:
3118+
- name
3119+
- value
3120+
type: object
3121+
type: array
3122+
x-kubernetes-list-type: atomic
3123+
path:
3124+
type: string
3125+
port:
3126+
anyOf:
3127+
- type: integer
3128+
- type: string
3129+
x-kubernetes-int-or-string: true
3130+
scheme:
3131+
type: string
3132+
required:
3133+
- port
3134+
type: object
3135+
initialDelaySeconds:
3136+
format: int32
3137+
type: integer
3138+
periodSeconds:
3139+
format: int32
3140+
type: integer
3141+
successThreshold:
3142+
format: int32
3143+
type: integer
3144+
tcpSocket:
3145+
properties:
3146+
host:
3147+
type: string
3148+
port:
3149+
anyOf:
3150+
- type: integer
3151+
- type: string
3152+
x-kubernetes-int-or-string: true
3153+
required:
3154+
- port
3155+
type: object
3156+
terminationGracePeriodSeconds:
3157+
format: int64
3158+
type: integer
3159+
timeoutSeconds:
3160+
format: int32
3161+
type: integer
3162+
type: object
30823163
terminationGracePeriodSeconds:
30833164
format: int64
30843165
maximum: 31536000

‎internal/kgateway/deployer/gateway_parameters.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,7 @@ func (k *kGatewayParameters) getValues(gw *api.Gateway, gwParam *v1alpha1.Gatewa
404404
gateway.NodeSelector = podConfig.GetNodeSelector()
405405
gateway.Affinity = podConfig.GetAffinity()
406406
gateway.Tolerations = podConfig.GetTolerations()
407+
gateway.StartupProbe = podConfig.GetStartupProbe()
407408
gateway.ReadinessProbe = podConfig.GetReadinessProbe()
408409
gateway.LivenessProbe = podConfig.GetLivenessProbe()
409410
gateway.GracefulShutdown = podConfig.GetGracefulShutdown()

‎internal/kgateway/helm/kgateway/templates/gateway/agent-gateway-deployment.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,27 @@ spec:
5151
securityContext:
5252
{{- toYaml $gateway.securityContext | nindent 12 }}
5353
{{- end }} {{/* if $gateway.securityContext */}}
54+
{{- if $gateway.startupProbe }}
55+
startupProbe:
56+
{{- toYaml $gateway.startupProbe | nindent 12 }}
57+
{{- end }}{{/* if $gateway.startupProbe */}}
58+
{{- if $gateway.readinessProbe }}
59+
readinessProbe:
60+
{{- toYaml $gateway.readinessProbe | nindent 12 }}
61+
{{- end }}{{/* if $gateway.readinessProbe */}}
62+
{{- if $gateway.livenessProbe }}
63+
livenessProbe:
64+
{{- toYaml $gateway.livenessProbe | nindent 12 }}
65+
{{- end }}{{/* if $gateway.livenessProbe */}}
66+
{{- if ($gateway.gracefulShutdown).enabled }}
67+
lifecycle:
68+
preStop:
69+
exec:
70+
command:
71+
- /bin/sh
72+
- -c
73+
- wget --post-data "" -O /dev/null 127.0.0.1:15000/quitquitquit; sleep {{ $gateway.gracefulShutdown.sleepTimeSeconds | default "10" }}
74+
{{- end}}{{/* if ($gateway.gracefulShutdown).enabled */}}
5475
{{- if $gateway.resources }}
5576
resources:
5677
{{- toYaml $gateway.resources | nindent 12 }}

‎internal/kgateway/helm/kgateway/templates/gateway/proxy-deployment.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,10 @@ spec:
101101
- name: http-monitoring
102102
containerPort: 9091
103103
{{- end }}
104+
{{- if $gateway.startupProbe }}
105+
startupProbe:
106+
{{ toYaml $gateway.startupProbe | indent 10}}
107+
{{- end }}{{/*if $gateway.startupProbe*/}}
104108
{{- if $gateway.readinessProbe }}
105109
readinessProbe:
106110
{{ toYaml $gateway.readinessProbe | indent 10}}

‎pkg/deployer/deployer_test.go‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@ import (
3232
api "sigs.k8s.io/gateway-api/apis/v1"
3333
apixv1a1 "sigs.k8s.io/gateway-api/apisx/v1alpha1"
3434

35-
"github.com/kgateway-dev/kgateway/v2/test/gomega/matchers"
36-
3735
"github.com/kgateway-dev/kgateway/v2/api/settings"
3836
gw2_v1alpha1 "github.com/kgateway-dev/kgateway/v2/api/v1alpha1"
3937
"github.com/kgateway-dev/kgateway/v2/internal/kgateway/controller"
@@ -48,6 +46,7 @@ import (
4846
"github.com/kgateway-dev/kgateway/v2/pkg/pluginsdk/collections"
4947
"github.com/kgateway-dev/kgateway/v2/pkg/pluginsdk/krtutil"
5048
"github.com/kgateway-dev/kgateway/v2/pkg/schemes"
49+
"github.com/kgateway-dev/kgateway/v2/test/gomega/matchers"
5150

5251
// TODO BML tests in this suite fail if this no-op import is not imported first.
5352
//
@@ -1888,6 +1887,7 @@ var _ = Describe("Deployer", func() {
18881887
params := fullyDefinedGatewayParameters(wellknown.DefaultGatewayParametersName, defaultNamespace)
18891888
params.Spec.Kube.PodTemplate.LivenessProbe = generateLivenessProbe()
18901889
params.Spec.Kube.PodTemplate.ReadinessProbe = generateReadinessProbe()
1890+
params.Spec.Kube.PodTemplate.StartupProbe = generateStartupProbe()
18911891
params.Spec.Kube.PodTemplate.TerminationGracePeriodSeconds = ptr.To(int64(5))
18921892
params.Spec.Kube.PodTemplate.GracefulShutdown = &gw2_v1alpha1.GracefulShutdownSpec{
18931893
Enabled: ptr.To(true),
@@ -2225,6 +2225,7 @@ var _ = Describe("Deployer", func() {
22252225
envoyContainer := dep.Spec.Template.Spec.Containers[0]
22262226
Expect(envoyContainer.LivenessProbe).To(BeEquivalentTo(generateLivenessProbe()))
22272227
Expect(envoyContainer.ReadinessProbe).To(BeEquivalentTo(generateReadinessProbe()))
2228+
Expect(envoyContainer.StartupProbe).To(BeEquivalentTo(generateStartupProbe()))
22282229
Expect(envoyContainer.Lifecycle.PreStop.Exec.Command).To(BeEquivalentTo([]string{
22292230
"/bin/sh",
22302231
"-c",
@@ -3271,20 +3272,32 @@ func generateLivenessProbe() *corev1.Probe {
32713272
}
32723273
}
32733274

3275+
func generateStartupProbe() *corev1.Probe {
3276+
return &corev1.Probe{
3277+
ProbeHandler: corev1.ProbeHandler{
3278+
HTTPGet: &corev1.HTTPGetAction{
3279+
Path: "/ready",
3280+
Port: intstr.FromInt(8082),
3281+
},
3282+
},
3283+
InitialDelaySeconds: 0,
3284+
PeriodSeconds: 1,
3285+
TimeoutSeconds: 2,
3286+
FailureThreshold: 60,
3287+
SuccessThreshold: 1,
3288+
}
3289+
}
3290+
32743291
func generateReadinessProbe() *corev1.Probe {
32753292
return &corev1.Probe{
32763293
ProbeHandler: corev1.ProbeHandler{
32773294
HTTPGet: &corev1.HTTPGetAction{
3278-
Scheme: "HTTP",
3279-
Port: intstr.IntOrString{
3280-
IntVal: 8082,
3281-
},
3282-
Path: "/envoy-hc",
3295+
Path: "/ready",
3296+
Port: intstr.FromInt(8082),
32833297
},
32843298
},
32853299
InitialDelaySeconds: 5,
3286-
PeriodSeconds: 5,
3287-
FailureThreshold: 2,
3300+
PeriodSeconds: 10,
32883301
}
32893302
}
32903303

0 commit comments

Comments
 (0)