From be73faa42f41ae3a06969eca1bc17c5ec21fa242 Mon Sep 17 00:00:00 2001 From: David Jumani Date: Wed, 13 Nov 2024 11:48:33 -0500 Subject: [PATCH 1/8] update apis --- docs/content/reference/values.txt | 47 +++++ ...ateway.gloo.solo.io_gatewayparameters.yaml | 171 ++++++++++++++++++ install/helm/gloo/generate/values.go | 11 ++ projects/gateway2/api/v1alpha1/kube_types.go | 82 +++++++++ .../api/v1alpha1/zz_generated.deepcopy.go | 45 +++++ projects/gateway2/deployer/values.go | 19 +- 6 files changed, 368 insertions(+), 7 deletions(-) diff --git a/docs/content/reference/values.txt b/docs/content/reference/values.txt index 49cc3166acf..af026605931 100644 --- a/docs/content/reference/values.txt +++ b/docs/content/reference/values.txt @@ -184,6 +184,53 @@ |kubeGateway.gatewayParameters.glooGateway.aiExtension.ports[].protocol|string||| |kubeGateway.gatewayParameters.glooGateway.aiExtension.ports[].hostIP|string||| |kubeGateway.gatewayParameters.glooGateway.floatingUserId|bool||If true, allows the cluster to dynamically assign a user ID for the processes running in the container. Default is false.| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.gracefulShutdown.enabled|bool||Enable grace period before shutdown to finish current requests while Envoy health checks fail to e.g. notify external load balancers. *NOTE:* This will not have any effect if you have not defined health checks via the health check filter| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.gracefulShutdown.sleepTimeSeconds|int||Time (in seconds) for the preStop hook to wait before allowing Envoy to terminate| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.terminationGracePeriodSeconds|int||Time in seconds to wait for the pod to terminate gracefully. See [kubernetes docs](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#istio-lifecycle) for more info.| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.probes|bool||Set to true to enable a readiness probe (default is false). Then, you can also enable a liveness probe.| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.livenessProbeEnabled|bool||Set to true to enable a liveness probe (default is false).| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.exec.command[]|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.httpGet.path|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.httpGet.port|int64||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.httpGet.port|int32||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.httpGet.port|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.httpGet.host|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.httpGet.scheme|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.httpGet.httpHeaders[].name|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.httpGet.httpHeaders[].value|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.tcpSocket.port|int64||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.tcpSocket.port|int32||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.tcpSocket.port|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.tcpSocket.host|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.grpc.port|int32||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.grpc.service|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.initialDelaySeconds|int32||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.timeoutSeconds|int32||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.periodSeconds|int32||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.successThreshold|int32||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.failureThreshold|int32||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.terminationGracePeriodSeconds|int64||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.exec.command[]|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.httpGet.path|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.httpGet.port|int64||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.httpGet.port|int32||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.httpGet.port|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.httpGet.host|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.httpGet.scheme|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.httpGet.httpHeaders[].name|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.httpGet.httpHeaders[].value|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.tcpSocket.port|int64||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.tcpSocket.port|int32||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.tcpSocket.port|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.tcpSocket.host|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.grpc.port|int32||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.grpc.service|string||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.initialDelaySeconds|int32||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.timeoutSeconds|int32||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.periodSeconds|int32||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.successThreshold|int32||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.failureThreshold|int32||| +|kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.terminationGracePeriodSeconds|int64||| |kubeGateway.portal.enabled|bool|false|Enable the Gloo Gateway Portal controller and web server.| |settings.watchNamespaces[]|string||whitelist of namespaces for Gloo Edge to watch for services and CRDs. Empty list means all namespaces. If this and WatchNamespaceSelectors are specified, this takes precedence and WatchNamespaceSelectors is ignored| |settings.watchNamespaceSelectors|interface||A list of Kubernetes selectors that specify the set of namespaces to restrict the namespaces that Gloo controllers take into consideration when watching for resources. Elements in the list are disjunctive (OR semantics), i.e. a namespace will be included if it matches any selector. An empty list means all namespaces. If this and WatchNamespaces are specified, WatchNamespaces takes precedence and this is ignored| diff --git a/install/helm/gloo/crds/gateway.gloo.solo.io_gatewayparameters.yaml b/install/helm/gloo/crds/gateway.gloo.solo.io_gatewayparameters.yaml index 322516cabae..7e7fb5fbe5f 100644 --- a/install/helm/gloo/crds/gateway.gloo.solo.io_gatewayparameters.yaml +++ b/install/helm/gloo/crds/gateway.gloo.solo.io_gatewayparameters.yaml @@ -1667,6 +1667,13 @@ spec: additionalProperties: type: string type: object + gracefulShutdown: + properties: + enabled: + type: boolean + sleepTimeSeconds: + type: integer + type: object imagePullSecrets: items: properties: @@ -1676,10 +1683,172 @@ spec: type: object x-kubernetes-map-type: atomic type: array + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object nodeSelector: additionalProperties: type: string type: object + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object securityContext: properties: appArmorProfile: @@ -1757,6 +1926,8 @@ spec: type: string type: object type: object + terminationGracePeriodSeconds: + type: integer tolerations: items: properties: diff --git a/install/helm/gloo/generate/values.go b/install/helm/gloo/generate/values.go index fdfd06998c8..87ce1536aa4 100644 --- a/install/helm/gloo/generate/values.go +++ b/install/helm/gloo/generate/values.go @@ -339,9 +339,20 @@ type GatewayParameters struct { Stats *GatewayParamsStatsConfig `json:"stats,omitempty" desc:"Config used to manage the stats endpoints exposed on the deployed proxies"` AIExtension *GatewayParamsAIExtension `json:"aiExtension,omitempty" desc:"Config used to manage the Gloo Gateway AI extension."` FloatingUserId *bool `json:"floatingUserId,omitempty" desc:"If true, allows the cluster to dynamically assign a user ID for the processes running in the container. Default is false."` + PodTemplate *GatewayParamsPodTemplate `json:"podTemplate,omitempty"` // TODO(npolshak): Add support for GlooMtls } +// GatewayProxyPodTemplate contains the Helm API available to configure the PodTemplate on the gateway Deployment +type GatewayParamsPodTemplate struct { + GracefulShutdown *GracefulShutdownSpec `json:"gracefulShutdown,omitempty"` + TerminationGracePeriodSeconds *int `json:"terminationGracePeriodSeconds,omitempty" desc:"Time in seconds to wait for the pod to terminate gracefully. See [kubernetes docs](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#istio-lifecycle) for more info."` + Probes *bool `json:"probes,omitempty" desc:"Set to true to enable a readiness probe (default is false). Then, you can also enable a liveness probe."` + LivenessProbeEnabled *bool `json:"livenessProbeEnabled,omitempty" desc:"Set to true to enable a liveness probe (default is false)."` + CustomReadinessProbe *corev1.Probe `json:"customReadinessProbe,omitempty"` + CustomLivenessProbe *corev1.Probe `json:"customLivenessProbe,omitempty"` +} + type GatewayParamsStatsConfig struct { Enabled *bool `json:"enabled,omitempty" desc:"Enable the prometheus endpoint"` RoutePrefixRewrite *string `json:"routePrefixRewrite,omitempty" desc:"Set the prefix rewrite used for the prometheus endpoint"` diff --git a/projects/gateway2/api/v1alpha1/kube_types.go b/projects/gateway2/api/v1alpha1/kube_types.go index 25f28c269ce..64028b3e39a 100644 --- a/projects/gateway2/api/v1alpha1/kube_types.go +++ b/projects/gateway2/api/v1alpha1/kube_types.go @@ -199,6 +199,34 @@ type Pod struct { // // +kubebuilder:validation:Optional Tolerations []*corev1.Toleration `json:"tolerations,omitempty"` + + // If specified, the pod's graceful shutdown spec. + // + // +kubebuilder:validation:Optional + GracefulShutdown *GracefulShutdownSpec `json:"gracefulShutdown,omitempty"` + + // If specified, the pod's termination grace period in seconds. See + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#pod-v1-core + // for details + // + // +kubebuilder:validation:Optional + TerminationGracePeriodSeconds *int `json:"terminationGracePeriodSeconds,omitempty"` + + // If specified, the pod's readiness probe. Periodic probe of container service readiness. + // Container will be removed from service endpoints if the probe fails. See + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#probe-v1-core + // for details. + // + // +kubebuilder:validation:Optional + ReadinessProbe *corev1.Probe `json:"readinessProbe,omitempty"` + + // If specified, the pod's liveness probe. Periodic probe of container service readiness. + // Container will be restarted if the probe fails. See + // https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#probe-v1-core + // for details. + // + // +kubebuilder:validation:Optional + LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty"` } func (in *Pod) GetExtraLabels() map[string]string { @@ -249,3 +277,57 @@ func (in *Pod) GetTolerations() []*corev1.Toleration { } return in.Tolerations } + +func (in *Pod) GetReadinessProbe() *corev1.Probe { + if in == nil { + return nil + } + return in.ReadinessProbe +} + +func (in *Pod) GetGracefulShutdown() *GracefulShutdownSpec { + if in == nil { + return nil + } + return in.GracefulShutdown +} + +func (in *Pod) GetTerminationGracePeriodSeconds() *int { + if in == nil { + return nil + } + return in.TerminationGracePeriodSeconds +} + +func (in *Pod) GetLivenessProbe() *corev1.Probe { + if in == nil { + return nil + } + return in.LivenessProbe +} + +type GracefulShutdownSpec struct { + // Enable grace period before shutdown to finish current requests while Envoy health checks fail to e.g. notify external load balancers. *NOTE:* This will not have any effect if you have not defined health checks via the health check filter + // + // +kubebuilder:validation:Optional + Enabled *bool `json:"enabled,omitempty"` + + // Time (in seconds) for the preStop hook to wait before allowing Envoy to terminate + // + // +kubebuilder:validation:Optional + SleepTimeSeconds *int `json:"sleepTimeSeconds,omitempty"` +} + +func (in *GracefulShutdownSpec) GetEnabled() *bool { + if in == nil { + return nil + } + return in.Enabled +} + +func (in *GracefulShutdownSpec) GetSleepTimeSeconds() *int { + if in == nil { + return nil + } + return in.SleepTimeSeconds +} diff --git a/projects/gateway2/api/v1alpha1/zz_generated.deepcopy.go b/projects/gateway2/api/v1alpha1/zz_generated.deepcopy.go index 5f6d242d0f7..212c08e2b49 100644 --- a/projects/gateway2/api/v1alpha1/zz_generated.deepcopy.go +++ b/projects/gateway2/api/v1alpha1/zz_generated.deepcopy.go @@ -372,6 +372,31 @@ func (in *GatewayParametersStatus) DeepCopy() *GatewayParametersStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GracefulShutdownSpec) DeepCopyInto(out *GracefulShutdownSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.SleepTimeSeconds != nil { + in, out := &in.SleepTimeSeconds, &out.SleepTimeSeconds + *out = new(int) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GracefulShutdownSpec. +func (in *GracefulShutdownSpec) DeepCopy() *GracefulShutdownSpec { + if in == nil { + return nil + } + out := new(GracefulShutdownSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Image) DeepCopyInto(out *Image) { *out = *in @@ -608,6 +633,26 @@ func (in *Pod) DeepCopyInto(out *Pod) { } } } + if in.GracefulShutdown != nil { + in, out := &in.GracefulShutdown, &out.GracefulShutdown + *out = new(GracefulShutdownSpec) + (*in).DeepCopyInto(*out) + } + if in.TerminationGracePeriodSeconds != nil { + in, out := &in.TerminationGracePeriodSeconds, &out.TerminationGracePeriodSeconds + *out = new(int) + **out = **in + } + if in.ReadinessProbe != nil { + in, out := &in.ReadinessProbe, &out.ReadinessProbe + *out = new(v1.Probe) + (*in).DeepCopyInto(*out) + } + if in.LivenessProbe != nil { + in, out := &in.LivenessProbe, &out.LivenessProbe + *out = new(v1.Probe) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Pod. diff --git a/projects/gateway2/deployer/values.go b/projects/gateway2/deployer/values.go index 7cc5741cceb..4fc0f2a6621 100644 --- a/projects/gateway2/deployer/values.go +++ b/projects/gateway2/deployer/values.go @@ -1,6 +1,7 @@ package deployer import ( + "github.com/solo-io/gloo/projects/gateway2/api/v1alpha1" corev1 "k8s.io/api/core/v1" ) @@ -28,13 +29,17 @@ type helmGateway struct { ServiceAccount *helmServiceAccount `json:"serviceAccount,omitempty"` // pod template values - ExtraPodAnnotations map[string]string `json:"extraPodAnnotations,omitempty"` - ExtraPodLabels map[string]string `json:"extraPodLabels,omitempty"` - ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` - PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - Affinity *corev1.Affinity `json:"affinity,omitempty"` - Tolerations []*corev1.Toleration `json:"tolerations,omitempty"` + ExtraPodAnnotations map[string]string `json:"extraPodAnnotations,omitempty"` + ExtraPodLabels map[string]string `json:"extraPodLabels,omitempty"` + ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` + PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + Tolerations []*corev1.Toleration `json:"tolerations,omitempty"` + ReadinessProbe *corev1.Probe `json:"readinessProbe,omitempty"` + LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty"` + GracefulShutdown *v1alpha1.GracefulShutdownSpec `json:"gracefulShutdown,omitempty"` + TerminationGracePeriodSeconds *int `json:"terminationGracePeriodSeconds,omitempty"` // sds container values SdsContainer *helmSdsContainer `json:"sdsContainer,omitempty"` From 7ee19929bc7290a5568a2987691e57593a2e4c9f Mon Sep 17 00:00:00 2001 From: David Jumani Date: Wed, 13 Nov 2024 11:48:59 -0500 Subject: [PATCH 2/8] helm changes --- .../gloo/templates/43-gatewayparameters.yaml | 41 ++++++++++++++++++- projects/gateway2/deployer/deployer.go | 4 ++ .../templates/gateway/proxy-deployment.yaml | 40 ++++++++++++++---- 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/install/helm/gloo/templates/43-gatewayparameters.yaml b/install/helm/gloo/templates/43-gatewayparameters.yaml index e7bda342e21..c55457e9c3a 100644 --- a/install/helm/gloo/templates/43-gatewayparameters.yaml +++ b/install/helm/gloo/templates/43-gatewayparameters.yaml @@ -72,7 +72,46 @@ spec: {{- end }}{{/* if $gg.envoyContainer */}} podTemplate: extraLabels: - {{- include "gloo-gateway.constLabels" . | nindent 8 }} + {{- include "gloo-gateway.constLabels" . | nindent 8 }} +{{- if $gg.podTemplate }} +{{- if $gg.podTemplate.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ $gg.podTemplate.terminationGracePeriodSeconds }} +{{- end }}{{/* if $gg.podTemplate.terminationGracePeriodSeconds */}} +{{- if $gg.podTemplate.gracefulShutdown }} + gracefulShutdown: + {{- toYaml $gg.podTemplate.gracefulShutdown | nindent 8 }} +{{- end }}{{/* if $gg.podTemplate.gracefulShutdown */}} +{{- if $gg.podTemplate.probes }} + readinessProbe: +{{- if $gg.podTemplate.customReadinessProbe }} +{{ toYaml $gg.podTemplate.customReadinessProbe | indent 8}} +{{- else }} + httpGet: + scheme: HTTP + port: 8082 + path: /envoy-hc + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 2 +{{- end}}{{/* if $gg.podTemplate.customReadinessProbe */}} +{{- if $gg.podTemplate.livenessProbeEnabled }} + livenessProbe: +{{- if $gg.podTemplate.customLivenessProbe }} +{{ toYaml $gg.podTemplate.customLivenessProbe | indent 8}} +{{- else }} + exec: + command: + - wget + - -O + - /dev/null + - 127.0.0.1:19000/server_info + initialDelaySeconds: 3 + periodSeconds: 10 + failureThreshold: 3 +{{- end }}{{/*if $gg.podTemplate.customLivenessProbe*/}} +{{- end }}{{/*if $gg.podTemplate.livenessProbeEnabled*/}} +{{- end }}{{/*if $gg.podTemplate.probes*/}} +{{- end }}{{/*if $gg.podTemplate */}} {{- if $gg.sdsContainer }} sdsContainer: image: diff --git a/projects/gateway2/deployer/deployer.go b/projects/gateway2/deployer/deployer.go index bb03aae028d..eb0ce8b2ac0 100644 --- a/projects/gateway2/deployer/deployer.go +++ b/projects/gateway2/deployer/deployer.go @@ -326,6 +326,10 @@ func (d *Deployer) getValues(gw *api.Gateway, gwParam *v1alpha1.GatewayParameter gateway.NodeSelector = podConfig.GetNodeSelector() gateway.Affinity = podConfig.GetAffinity() gateway.Tolerations = podConfig.GetTolerations() + gateway.ReadinessProbe = podConfig.GetReadinessProbe() + gateway.LivenessProbe = podConfig.GetLivenessProbe() + gateway.GracefulShutdown = podConfig.GetGracefulShutdown() + gateway.TerminationGracePeriodSeconds = podConfig.GetTerminationGracePeriodSeconds() // envoy container values logLevel := envoyContainerConfig.GetBootstrap().GetLogLevel() diff --git a/projects/gateway2/helm/gloo-gateway/templates/gateway/proxy-deployment.yaml b/projects/gateway2/helm/gloo-gateway/templates/gateway/proxy-deployment.yaml index 9796b4c14f8..200263fec18 100644 --- a/projects/gateway2/helm/gloo-gateway/templates/gateway/proxy-deployment.yaml +++ b/projects/gateway2/helm/gloo-gateway/templates/gateway/proxy-deployment.yaml @@ -89,15 +89,25 @@ spec: - name: http-monitoring containerPort: 9091 {{- end }} - exec: - command: - - wget - - -O - - /dev/null - - 127.0.0.1:19000/ready - initialDelaySeconds: 3 - periodSeconds: 10 - failureThreshold: 3 +{{- if $gateway.readinessProbe }} + readinessProbe: +{{ toYaml $gateway.readinessProbe | indent 10}} +{{- end }}{{/*if $gateway.readinessProbe*/}} +{{- if $gateway.livenessProbe }} + livenessProbe: +{{ toYaml $gateway.livenessProbe | indent 10}} +{{- end }}{{/*if $gateway.livenessProbe*/}} +{{- if $gateway.gracefulShutdown }} +{{- if $gateway.gracefulShutdown.enabled }} + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - wget --post-data "" -O /dev/null 127.0.0.1:19000/healthcheck/fail; sleep {{ $gateway.gracefulShutdown.sleepTimeSeconds | default "10" }} +{{- end}}{{/*if $gateway.gracefulShutdown.enabled*/}} +{{- end}}{{/*if $gateway.gracefulShutdown*/}} {{- if $gateway.resources }} resources: {{- toYaml $gateway.resources | nindent 10 }} @@ -301,6 +311,9 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + {{- if $gateway.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ $gateway.terminationGracePeriodSeconds }} + {{- end }} volumes: - configMap: name: {{ include "gloo-gateway.gateway.fullname" . }} @@ -439,6 +452,15 @@ data: route: cluster: admin_port_cluster http_filters: +{{- if $gateway.readinessProbe }} + - name: envoy.filters.http.health_check + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck + pass_through_mode: false + headers: + - name: ":path" + exact_match: "/envoy-hc" +{{- end }}{{/*if $gateway.readinessProbe*/}} - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router From 33ebc017b507d761666c8d2173a4519ad5a5712d Mon Sep 17 00:00:00 2001 From: David Jumani Date: Wed, 13 Nov 2024 11:49:39 -0500 Subject: [PATCH 3/8] add tests --- install/test/k8sgateway_test.go | 144 ++++++++++++++++++++ projects/gateway2/deployer/deployer_test.go | 79 +++++++++++ 2 files changed, 223 insertions(+) diff --git a/install/test/k8sgateway_test.go b/install/test/k8sgateway_test.go index 845fec4fca2..85ed653a25b 100644 --- a/install/test/k8sgateway_test.go +++ b/install/test/k8sgateway_test.go @@ -15,6 +15,8 @@ import ( . "github.com/solo-io/k8s-utils/manifesttestutils" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" ) var _ = Describe("Kubernetes Gateway API integration", func() { @@ -355,6 +357,148 @@ var _ = Describe("Kubernetes Gateway API integration", func() { Entry("locally undefined, globally undefined", false), ) }) + + Context("probes and graceful shutdown", func() { + When("nothing is specified", func() { + It("does not render probes and graceful shutdown", func() { + gwp := getDefaultGatewayParameters(testManifest) + + gwpPT := gwp.Spec.Kube.PodTemplate + Expect(gwpPT).ToNot(BeNil()) + + Expect(gwpPT.LivenessProbe).To(BeNil()) + Expect(gwpPT.ReadinessProbe).To(BeNil()) + Expect(gwpPT.GracefulShutdown).To(BeNil()) + Expect(gwpPT.TerminationGracePeriodSeconds).To(BeNil()) + }) + }) + + When("probes are enabled", func() { + BeforeEach(func() { + extraValuesArgs := []string{ + "kubeGateway.gatewayParameters.glooGateway.podTemplate.probes=true", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.livenessProbeEnabled=true", + } + + valuesArgs = append(valuesArgs, extraValuesArgs...) + }) + + It("sets the default values of the probes", func() { + gwp := getDefaultGatewayParameters(testManifest) + gwpPT := gwp.Spec.Kube.PodTemplate + Expect(*gwpPT.LivenessProbe).To(BeEquivalentTo(corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "wget", + "-O", + "/dev/null", + "127.0.0.1:19000/server_info", + }, + }, + }, + InitialDelaySeconds: 3, + PeriodSeconds: 10, + FailureThreshold: 3, + })) + Expect(*gwpPT.ReadinessProbe).To(BeEquivalentTo(corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: "HTTP", + Port: intstr.IntOrString{ + IntVal: 8082, + }, + Path: "/envoy-hc", + }, + }, + InitialDelaySeconds: 5, + PeriodSeconds: 5, + FailureThreshold: 2, + })) + + }) + }) + + When("custom probes are defined", func() { + BeforeEach(func() { + extraValuesArgs := []string{ + "kubeGateway.gatewayParameters.glooGateway.podTemplate.probes=true", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.httpGet.scheme=HTTP", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.httpGet.port=9090", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.httpGet.path=/custom-readiness", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.failureThreshold=1", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.initialDelaySeconds=2", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe.periodSeconds=3", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.livenessProbeEnabled=true", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.exec.command[0]=wget", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.exec.command[1]=-O", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.exec.command[2]=/dev/null", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.exec.command[3]=127.0.0.1:9090/custom-liveness", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.failureThreshold=4", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.initialDelaySeconds=5", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe.periodSeconds=6", + } + + valuesArgs = append(valuesArgs, extraValuesArgs...) + }) + + It("sets the custom values of the probes", func() { + gwp := getDefaultGatewayParameters(testManifest) + gwpPT := gwp.Spec.Kube.PodTemplate + Expect(*gwpPT.ReadinessProbe).To(BeEquivalentTo(corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: "HTTP", + Port: intstr.IntOrString{ + IntVal: 9090, + }, + Path: "/custom-readiness", + }, + }, + FailureThreshold: 1, + InitialDelaySeconds: 2, + PeriodSeconds: 3, + })) + Expect(*gwpPT.LivenessProbe).To(BeEquivalentTo(corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "wget", + "-O", + "/dev/null", + "127.0.0.1:9090/custom-liveness", + }, + }, + }, + FailureThreshold: 4, + InitialDelaySeconds: 5, + PeriodSeconds: 6, + })) + }) + }) + + When("gracefulShutdown and terminationGracePeriod is enabled", func() { + BeforeEach(func() { + extraValuesArgs := []string{ + "kubeGateway.gatewayParameters.glooGateway.podTemplate.terminationGracePeriodSeconds=7", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.gracefulShutdown.enabled=true", + "kubeGateway.gatewayParameters.glooGateway.podTemplate.gracefulShutdown.sleepTimeSeconds=5", + } + + valuesArgs = append(valuesArgs, extraValuesArgs...) + }) + + It("sets the custom values", func() { + gwp := getDefaultGatewayParameters(testManifest) + gwpPT := gwp.Spec.Kube.PodTemplate + Expect(*gwpPT.TerminationGracePeriodSeconds).To(Equal(7)) + Expect(*gwpPT.GracefulShutdown).To(BeEquivalentTo(v1alpha1.GracefulShutdownSpec{ + Enabled: pointer.Bool(true), + SleepTimeSeconds: pointer.Int(5), + })) + }) + }) + }) }) When("kube gateway integration is disabled (default)", func() { diff --git a/projects/gateway2/deployer/deployer_test.go b/projects/gateway2/deployer/deployer_test.go index 5a928f28a6f..9890800f16d 100644 --- a/projects/gateway2/deployer/deployer_test.go +++ b/projects/gateway2/deployer/deployer_test.go @@ -25,7 +25,9 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/utils/pointer" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -752,6 +754,18 @@ var _ = Describe("Deployer", func() { return fullyDefinedGatewayParameters(wellknown.DefaultGatewayParametersName, defaultNamespace) } + fullyDefinedGatewayParamsWithProbes = func() *gw2_v1alpha1.GatewayParameters { + params := fullyDefinedGatewayParameters(wellknown.DefaultGatewayParametersName, defaultNamespace) + params.Spec.Kube.PodTemplate.LivenessProbe = generateLivenessProbe() + params.Spec.Kube.PodTemplate.ReadinessProbe = generateReadinessProbe() + params.Spec.Kube.PodTemplate.TerminationGracePeriodSeconds = pointer.Int(5) + params.Spec.Kube.PodTemplate.GracefulShutdown = &gw2_v1alpha1.GracefulShutdownSpec{ + Enabled: pointer.Bool(true), + SleepTimeSeconds: pointer.Int(7), + } + return params + } + fullyDefinedGatewayParamsWithFloatingUserId = func() *gw2_v1alpha1.GatewayParameters { params := fullyDefinedGatewayParameters(wellknown.DefaultGatewayParametersName, defaultNamespace) params.Spec.Kube.FloatingUserId = ptr.To(true) @@ -795,6 +809,7 @@ var _ = Describe("Deployer", func() { Expect(dep.Spec.Template.Spec.Containers[0].Image).To(ContainSubstring(":" + version.Version)) } Expect(dep.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(*expectedGwp.EnvoyContainer.Image.PullPolicy)) + Expect(dep.Spec.Template.Annotations).To(matchers.ContainMapElements(expectedGwp.PodTemplate.ExtraAnnotations)) Expect(dep.Spec.Template.Annotations).To(HaveKeyWithValue("prometheus.io/scrape", "true")) Expect(dep.Spec.Template.Spec.SecurityContext.RunAsUser).To(Equal(expectedGwp.PodTemplate.SecurityContext.RunAsUser)) @@ -973,6 +988,27 @@ var _ = Describe("Deployer", func() { return nil } + fullyDefinedValidationWithProbes := func(objs clientObjects, inp *input) error { + err := fullyDefinedValidationWithoutRunAsUser(objs, inp) + if err != nil { + return err + } + + dep := objs.findDeployment(defaultNamespace, defaultDeploymentName) + Expect(*dep.Spec.Template.Spec.TerminationGracePeriodSeconds).To(Equal(int64(5))) + + envoyContainer := dep.Spec.Template.Spec.Containers[0] + Expect(envoyContainer.LivenessProbe).To(BeEquivalentTo(generateLivenessProbe())) + Expect(envoyContainer.ReadinessProbe).To(BeEquivalentTo(generateReadinessProbe())) + Expect(envoyContainer.Lifecycle.PreStop.Exec.Command).To(BeEquivalentTo([]string{ + "/bin/sh", + "-c", + "wget --post-data \"\" -O /dev/null 127.0.0.1:19000/healthcheck/fail; sleep 7", + })) + + return nil + } + fullyDefinedValidationFloatingUserId := func(objs clientObjects, inp *input) error { err := fullyDefinedValidationWithoutRunAsUser(objs, inp) if err != nil { @@ -1143,6 +1179,14 @@ var _ = Describe("Deployer", func() { }, &expectedOutput{ validationFunc: fullyDefinedValidation, }), + Entry("Fully defined GatewayParameters with probes", &input{ + dInputs: istioEnabledDeployerInputs(), + gw: defaultGateway(), + defaultGwp: fullyDefinedGatewayParamsWithProbes(), + }, &expectedOutput{ + validationFunc: fullyDefinedValidationWithProbes, + }), + Entry("Fully defined GatewayParameters with floating user id", &input{ dInputs: istioEnabledDeployerInputs(), gw: defaultGateway(), @@ -1550,3 +1594,38 @@ func fullyDefinedGatewayParameters(name, namespace string) *gw2_v1alpha1.Gateway }, } } + +func generateLivenessProbe() *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "wget", + "-O", + "/dev/null", + "127.0.0.1:19000/server_info", + }, + }, + }, + InitialDelaySeconds: 3, + PeriodSeconds: 10, + FailureThreshold: 3, + } +} + +func generateReadinessProbe() *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: "HTTP", + Port: intstr.IntOrString{ + IntVal: 8082, + }, + Path: "/envoy-hc", + }, + }, + InitialDelaySeconds: 5, + PeriodSeconds: 5, + FailureThreshold: 2, + } +} From debf04af5607053a82bb5f8d84f6d9dfd6b8c424 Mon Sep 17 00:00:00 2001 From: David Jumani Date: Wed, 13 Nov 2024 11:53:00 -0500 Subject: [PATCH 4/8] add probes to tests --- .../tests/manifests/profiles/kubernetes-gateway.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/kubernetes/e2e/tests/manifests/profiles/kubernetes-gateway.yaml b/test/kubernetes/e2e/tests/manifests/profiles/kubernetes-gateway.yaml index 4941924918d..89067f667a6 100644 --- a/test/kubernetes/e2e/tests/manifests/profiles/kubernetes-gateway.yaml +++ b/test/kubernetes/e2e/tests/manifests/profiles/kubernetes-gateway.yaml @@ -13,6 +13,16 @@ global: # Configuration for the Kubernetes Gateway integration in Gloo Gateway kubeGateway: enabled: true + # Enable the probes to ensure zero downtime + gatewayParameters: + glooGateway: + podTemplate: + terminationGracePeriodSeconds: 7 + gracefulShutdown: + enabled: true + sleepTimeSeconds: 5 + probes: true + livenessProbeEnabled: true # Configuration for the statically deployed gateway-proxy that ships by default with Gloo Gateway gatewayProxies: From fd6152efc1d25950fe995109143a4a63b91ceb22 Mon Sep 17 00:00:00 2001 From: David Jumani Date: Wed, 13 Nov 2024 13:05:57 -0500 Subject: [PATCH 5/8] add changelog --- .../add-readiness-liveness-probe.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 changelog/v1.18.0-beta35/add-readiness-liveness-probe.yaml diff --git a/changelog/v1.18.0-beta35/add-readiness-liveness-probe.yaml b/changelog/v1.18.0-beta35/add-readiness-liveness-probe.yaml new file mode 100644 index 00000000000..fae9796e072 --- /dev/null +++ b/changelog/v1.18.0-beta35/add-readiness-liveness-probe.yaml @@ -0,0 +1,16 @@ +changelog: +- type: NEW_FEATURE + issueLink: https://github.com/solo-io/solo-projects/issues/7084 + resolvesIssue: false + description: Adds the ability for users to enable as well as add custom readiness and liveness probes to the Kubernetes Gloo Gateway. +- type: HELM + issueLink: https://github.com/solo-io/solo-projects/issues/7084 + resolvesIssue: false + description: >- + Adds the following new fields that configure the Kubernetes Gloo Gateway pod : + - `kubeGateway.gatewayParameters.glooGateway.podTemplate.terminationGracePeriodSeconds` to specify the terminationGracePeriodSeconds. + - `kubeGateway.gatewayParameters.glooGateway.podTemplate.gracefulShutdown` to configure the graceful shutdown config for the envoy container. + - `kubeGateway.gatewayParameters.glooGateway.podTemplate.customLivenessProbe` to specify a custom liveness probe for the envoy container. + - `kubeGateway.gatewayParameters.glooGateway.podTemplate.livenessProbeEnabled` to enable the liveness probe. If the customLivenessProbe is not specified, a default liveness probe is set. + - `kubeGateway.gatewayParameters.glooGateway.podTemplate.customReadinessProbe` to specify a custom readiness probe for the envoy container. + - `kubeGateway.gatewayParameters.glooGateway.podTemplate.probes` to enable the readiness probe. If the customReadinessProbe is not specified, a default readiness probe is set. From 74de62bfd23a4751681464ba521c575783450883 Mon Sep 17 00:00:00 2001 From: David Jumani Date: Wed, 13 Nov 2024 13:23:55 -0500 Subject: [PATCH 6/8] codegen --- .../api/v1alpha1/gateway_parameters.md | 642 ++++++++++++++++++ 1 file changed, 642 insertions(+) diff --git a/docs/content/reference/api/github.com/solo-io/gloo/projects/gateway2/api/v1alpha1/gateway_parameters.md b/docs/content/reference/api/github.com/solo-io/gloo/projects/gateway2/api/v1alpha1/gateway_parameters.md index 21b69b9b825..700cad8eb9f 100644 --- a/docs/content/reference/api/github.com/solo-io/gloo/projects/gateway2/api/v1alpha1/gateway_parameters.md +++ b/docs/content/reference/api/github.com/solo-io/gloo/projects/gateway2/api/v1alpha1/gateway_parameters.md @@ -4697,6 +4697,13 @@ Resource Types:
false + + gracefulShutdown + object + +
+ + false imagePullSecrets []object @@ -4704,6 +4711,13 @@ Resource Types:
false + + livenessProbe + object + +
+ + false nodeSelector map[string]string @@ -4711,6 +4725,13 @@ Resource Types:
false + + readinessProbe + object + +
+ + false securityContext object @@ -4718,6 +4739,13 @@ Resource Types:
false + + terminationGracePeriodSeconds + integer + +
+ + false tolerations []object @@ -6087,6 +6115,40 @@ Resource Types: +### GatewayParameters.spec.kube.podTemplate.gracefulShutdown +[↩ Parent](#gatewayparametersspeckubepodtemplate) + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
enabledboolean +
+ 		false
sleepTimeSecondsinteger +
+ 		false
+ + ### GatewayParameters.spec.kube.podTemplate.imagePullSecrets[index] [↩ Parent](#gatewayparametersspeckubepodtemplate) @@ -6116,6 +6178,586 @@ Resource Types: +### GatewayParameters.spec.kube.podTemplate.livenessProbe +[↩ Parent](#gatewayparametersspeckubepodtemplate) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
execobject +
+ 		false
failureThresholdinteger +
+
+ Format: int32
+ 		false
grpcobject +
+ 		false
httpGetobject +
+ 		false
initialDelaySecondsinteger +
+
+ Format: int32
+ 		false
periodSecondsinteger +
+
+ Format: int32
+ 		false
successThresholdinteger +
+
+ Format: int32
+ 		false
tcpSocketobject +
+ 		false
terminationGracePeriodSecondsinteger +
+
+ Format: int64
+ 		false
timeoutSecondsinteger +
+
+ Format: int32
+ 		false
+ + +### GatewayParameters.spec.kube.podTemplate.livenessProbe.exec +[↩ Parent](#gatewayparametersspeckubepodtemplatelivenessprobe) + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
command[]string +
+ 		false
+ + +### GatewayParameters.spec.kube.podTemplate.livenessProbe.grpc +[↩ Parent](#gatewayparametersspeckubepodtemplatelivenessprobe) + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
portinteger +
+
+ Format: int32
+ 		true
servicestring +
+
+ Default:
+ 		false
+ + +### GatewayParameters.spec.kube.podTemplate.livenessProbe.httpGet +[↩ Parent](#gatewayparametersspeckubepodtemplatelivenessprobe) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
portint or string +
+ 		true
hoststring +
+ 		false
httpHeaders[]object +
+ 		false
pathstring +
+ 		false
schemestring +
+ 		false
+ + +### GatewayParameters.spec.kube.podTemplate.livenessProbe.httpGet.httpHeaders[index] +[↩ Parent](#gatewayparametersspeckubepodtemplatelivenessprobehttpget) + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring +
+ 		true
valuestring +
+ 		true
+ + +### GatewayParameters.spec.kube.podTemplate.livenessProbe.tcpSocket +[↩ Parent](#gatewayparametersspeckubepodtemplatelivenessprobe) + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
portint or string +
+ 		true
hoststring +
+ 		false
+ + +### GatewayParameters.spec.kube.podTemplate.readinessProbe +[↩ Parent](#gatewayparametersspeckubepodtemplate) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
execobject +
+ 		false
failureThresholdinteger +
+
+ Format: int32
+ 		false
grpcobject +
+ 		false
httpGetobject +
+ 		false
initialDelaySecondsinteger +
+
+ Format: int32
+ 		false
periodSecondsinteger +
+
+ Format: int32
+ 		false
successThresholdinteger +
+
+ Format: int32
+ 		false
tcpSocketobject +
+ 		false
terminationGracePeriodSecondsinteger +
+
+ Format: int64
+ 		false
timeoutSecondsinteger +
+
+ Format: int32
+ 		false
+ + +### GatewayParameters.spec.kube.podTemplate.readinessProbe.exec +[↩ Parent](#gatewayparametersspeckubepodtemplatereadinessprobe) + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
command[]string +
+ 		false
+ + +### GatewayParameters.spec.kube.podTemplate.readinessProbe.grpc +[↩ Parent](#gatewayparametersspeckubepodtemplatereadinessprobe) + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
portinteger +
+
+ Format: int32
+ 		true
servicestring +
+
+ Default:
+ 		false
+ + +### GatewayParameters.spec.kube.podTemplate.readinessProbe.httpGet +[↩ Parent](#gatewayparametersspeckubepodtemplatereadinessprobe) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
portint or string +
+ 		true
hoststring +
+ 		false
httpHeaders[]object +
+ 		false
pathstring +
+ 		false
schemestring +
+ 		false
+ + +### GatewayParameters.spec.kube.podTemplate.readinessProbe.httpGet.httpHeaders[index] +[↩ Parent](#gatewayparametersspeckubepodtemplatereadinessprobehttpget) + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring +
+ 		true
valuestring +
+ 		true
+ + +### GatewayParameters.spec.kube.podTemplate.readinessProbe.tcpSocket +[↩ Parent](#gatewayparametersspeckubepodtemplatereadinessprobe) + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
portint or string +
+ 		true
hoststring +
+ 		false
+ + ### GatewayParameters.spec.kube.podTemplate.securityContext [↩ Parent](#gatewayparametersspeckubepodtemplate) From 43266cdbe2b3d650b0a22b55712e6050bd711063 Mon Sep 17 00:00:00 2001 From: David Jumani Date: Wed, 13 Nov 2024 14:35:54 -0500 Subject: [PATCH 7/8] add merge --- projects/gateway2/deployer/merge.go | 132 ++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/projects/gateway2/deployer/merge.go b/projects/gateway2/deployer/merge.go index 7805e7404a0..608c0b36e7b 100644 --- a/projects/gateway2/deployer/merge.go +++ b/projects/gateway2/deployer/merge.go @@ -3,6 +3,7 @@ package deployer import ( "github.com/solo-io/gloo/projects/gateway2/api/v1alpha1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) // mergePointers will decide whether to use dst or src without dereferencing or recursing @@ -49,6 +50,15 @@ func deepMergeSlices[T any](dst, src []T) []T { return dst } +func overrideSlices[T any](dst, src []T) []T { + // nil src override means just use dst + if src == nil { + return dst + } + + return src +} + // Check against base value func mergeComparable[T comparable](dst, src T) T { var t T @@ -130,6 +140,10 @@ func deepMergePodTemplate(dst, src *v1alpha1.Pod) *v1alpha1.Pod { dst.NodeSelector = deepMergeMaps(dst.GetNodeSelector(), src.GetNodeSelector()) dst.Affinity = deepMergeAffinity(dst.GetAffinity(), src.GetAffinity()) dst.Tolerations = deepMergeSlices(dst.GetTolerations(), src.GetTolerations()) + dst.GracefulShutdown = deepMergeGracefulShutdown(dst.GetGracefulShutdown(), src.GetGracefulShutdown()) + dst.TerminationGracePeriodSeconds = mergePointers(dst.TerminationGracePeriodSeconds, src.TerminationGracePeriodSeconds) + dst.ReadinessProbe = deepMergeProbe(dst.GetReadinessProbe(), src.GetReadinessProbe()) + dst.LivenessProbe = deepMergeProbe(dst.GetLivenessProbe(), src.GetLivenessProbe()) return dst } @@ -290,6 +304,124 @@ func deepMergePodAntiAffinity(dst, src *corev1.PodAntiAffinity) *corev1.PodAntiA return dst } +func deepMergeProbe(dst, src *corev1.Probe) *corev1.Probe { + // nil src override means just use dst + if src == nil { + return dst + } + + if dst == nil { + return src + } + + dst.ProbeHandler = deepMergeProbeHandler(dst.ProbeHandler, src.ProbeHandler) + dst.InitialDelaySeconds = mergeComparable(dst.InitialDelaySeconds, src.InitialDelaySeconds) + dst.TimeoutSeconds = mergeComparable(dst.TimeoutSeconds, src.TimeoutSeconds) + dst.PeriodSeconds = mergeComparable(dst.PeriodSeconds, src.PeriodSeconds) + dst.SuccessThreshold = mergeComparable(dst.SuccessThreshold, src.SuccessThreshold) + dst.FailureThreshold = mergeComparable(dst.FailureThreshold, src.FailureThreshold) + dst.TerminationGracePeriodSeconds = mergePointers(dst.TerminationGracePeriodSeconds, src.TerminationGracePeriodSeconds) + + return dst +} + +func deepMergeProbeHandler(dst, src corev1.ProbeHandler) corev1.ProbeHandler { + dst.Exec = deepMergeExecAction(dst.Exec, src.Exec) + dst.HTTPGet = deepMergeHTTPGetAction(dst.HTTPGet, src.HTTPGet) + dst.TCPSocket = deepMergeTCPSocketAction(dst.TCPSocket, src.TCPSocket) + dst.GRPC = deepMergeGRPCAction(dst.GRPC, src.GRPC) + + return dst +} + +func deepMergeExecAction(dst, src *corev1.ExecAction) *corev1.ExecAction { + // nil src override means just use dst + if src == nil { + return dst + } + + if dst == nil { + return src + } + + // Don't merge the command string as that can break the entire probe + dst.Command = overrideSlices(dst.Command, src.Command) + + return dst +} + +func deepMergeHTTPGetAction(dst, src *corev1.HTTPGetAction) *corev1.HTTPGetAction { + // nil src override means just use dst + if src == nil { + return dst + } + + if dst == nil { + return src + } + + dst.Path = mergeComparable(dst.Path, src.Path) + dst.Port = mergeIntOrString(dst.Port, src.Port) + dst.Host = mergeComparable(dst.Host, src.Host) + dst.Scheme = mergeComparable(dst.Scheme, src.Scheme) + dst.HTTPHeaders = deepMergeSlices(dst.HTTPHeaders, src.HTTPHeaders) + + return dst +} + +func mergeIntOrString(dst, src intstr.IntOrString) intstr.IntOrString { + // Do not deep merge as this can cause a conflict between the name and number of the port to access on the container + return mergeComparable(dst, src) +} + +func deepMergeTCPSocketAction(dst, src *corev1.TCPSocketAction) *corev1.TCPSocketAction { + // nil src override means just use dst + if src == nil { + return dst + } + + if dst == nil { + return src + } + + dst.Port = mergeIntOrString(dst.Port, src.Port) + dst.Host = mergeComparable(dst.Host, src.Host) + + return dst +} + +func deepMergeGRPCAction(dst, src *corev1.GRPCAction) *corev1.GRPCAction { + // nil src override means just use dst + if src == nil { + return dst + } + + if dst == nil { + return src + } + + dst.Port = mergeComparable(dst.Port, src.Port) + dst.Service = mergePointers(dst.Service, src.Service) + + return dst +} + +func deepMergeGracefulShutdown(dst, src *v1alpha1.GracefulShutdownSpec) *v1alpha1.GracefulShutdownSpec { + // nil src override means just use dst + if src == nil { + return dst + } + + if dst == nil { + return src + } + + dst.Enabled = mergePointers(dst.Enabled, src.Enabled) + dst.SleepTimeSeconds = mergePointers(dst.SleepTimeSeconds, src.SleepTimeSeconds) + + return dst +} + func deepMergeService(dst, src *v1alpha1.Service) *v1alpha1.Service { // nil src override means just use dst if src == nil { From 82785b3da1d6206c4ee4ce7a319afd3f8ad19749 Mon Sep 17 00:00:00 2001 From: Seth Heidkamp <61526534+sheidkamp@users.noreply.github.com> Date: Wed, 13 Nov 2024 15:26:51 -0500 Subject: [PATCH 8/8] Fix published Helm docs from main branch (#10334) --- changelog/v1.18.0-beta35/helm-doc-gen.yaml | 6 ++++++ docs/cmd/generate_docs.go | 8 +++----- pkg/github-action-utils/version.go | 4 +++- 3 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 changelog/v1.18.0-beta35/helm-doc-gen.yaml diff --git a/changelog/v1.18.0-beta35/helm-doc-gen.yaml b/changelog/v1.18.0-beta35/helm-doc-gen.yaml new file mode 100644 index 00000000000..4b97b85471d --- /dev/null +++ b/changelog/v1.18.0-beta35/helm-doc-gen.yaml @@ -0,0 +1,6 @@ +changelog: + - type: NON_USER_FACING + issueLink: https://github.com/solo-io/solo-projects/issues/6888 + resolvesIssue: false + description: >- + Update helm public docs generation to work from main branch and only pull in released changes diff --git a/docs/cmd/generate_docs.go b/docs/cmd/generate_docs.go index 0a5a61cd0d3..f0936f7de64 100644 --- a/docs/cmd/generate_docs.go +++ b/docs/cmd/generate_docs.go @@ -454,11 +454,9 @@ func fetchEnterpriseHelmValues(_ []string) error { if err != nil { return err } - version, err := semver.NewVersion(string(semverReleaseTag)) - if err != nil { - return err - } - minorReleaseTag := fmt.Sprintf("v%d.%d.x", version.Major(), version.Minor()) + + minorReleaseTag := "v" + string(semverReleaseTag) + files, err := githubutils.GetFilesFromGit(ctx, client, repoOwner, glooEnterpriseRepo, minorReleaseTag, path) if err != nil { return err diff --git a/pkg/github-action-utils/version.go b/pkg/github-action-utils/version.go index 167cbb7fe35..25189901f8e 100644 --- a/pkg/github-action-utils/version.go +++ b/pkg/github-action-utils/version.go @@ -35,7 +35,9 @@ func GetLatestEnterpriseVersion(repoRootPath string, repo string, owner string) return err } defer f.Close() - enterpriseVersion, err := version.GetLatestHelmChartVersionWithMaxVersion(version.EnterpriseHelmRepoIndex, version.GlooEE, true, maxGlooEVersion) + // get the latest version from the helm repo, include unstable versions so it works from the main branches + // for LTS branches, unstable versions will be filtered out by the version constraints + enterpriseVersion, err := version.GetLatestHelmChartVersionWithMaxVersion(version.EnterpriseHelmRepoIndex, version.GlooEE, false, maxGlooEVersion) if err != nil { return err }