Skip to content

Commit 0e43ed0

Browse files
shengnuoshengnuo
committed
Simply Ingress in NIMService and NeMo microservice CRs and let NIM Operator create HTTPRoute objects from Gateway API
Signed-off-by: Sheng Lin <shelin@nvidia.com>
1 parent d50ef6f commit 0e43ed0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+8251
-298
lines changed

‎api/apps/v1alpha1/common_types.go‎

Lines changed: 126 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,14 @@ limitations under the License.
1717
package v1alpha1
1818

1919
import (
20+
"fmt"
21+
2022
promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
2123
autoscalingv2 "k8s.io/api/autoscaling/v2"
2224
corev1 "k8s.io/api/core/v1"
2325
networkingv1 "k8s.io/api/networking/v1"
2426
"k8s.io/utils/ptr"
27+
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
2528
)
2629

2730
const (
@@ -38,7 +41,39 @@ const (
3841
// Expose defines attributes to expose the service.
3942
type Expose struct {
4043
Service Service `json:"service,omitempty"`
41-
Ingress Ingress `json:"ingress,omitempty"`
44+
// Deprecated: Use .spec.router instead.
45+
// Ingress Ingress `json:"ingress,omitempty"`
46+
// Deprecated: Use .spec.router instead.
47+
// HTTPRoute HTTPRoute `json:"httpRoute,omitempty"`
48+
}
49+
50+
// +kubebuilder:validation:XValidation:rule="!(has(self.gateway) && has(self.ingressClass))", message="ingressClass and gateway cannot be specified together"
51+
type Router struct {
52+
// HostDomainName is the domain name of the hostname matched by the router.
53+
// The hostname is constructed as "<nimServiceName>.<hostDomainName>", where the <nimServiceName> a subdomain of the matched hostname.
54+
// eg. example.com for "<nimServiceName>.example.com"
55+
// +kubebuilder:validation:MinLength=1
56+
// +kubebuilder:validation:MaxLength=63
57+
// +kubebuilder:validation:Pattern=`^(([a-z0-9][a-z0-9\-]*[a-z0-9])|[a-z0-9]+\.)*([a-z]+|xn\-\-[a-z0-9]+)\.?$`
58+
HostDomainName string `json:"hostDomainName,omitempty"`
59+
60+
// Annotations for the router, e.g. for ingress class or gateway
61+
Annotations map[string]string `json:"annotations,omitempty"`
62+
63+
// IngressClass is the class of the ingress controller to use for the created ingress.
64+
IngressClass *string `json:"ingressClass,omitempty"`
65+
66+
// Gateway is the gateway to use for the created HTTPRoute.
67+
Gateway *Gateway `json:"gateway,omitempty"`
68+
}
69+
70+
type Gateway struct {
71+
// +kubebuilder:validation:MinLength=1
72+
// Namespace of the gateway
73+
Namespace string `json:"namespace"`
74+
// +kubebuilder:validation:MinLength=1
75+
// Name of the gateway
76+
Name string `json:"name"`
4277
}
4378

4479
// Service defines attributes to create a service.
@@ -67,12 +102,6 @@ type Service struct {
67102
Annotations map[string]string `json:"annotations,omitempty"`
68103
}
69104

70-
// ExposeV1 defines attributes to expose the service.
71-
type ExposeV1 struct {
72-
Service Service `json:"service,omitempty"`
73-
Ingress IngressV1 `json:"ingress,omitempty"`
74-
}
75-
76105
// Metrics defines attributes to setup metrics collection.
77106
type Metrics struct {
78107
Enabled *bool `json:"enabled,omitempty"`
@@ -119,6 +148,32 @@ type Ingress struct {
119148
Spec networkingv1.IngressSpec `json:"spec,omitempty"`
120149
}
121150

151+
// HTTPRoute defines attributes to HTTPRoute in Gateway API.
152+
type HTTPRoute struct {
153+
Enabled *bool `json:"enabled,omitempty"`
154+
Annotations map[string]string `json:"annotations,omitempty"`
155+
Spec *HTTPRouteSpec `json:"spec,omitempty"`
156+
}
157+
158+
type HTTPRouteSpec struct {
159+
gatewayv1.CommonRouteSpec `json:",inline"`
160+
Host gatewayv1.Hostname `json:"host,omitempty"`
161+
Paths []HTTPPathMatch `json:"paths,omitempty"`
162+
}
163+
164+
type HTTPPathMatch struct {
165+
// Type specifies how to match against the path Value.
166+
// +optional
167+
// +kubebuilder:default=PathPrefix
168+
Type *gatewayv1.PathMatchType `json:"type,omitempty"`
169+
170+
// Value of the HTTP path to match against.
171+
// +optional
172+
// +kubebuilder:default="/"
173+
// +kubebuilder:validation:MaxLength=1024
174+
Value *string `json:"value,omitempty"`
175+
}
176+
122177
// IngressV1 defines attributes for ingress
123178
//
124179
// +kubebuilder:validation:XValidation:rule="(has(self.spec) && has(self.enabled) && self.enabled) || !has(self.enabled) || !self.enabled", message="spec cannot be nil when ingress is enabled"
@@ -142,46 +197,81 @@ type ResourceRequirements struct {
142197
Requests corev1.ResourceList `json:"requests,omitempty" protobuf:"bytes,2,rep,name=requests,casttype=ResourceList,castkey=ResourceName"`
143198
}
144199

145-
func (i *IngressV1) GenerateNetworkingV1IngressSpec(name string) networkingv1.IngressSpec {
146-
if i.Spec == nil {
147-
return networkingv1.IngressSpec{}
200+
func (r *Router) GenerateGatewayHTTPRouteSpec(name string) gatewayv1.HTTPRouteSpec {
201+
if r.Gateway == nil {
202+
return gatewayv1.HTTPRouteSpec{}
148203
}
149204

150-
ingressSpec := networkingv1.IngressSpec{
151-
IngressClassName: &i.Spec.IngressClassName,
152-
Rules: []networkingv1.IngressRule{
205+
return gatewayv1.HTTPRouteSpec{
206+
CommonRouteSpec: gatewayv1.CommonRouteSpec{
207+
ParentRefs: []gatewayv1.ParentReference{
208+
{
209+
Name: gatewayv1.ObjectName(r.Gateway.Name),
210+
Namespace: ptr.To(gatewayv1.Namespace(r.Gateway.Namespace)),
211+
},
212+
},
213+
},
214+
Hostnames: []gatewayv1.Hostname{gatewayv1.Hostname(r.getHostname(name))},
215+
Rules: []gatewayv1.HTTPRouteRule{
153216
{
154-
Host: i.Spec.Host,
155-
IngressRuleValue: networkingv1.IngressRuleValue{
156-
HTTP: &networkingv1.HTTPIngressRuleValue{},
217+
BackendRefs: []gatewayv1.HTTPBackendRef{
218+
{
219+
BackendRef: gatewayv1.BackendRef{
220+
BackendObjectReference: gatewayv1.BackendObjectReference{
221+
Name: gatewayv1.ObjectName(name),
222+
Port: ptr.To(gatewayv1.PortNumber(DefaultAPIPort)),
223+
},
224+
},
225+
},
226+
},
227+
Matches: []gatewayv1.HTTPRouteMatch{
228+
{
229+
Path: &gatewayv1.HTTPPathMatch{
230+
Type: ptr.To(gatewayv1.PathMatchPathPrefix),
231+
Value: ptr.To("/"),
232+
},
233+
},
157234
},
158235
},
159236
},
160237
}
238+
}
161239

162-
svcBackend := networkingv1.IngressBackend{
163-
Service: &networkingv1.IngressServiceBackend{
164-
Name: name,
165-
Port: networkingv1.ServiceBackendPort{
166-
Name: DefaultNamedPortAPI,
240+
func (r *Router) getHostname(name string) string {
241+
return fmt.Sprintf("%s.%s", name, r.HostDomainName)
242+
}
243+
244+
func (r *Router) GenerateIngressSpec(name string) networkingv1.IngressSpec {
245+
if r.IngressClass == nil {
246+
return networkingv1.IngressSpec{}
247+
}
248+
249+
return networkingv1.IngressSpec{
250+
IngressClassName: r.IngressClass,
251+
Rules: []networkingv1.IngressRule{
252+
{
253+
Host: r.getHostname(name),
254+
IngressRuleValue: networkingv1.IngressRuleValue{
255+
HTTP: &networkingv1.HTTPIngressRuleValue{
256+
Paths: []networkingv1.HTTPIngressPath{
257+
{
258+
Path: "/",
259+
PathType: ptr.To(networkingv1.PathTypePrefix),
260+
Backend: networkingv1.IngressBackend{
261+
Service: &networkingv1.IngressServiceBackend{
262+
Name: name,
263+
Port: networkingv1.ServiceBackendPort{
264+
Name: DefaultNamedPortAPI,
265+
},
266+
},
267+
},
268+
},
269+
},
270+
},
271+
},
167272
},
168273
},
169274
}
170-
if len(i.Spec.Paths) == 0 {
171-
ingressSpec.Rules[0].HTTP.Paths = append(ingressSpec.Rules[0].HTTP.Paths, networkingv1.HTTPIngressPath{
172-
Path: "/",
173-
PathType: ptr.To(networkingv1.PathTypePrefix),
174-
Backend: svcBackend,
175-
})
176-
}
177-
for _, path := range i.Spec.Paths {
178-
ingressSpec.Rules[0].HTTP.Paths = append(ingressSpec.Rules[0].HTTP.Paths, networkingv1.HTTPIngressPath{
179-
Path: path.Path,
180-
PathType: path.PathType,
181-
Backend: svcBackend,
182-
})
183-
}
184-
return ingressSpec
185275
}
186276

187277
type IngressSpec struct {

‎api/apps/v1alpha1/nemo_customizer_types.go‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3434
"k8s.io/apimachinery/pkg/util/intstr"
3535
"k8s.io/utils/ptr"
36+
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
3637

3738
rendertypes "github.com/NVIDIA/k8s-nim-operator/internal/render/types"
3839
utils "github.com/NVIDIA/k8s-nim-operator/internal/utils"
@@ -79,7 +80,8 @@ type NemoCustomizerSpec struct {
7980
Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
8081
// +kubebuilder:validation:XValidation:rule="!(has(self.service.grpcPort))", message="unsupported field: spec.expose.service.grpcPort"
8182
// +kubebuilder:validation:XValidation:rule="!(has(self.service.metricsPort))", message="unsupported field: spec.expose.service.metricsPort"
82-
Expose ExposeV1 `json:"expose,omitempty"`
83+
Expose Expose `json:"expose,omitempty"`
84+
Router Router `json:"router,omitempty"`
8385
Scale Autoscaling `json:"scale,omitempty"`
8486
Metrics Metrics `json:"metrics,omitempty"`
8587

@@ -635,12 +637,20 @@ func (n *NemoCustomizer) IsAutoScalingEnabled() bool {
635637

636638
// IsIngressEnabled returns true if ingress is enabled for NemoCustomizer deployment.
637639
func (n *NemoCustomizer) IsIngressEnabled() bool {
638-
return n.Spec.Expose.Ingress.Enabled != nil && *n.Spec.Expose.Ingress.Enabled
640+
return n.Spec.Router.IngressClass != nil && *n.Spec.Router.IngressClass != ""
639641
}
640642

641643
// GetIngressSpec returns the Ingress spec NemoCustomizer deployment.
642644
func (n *NemoCustomizer) GetIngressSpec() networkingv1.IngressSpec {
643-
return n.Spec.Expose.Ingress.GenerateNetworkingV1IngressSpec(n.GetName())
645+
return n.Spec.Router.GenerateIngressSpec(n.GetName())
646+
}
647+
648+
func (n *NemoCustomizer) IsHTTPRouteEnabled() bool {
649+
return n.Spec.Router.Gateway != nil && n.Spec.Router.Gateway.Name != "" && n.Spec.Router.Gateway.Namespace != ""
650+
}
651+
652+
func (n *NemoCustomizer) GetHTTPRouteSpec() gatewayv1.HTTPRouteSpec {
653+
return n.Spec.Router.GenerateGatewayHTTPRouteSpec(n.GetName())
644654
}
645655

646656
// IsServiceMonitorEnabled returns true if servicemonitor is enabled for NemoCustomizer deployment.
@@ -839,6 +849,20 @@ func (n *NemoCustomizer) GetIngressParams() *rendertypes.IngressParams {
839849
return params
840850
}
841851

852+
// GetHTTPRouteParams returns params to render HTTPRoute from templates.
853+
func (n *NemoCustomizer) GetHTTPRouteParams() *rendertypes.HTTPRouteParams {
854+
params := &rendertypes.HTTPRouteParams{}
855+
params.Enabled = n.IsHTTPRouteEnabled()
856+
857+
// Set metadata
858+
params.Name = n.GetName()
859+
params.Namespace = n.GetNamespace()
860+
params.Labels = n.GetServiceLabels()
861+
params.Annotations = n.GetHTTPRouteAnnotations()
862+
params.Spec = n.GetHTTPRouteSpec()
863+
return params
864+
}
865+
842866
// GetRoleParams returns params to render Role from templates.
843867
func (n *NemoCustomizer) GetRoleParams() *rendertypes.RoleParams {
844868
params := &rendertypes.RoleParams{}
@@ -1017,12 +1041,21 @@ func (n *NemoCustomizer) GetServicePort() int32 {
10171041
func (n *NemoCustomizer) GetIngressAnnotations() map[string]string {
10181042
NemoCustomizerAnnotations := n.GetNemoCustomizerAnnotations()
10191043

1020-
if n.Spec.Expose.Ingress.Annotations != nil {
1021-
return utils.MergeMaps(NemoCustomizerAnnotations, n.Spec.Expose.Ingress.Annotations)
1044+
if n.Spec.Router.Annotations != nil {
1045+
return utils.MergeMaps(NemoCustomizerAnnotations, n.Spec.Router.Annotations)
10221046
}
10231047
return NemoCustomizerAnnotations
10241048
}
10251049

1050+
func (n *NemoCustomizer) GetHTTPRouteAnnotations() map[string]string {
1051+
annotations := n.GetNemoCustomizerAnnotations()
1052+
1053+
if n.Spec.Router.Annotations != nil {
1054+
return utils.MergeMaps(annotations, n.Spec.Router.Annotations)
1055+
}
1056+
return annotations
1057+
}
1058+
10261059
// GetServiceAnnotations return standard and customized service annotations.
10271060
func (n *NemoCustomizer) GetServiceAnnotations() map[string]string {
10281061
NemoCustomizerAnnotations := n.GetNemoCustomizerAnnotations()

‎api/apps/v1alpha1/nemo_datastore_types.go‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3131
"k8s.io/apimachinery/pkg/util/intstr"
3232
"k8s.io/utils/ptr"
33+
gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
3334

3435
rendertypes "github.com/NVIDIA/k8s-nim-operator/internal/render/types"
3536
utils "github.com/NVIDIA/k8s-nim-operator/internal/utils"
@@ -70,7 +71,8 @@ type NemoDatastoreSpec struct {
7071
Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
7172
// +kubebuilder:validation:XValidation:rule="!(has(self.service.grpcPort))", message="unsupported field: spec.expose.service.grpcPort"
7273
// +kubebuilder:validation:XValidation:rule="!(has(self.service.metricsPort))", message="unsupported field: spec.expose.service.metricsPort"
73-
Expose ExposeV1 `json:"expose,omitempty"`
74+
Expose Expose `json:"expose,omitempty"`
75+
Router Router `json:"router,omitempty"`
7476
Scale Autoscaling `json:"scale,omitempty"`
7577
Metrics Metrics `json:"metrics,omitempty"`
7678
// +kubebuilder:validation:Minimum=1
@@ -764,12 +766,20 @@ func (n *NemoDatastore) IsAutoScalingEnabled() bool {
764766

765767
// IsIngressEnabled returns true if ingress is enabled for NemoDatastore deployment.
766768
func (n *NemoDatastore) IsIngressEnabled() bool {
767-
return n.Spec.Expose.Ingress.Enabled != nil && *n.Spec.Expose.Ingress.Enabled
769+
return n.Spec.Router.IngressClass != nil && *n.Spec.Router.IngressClass != ""
770+
}
771+
772+
func (n *NemoDatastore) IsHTTPRouteEnabled() bool {
773+
return n.Spec.Router.Gateway != nil && n.Spec.Router.Gateway.Name != "" && n.Spec.Router.Gateway.Namespace != ""
768774
}
769775

770776
// GetIngressSpec returns the Ingress spec NemoDatastore deployment.
771777
func (n *NemoDatastore) GetIngressSpec() networkingv1.IngressSpec {
772-
return n.Spec.Expose.Ingress.GenerateNetworkingV1IngressSpec(n.GetName())
778+
return n.Spec.Router.GenerateIngressSpec(n.GetName())
779+
}
780+
781+
func (n *NemoDatastore) GetHTTPRouteSpec() gatewayv1.HTTPRouteSpec {
782+
return n.Spec.Router.GenerateGatewayHTTPRouteSpec(n.GetName())
773783
}
774784

775785
// IsServiceMonitorEnabled returns true if servicemonitor is enabled for NemoDatastore deployment.
@@ -967,6 +977,20 @@ func (n *NemoDatastore) GetIngressParams() *rendertypes.IngressParams {
967977
return params
968978
}
969979

980+
// GetHTTPRouteParams returns params to render HTTPRoute from templates.
981+
func (n *NemoDatastore) GetHTTPRouteParams() *rendertypes.HTTPRouteParams {
982+
params := &rendertypes.HTTPRouteParams{}
983+
params.Enabled = n.IsHTTPRouteEnabled()
984+
985+
// Set metadata
986+
params.Name = n.GetName()
987+
params.Namespace = n.GetNamespace()
988+
params.Labels = n.GetServiceLabels()
989+
params.Annotations = n.GetHTTPRouteAnnotations()
990+
params.Spec = n.GetHTTPRouteSpec()
991+
return params
992+
}
993+
970994
// GetRoleParams returns params to render Role from templates.
971995
func (n *NemoDatastore) GetRoleParams() *rendertypes.RoleParams {
972996
params := &rendertypes.RoleParams{}
@@ -1071,12 +1095,21 @@ func (n *NemoDatastore) GetServiceMonitorParams() *rendertypes.ServiceMonitorPar
10711095
func (n *NemoDatastore) GetIngressAnnotations() map[string]string {
10721096
NemoDatastoreAnnotations := n.GetNemoDatastoreAnnotations()
10731097

1074-
if n.Spec.Expose.Ingress.Annotations != nil {
1075-
return utils.MergeMaps(NemoDatastoreAnnotations, n.Spec.Expose.Ingress.Annotations)
1098+
if n.Spec.Router.Annotations != nil {
1099+
return utils.MergeMaps(NemoDatastoreAnnotations, n.Spec.Router.Annotations)
10761100
}
10771101
return NemoDatastoreAnnotations
10781102
}
10791103

1104+
func (n *NemoDatastore) GetHTTPRouteAnnotations() map[string]string {
1105+
annotations := n.GetNemoDatastoreAnnotations()
1106+
1107+
if n.Spec.Router.Annotations != nil {
1108+
return utils.MergeMaps(annotations, n.Spec.Router.Annotations)
1109+
}
1110+
return annotations
1111+
}
1112+
10801113
func (n *NemoDatastore) GetServiceAnnotations() map[string]string {
10811114
NemoDatastoreAnnotations := n.GetNemoDatastoreAnnotations()
10821115

0 commit comments

Comments
 (0)