Merge branch 'main' into hbc/karpenter-helmchart

Author: bcho

karpenter/cmd/controller/main.go

@@ -85,6 +85,7 @@ func main() {
 		}, "azure")
 	}
 
+	clusterVersion := lo.Must(utilsk8s.RetrieveClusterVersion(op.GetConfig()))
 	clusterCA := lo.Must(utilsk8s.RetrieveClusterCA(op.GetConfig()))
 
 	// nebius cloud provider...
@@ -97,6 +98,7 @@ func main() {
 			hubCloudProvider,
 			flexoptions.MustNewNebiusSDK(ctx),
 			op.GetClient(),
+			clusterVersion,
 			clusterCA,
 			wgAlloc,
 		)

karpenter/examples/nebius/cpu_nodepool.yaml

@@ -16,6 +16,12 @@ spec:
         - key: karpenter.azure.com/sku-cpu
           operator: Gt
           values: ["8"]
+      startupTaints:
+        # NOTE: this start up taint is required to tell karpenter to avoid
+        #       adding the cilium taint for scheduling decision.
+        - key: "node.cilium.io/agent-not-ready"
+          effect: "NoSchedule"
+          value: "true"
   limits:
     cpu: "1000"
-    memory: 1000Gi
+    memory: 1000Gi

karpenter/examples/nebius/gpu_nodepool.yaml

@@ -17,6 +17,12 @@ spec:
         - key: kubernetes.io/os
           operator: In
           values: ["linux"]
+      startupTaints:
+        # NOTE: this start up taint is required to tell karpenter to avoid
+        #       adding the cilium taint for scheduling decision.
+        - key: "node.cilium.io/agent-not-ready"
+          effect: "NoSchedule"
+          value: "true"
   limits:
     cpu: "1000"
     memory: 10000Gi

karpenter/go.mod

@@ -10,11 +10,12 @@ require (
 	github.com/go-logr/zapr v1.3.0
 	github.com/nebius/gosdk v0.0.0-20260218100913-7fb27c45819a
 	github.com/samber/lo v1.52.0
+	golang.org/x/sync v0.19.0
 	google.golang.org/grpc v1.79.1
-	google.golang.org/protobuf v1.36.11
 	k8s.io/api v0.35.1
 	k8s.io/apimachinery v0.35.1
 	k8s.io/client-go v0.35.1
+	k8s.io/utils v0.0.0-20260108192941-914a6e750570
 	sigs.k8s.io/controller-runtime v0.23.1
 	sigs.k8s.io/karpenter v1.7.1
 )
@@ -166,13 +167,13 @@ require (
 	golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
 	golang.org/x/net v0.49.0 // indirect
 	golang.org/x/oauth2 v0.34.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
 	golang.org/x/sys v0.40.0 // indirect
 	golang.org/x/term v0.39.0 // indirect
 	golang.org/x/text v0.33.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/dnaeon/go-vcr.v3 v3.2.0 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
@@ -186,7 +187,6 @@ require (
 	k8s.io/csi-translation-lib v0.35.0 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20260127142750-a19766b6e2d4 // indirect
-	k8s.io/utils v0.0.0-20260108192941-914a6e750570 // indirect
 	sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.14.3 // indirect
 	sigs.k8s.io/cluster-api v1.12.3 // indirect
 	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect

karpenter/pkg/apis/crds/flex.aks.azure.com_nebiusnodeclasses.yaml

@@ -61,6 +61,13 @@ spec:
                 description: OSDiskSizeGB is the size of the OS disk in GB.
                 format: int32
                 type: integer
+              maxPodsPerNode:
+                default: 110
+                description: |-
+                  MaxPodsPerNode is the maximum number of pods that can be scheduled on a single node.
+                  This value is advertised in the node's capacity and affects Karpenter's scheduling decisions.
+                format: int32
+                type: integer
               projectID:
                 description: ProjectID is the nebius project id to launch nodes in.
                 type: string

karpenter/pkg/apis/v1alpha1/nebius.go

@@ -76,6 +76,12 @@ type NebiusNodeClassSpec struct {
 	// +optional
 	WireguardPeerCIDR *string `json:"wireguardPeerCIDR,omitempty"`
 
+	// MaxPodsPerNode is the maximum number of pods that can be scheduled on a single node.
+	// This value is advertised in the node's capacity and affects Karpenter's scheduling decisions.
+	// +default=110
+	// +optional
+	MaxPodsPerNode *int32 `json:"maxPodsPerNode,omitempty"`
+
 	// TODO: other fields (kublet etc)
 }
 

karpenter/pkg/apis/v1alpha1/zz_generated.deepcopy.go

@@ -21,5 +21,6 @@ func IsQuotaError(err error) bool {
 	// FIXME: check how to identify the error using nebius sdk
 	errString := err.Error()
 	return strings.Contains(errString, "Quota failure") ||
-		strings.Contains(errString, "Not enough resources")
+		strings.Contains(errString, "Not enough resources") ||
+		strings.Contains(errString, "insufficient capacity") // example: insufficient capacity, rpc error: code = ResourceExhausted
 }