fix: compute node allocatable from real reservations and usable memory

README.md

@@ -149,9 +149,59 @@ Provisioned nodes carry, in addition to the well-known Karpenter labels:
 | `labels` | `map[string]string` | no | — | Extra hcloud labels on the Hetzner server (useful for cost attribution or firewall label-selectors) |
 | `userData` | `string` | no | — | Inline cloud-init / Talos machine config. Overridden by `userDataSecretRef` when both are set. |
 | `userDataSecretRef` | `object {namespace, name, key}` | no | — | Source `userData` from a Secret instead of inline. The Secret is read at server-create time; its value never appears in the NodeClass spec or git. Takes precedence over `userData`. |
+| `kubelet.systemReserved` | `map[string]string` | no | — | What your bootstrap passes to the kubelet as `--system-reserved`. Keys: `cpu`, `memory`, `ephemeral-storage`, `pid`. |
+| `kubelet.kubeReserved` | `map[string]string` | no | — | What your bootstrap passes as `--kube-reserved`. Same keys. |
+| `kubelet.evictionHard` | `map[string]string` | no | `memory.available: 100Mi`, `nodefs.available: 10%` | What your bootstrap passes as `--eviction-hard`. Values are a quantity (`400Mi`) or a percentage (`10%`). |
 
 Status exposes `conditions` (`ImagesReady`, `NetworkReady`, `ResourcesReady`, `UserDataReady`, aggregated into `Ready`) and `resolvedImages` (image ID per architecture).
 
+### Declaring kubelet reservations
+
+Karpenter decides which server type a pod fits on by subtracting reserved
+resources from a type's capacity. It cannot see your bootstrap, so anything your
+userData reserves has to be declared here as well:
+
+```yaml
+spec:
+  kubelet:
+    systemReserved: {cpu: 200m, memory: 512Mi}
+    kubeReserved:   {cpu: 200m, memory: 512Mi}
+    evictionHard:   {memory.available: 400Mi}
+```
+
+**These values describe your userData; they do not configure it.** Setting them
+here does not change what the node reserves, and the two must be kept in
+agreement. Declaring less than the bootstrap actually reserves is the failure
+worth knowing about: Karpenter then believes a server is larger than it is,
+picks one too small for the pod, and the pod stays `Pending` while the empty node
+is consolidated away and replaced — indefinitely, without an error anywhere.
+
+Karpenter's own `nodeclaim/consistency` check will not catch this. It compares
+capacity rather than allocatable, and only reports a shortfall beyond 10%.
+
+Omitting the block leaves the previous behaviour in place — a flat `100m`/`100Mi`
+reserve plus the kubelet's own default eviction thresholds — so upgrading cannot
+silently raise a node's advertised capacity. Declaring a block replaces that
+default with exactly what you state, so declare everything your bootstrap
+reserves, not just the parts that differ.
+
+### Advertised vs. usable memory
+
+Hetzner advertises the memory a VM is allocated, but the guest kernel never sees
+all of it — firmware, the kernel image and per-page structures take a cut that
+grows with the size of the machine. A server advertised as 32Gi reports about
+31.3Gi; one advertised as 4Gi reports about 3.7Gi.
+
+The provider holds back `VM_MEMORY_OVERHEAD_PERCENT` (default `0.075`) to cover
+this. It is only an estimate, and it is only used until a node of that server
+type and image registers: from then on the capacity that node reported is used
+instead, keyed by server type and resolved image. Look for
+`recorded server type memory capacity measured from a registered node` in the
+logs.
+
+If nodes still register smaller than Karpenter expected, raise the percentage.
+Lower it only with measurements in hand.
+
 ## Examples
 
 Ready-to-apply examples live in the [`examples/`](examples/) directory.  Each
@@ -177,6 +227,7 @@ comments explaining every field.
 |---------|----------|-------------|
 | `HCLOUD_TOKEN` | yes | Hetzner Cloud API token |
 | `CLUSTER_NAME` | yes | Cluster identifier; scopes managed servers |
+| `VM_MEMORY_OVERHEAD_PERCENT` | no (0.075) | Fraction of advertised memory assumed invisible to the guest, used until a node of that type reports its real capacity. Must be `>= 0` and `< 1`. |
 | `METRICS_PORT` | no (8080) | Prometheus metrics port |
 | `HEALTH_PROBE_PORT` | no (8081) | Health/readiness probe port |
 

charts/karpenter-provider-hetzner/crds/karpenter.hetzner.cloud_hcloudnodeclasses.yaml

@@ -83,6 +83,61 @@ spec:
                 required:
                 - family
                 type: object
+              kubelet:
+                description: |-
+                  Kubelet declares the reservations this node class's bootstrap applies to
+                  the kubelet, so Karpenter can subtract them when computing how much of a
+                  server a pod can actually use.
+
+                  This provider does not render the bootstrap: userData is supplied by the
+                  operator, so these values describe what that userData does rather than
+                  configure it. They must be kept in agreement with it. Declaring less than
+                  the bootstrap reserves makes Karpenter believe a server is larger than it
+                  is, which strands pods on nodes too small for them.
+                properties:
+                  evictionHard:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      EvictionHard is the kubelet's hard eviction thresholds. Memory held back
+                      for eviction is memory a pod cannot have, so it comes out of allocatable
+                      the same way a reservation does. Values are either a quantity ("400Mi") or
+                      a percentage of capacity ("10%").
+
+                      Soft eviction is deliberately not modelled: it is a warning threshold that
+                      does not reduce allocatable.
+                    type: object
+                    x-kubernetes-validations:
+                    - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']
+                      rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])
+                  kubeReserved:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      KubeReserved is resources reserved for Kubernetes system daemons, matching
+                      the kubelet's --kube-reserved.
+                    type: object
+                    x-kubernetes-validations:
+                    - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid']
+                      rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage'
+                        || x=='pid')
+                    - message: kubeReserved value cannot be a negative resource quantity
+                      rule: self.all(x, !self[x].startsWith('-'))
+                  systemReserved:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      SystemReserved is resources reserved for OS system daemons, matching the
+                      kubelet's --system-reserved.
+                    type: object
+                    x-kubernetes-validations:
+                    - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid']
+                      rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage'
+                        || x=='pid')
+                    - message: systemReserved value cannot be a negative resource
+                        quantity
+                      rule: self.all(x, !self[x].startsWith('-'))
+                type: object
               labels:
                 additionalProperties:
                   type: string

charts/karpenter-provider-hetzner/templates/deployment.yaml

@@ -68,6 +68,12 @@ spec:
                   key: {{ .Values.auth.secretRef.key }}
             - name: CLUSTER_NAME
               value: {{ .Values.clusterName | quote }}
+            # Emitted unconditionally rather than through `with`, which skips
+            # falsy values: 0 is a legitimate setting (trust Hetzner's advertised
+            # memory exactly) and must reach the operator, not fall back to the
+            # default.
+            - name: VM_MEMORY_OVERHEAD_PERCENT
+              value: {{ .Values.vmMemoryOverheadPercent | quote }}
             - name: METRICS_PORT
               value: {{ .Values.metrics.port | quote }}
             - name: HEALTH_PROBE_PORT

charts/karpenter-provider-hetzner/values.yaml

@@ -52,6 +52,16 @@ args: []
 # Required: scopes managed servers so multiple clusters can share one Hetzner project.
 clusterName: ""
 
+# Fraction of a server type's advertised memory assumed to be invisible to the
+# guest. Hetzner advertises what the VM is allocated; the kernel and firmware
+# take a cut that varies with machine size, so this is an estimate, used only
+# until a node of that type registers and reports its real capacity.
+#
+# Raise it if nodes still register smaller than Karpenter expects. Lower it only
+# with measurements in hand: too low tells Karpenter a server is bigger than it
+# is, which strands pods on nodes that cannot hold them.
+vmMemoryOverheadPercent: 0.075
+
 metrics:
   port: 8080
 healthProbe:

cmd/controller/main.go

@@ -15,6 +15,7 @@ import (
 	_ "github.com/paperclipinc/karpenter-provider-hetzner/pkg/apis/v1"
 
 	hetznercp "github.com/paperclipinc/karpenter-provider-hetzner/pkg/cloudprovider"
+	instancetypecapacity "github.com/paperclipinc/karpenter-provider-hetzner/pkg/controllers/instancetype/capacity"
 	"github.com/paperclipinc/karpenter-provider-hetzner/pkg/controllers/nodeclass"
 	hetznerop "github.com/paperclipinc/karpenter-provider-hetzner/pkg/operator"
 	"github.com/paperclipinc/karpenter-provider-hetzner/pkg/providers/imagefamily"
@@ -40,7 +41,7 @@ func main() {
 
 	// Create the three providers.
 	instanceProvider := instance.NewProviderWithPlacementGroups(&hcloudClient.Server, &hcloudClient.PlacementGroup, cfg.ClusterName, &hcloudClient.Action)
-	typeProvider := instancetype.NewProvider(&hcloudClient.ServerType)
+	typeProvider := instancetype.NewProvider(&hcloudClient.ServerType, cfg.VMMemoryOverheadPercent)
 	imageProvider := imagefamily.NewProvider(&hcloudClient.Image)
 
 	// Create the cloud provider.
@@ -74,5 +75,6 @@ func main() {
 			op.InstanceTypeStore,
 		),
 		nodeClassController,
+		instancetypecapacity.NewController(op.GetClient(), typeProvider),
 	)...).Start(ctx)
 }

pkg/apis/v1/hcloudnodeclass_types.go

@@ -68,6 +68,48 @@ type HCloudNodeClassSpec struct {
 	// +kubebuilder:default=true
 	// +optional
 	EnablePublicIPv6 *bool `json:"enablePublicIPv6,omitempty"`
+
+	// Kubelet declares the reservations this node class's bootstrap applies to
+	// the kubelet, so Karpenter can subtract them when computing how much of a
+	// server a pod can actually use.
+	//
+	// This provider does not render the bootstrap: userData is supplied by the
+	// operator, so these values describe what that userData does rather than
+	// configure it. They must be kept in agreement with it. Declaring less than
+	// the bootstrap reserves makes Karpenter believe a server is larger than it
+	// is, which strands pods on nodes too small for them.
+	// +optional
+	Kubelet *KubeletConfiguration `json:"kubelet,omitempty"`
+}
+
+// KubeletConfiguration mirrors the subset of kubelet settings that change how
+// much of a node is schedulable. Only the reservation knobs are modelled;
+// anything that does not move allocatable is deliberately absent.
+type KubeletConfiguration struct {
+	// SystemReserved is resources reserved for OS system daemons, matching the
+	// kubelet's --system-reserved.
+	// +kubebuilder:validation:XValidation:message="valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid']",rule="self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid')"
+	// +kubebuilder:validation:XValidation:message="systemReserved value cannot be a negative resource quantity",rule="self.all(x, !self[x].startsWith('-'))"
+	// +optional
+	SystemReserved map[string]string `json:"systemReserved,omitempty"`
+
+	// KubeReserved is resources reserved for Kubernetes system daemons, matching
+	// the kubelet's --kube-reserved.
+	// +kubebuilder:validation:XValidation:message="valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid']",rule="self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid')"
+	// +kubebuilder:validation:XValidation:message="kubeReserved value cannot be a negative resource quantity",rule="self.all(x, !self[x].startsWith('-'))"
+	// +optional
+	KubeReserved map[string]string `json:"kubeReserved,omitempty"`
+
+	// EvictionHard is the kubelet's hard eviction thresholds. Memory held back
+	// for eviction is memory a pod cannot have, so it comes out of allocatable
+	// the same way a reservation does. Values are either a quantity ("400Mi") or
+	// a percentage of capacity ("10%").
+	//
+	// Soft eviction is deliberately not modelled: it is a warning threshold that
+	// does not reduce allocatable.
+	// +kubebuilder:validation:XValidation:message="valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])"
+	// +optional
+	EvictionHard map[string]string `json:"evictionHard,omitempty"`
 }
 
 // UserDataSecretReference points at a Secret key holding the server userData.

pkg/cloudprovider/cloudprovider.go

@@ -94,7 +94,7 @@ func (cp *CloudProvider) Create(ctx context.Context, nodeClaim *karpv1.NodeClaim
 	}
 
 	// Get instance types for the node class locations.
-	instanceTypes, err := cp.typeProvider.List(ctx, nodeClass.Spec.Locations)
+	instanceTypes, err := cp.typeProvider.List(ctx, nodeClass)
 	if err != nil {
 		return nil, fmt.Errorf("listing instance types: %w", err)
 	}
@@ -251,7 +251,7 @@ func (cp *CloudProvider) GetInstanceTypes(ctx context.Context, nodePool *karpv1.
 		return nil, fmt.Errorf("resolving node class for node pool %s: %w", nodePool.Name, err)
 	}
 
-	return cp.typeProvider.List(ctx, nodeClass.Spec.Locations)
+	return cp.typeProvider.List(ctx, nodeClass)
 }
 
 // IsDrifted determines whether the given NodeClaim has drifted from its desired state.

pkg/cloudprovider/cloudprovider_test.go

@@ -134,7 +134,7 @@ func buildCP(t *testing.T, nc *apiv1.HCloudNodeClass, server *hcloud.Server) (*c
 
 	cp := cloudprovider.NewCloudProvider(kube,
 		instance.NewProvider(fsc, "test-cluster"),
-		instancetype.NewProvider(stc),
+		instancetype.NewProvider(stc, 0),
 		imagefamily.NewProvider(imgc))
 
 	nodeClaim := &karpv1.NodeClaim{ObjectMeta: metav1.ObjectMeta{Name: "claim"}}
@@ -207,7 +207,7 @@ func buildCPWithTypes(t *testing.T, nc *apiv1.HCloudNodeClass, types []*hcloud.S
 	fsc := &fakeServerClient{servers: map[int64]*hcloud.Server{}}
 	stc := &fakeServerTypeClient{types: types}
 	imgc := &fakeImageClient{images: []*hcloud.Image{{ID: 42, Description: "Ubuntu 24.04", Architecture: hcloud.ArchitectureX86}}}
-	typeProvider := instancetype.NewProvider(stc)
+	typeProvider := instancetype.NewProvider(stc, 0)
 	cp := cloudprovider.NewCloudProvider(kube,
 		instance.NewProvider(fsc, "test-cluster"),
 		typeProvider,
@@ -308,7 +308,7 @@ func TestCreate_InsufficientCapacityMarksUnavailable(t *testing.T) {
 		t.Fatal("expected error on capacity failure")
 	}
 	// The offering for (cx22, nbg1) should now be marked unavailable.
-	its, lerr := typeProvider.List(context.Background(), []string{"nbg1"})
+	its, lerr := typeProvider.List(context.Background(), &apiv1.HCloudNodeClass{Spec: apiv1.HCloudNodeClassSpec{Locations: []string{"nbg1"}}})
 	if lerr != nil {
 		t.Fatal(lerr)
 	}
@@ -460,7 +460,7 @@ func TestCreate_UserDataFromSecret(t *testing.T) {
 	imgc := &fakeImageClient{images: []*hcloud.Image{{ID: 42, Description: "Ubuntu 24.04", Architecture: hcloud.ArchitectureX86}}}
 	cp := cloudprovider.NewCloudProvider(kube,
 		instance.NewProvider(fsc, "test-cluster"),
-		instancetype.NewProvider(stc),
+		instancetype.NewProvider(stc, 0),
 		imagefamily.NewProvider(imgc))
 
 	if _, err := cp.Create(context.Background(), createNodeClaim()); err != nil {
@@ -485,7 +485,7 @@ func TestCreate_UserDataInlineWhenNoRef(t *testing.T) {
 	imgc := &fakeImageClient{images: []*hcloud.Image{{ID: 42, Description: "Ubuntu 24.04", Architecture: hcloud.ArchitectureX86}}}
 	cp := cloudprovider.NewCloudProvider(kube,
 		instance.NewProvider(fsc, "test-cluster"),
-		instancetype.NewProvider(stc),
+		instancetype.NewProvider(stc, 0),
 		imagefamily.NewProvider(imgc))
 
 	if _, err := cp.Create(context.Background(), createNodeClaim()); err != nil {
@@ -519,7 +519,7 @@ func TestCreate_UserDataSecretKeyMissing(t *testing.T) {
 	imgc := &fakeImageClient{images: []*hcloud.Image{{ID: 42, Description: "Ubuntu 24.04", Architecture: hcloud.ArchitectureX86}}}
 	cp := cloudprovider.NewCloudProvider(kube,
 		instance.NewProvider(fsc, "test-cluster"),
-		instancetype.NewProvider(stc),
+		instancetype.NewProvider(stc, 0),
 		imagefamily.NewProvider(imgc))
 
 	_, err := cp.Create(context.Background(), createNodeClaim())
@@ -546,7 +546,7 @@ func TestCreate_UserDataSecretMissing(t *testing.T) {
 	imgc := &fakeImageClient{images: []*hcloud.Image{{ID: 42, Description: "Ubuntu 24.04", Architecture: hcloud.ArchitectureX86}}}
 	cp := cloudprovider.NewCloudProvider(kube,
 		instance.NewProvider(fsc, "test-cluster"),
-		instancetype.NewProvider(stc),
+		instancetype.NewProvider(stc, 0),
 		imagefamily.NewProvider(imgc))
 
 	_, err := cp.Create(context.Background(), createNodeClaim())