19446 Kubernetes CPU and Memory resource services: Support Pod-level limits and requests

CMK-31901

Change-Id: I65b145e91ee0d942b964128140dbb83dacfdf7f1

Author: relrod

.werks/19446.md

@@ -0,0 +1,29 @@
+[//]: # (werk v3)
+# Kubernetes CPU and Memory resource services: Support Pod-level limits and requests
+
+key        | value
+---------- | ---
+date       | 2026-03-31T03:46:02.522347+00:00
+version    | 2.5.0b5
+class      | feature
+edition    | community
+component  | checks
+level      | 1
+compatible | no
+
+The _CPU resources_ and _Memory resources_ services on Kubernetes hosts now take
+Pod-level resource limits and requests into account. Previously, only
+container-level values were summed.
+
+Request and limit are handled independently: if a Pod specifies a pod-level
+request, that value is used instead of summing the container requests for that
+Pod. Likewise for limits. A Pod can have a pod-level request but container-level
+limits, or vice versa.
+
+The service output now indicates pod-level values separately, e.g.
+"2/3 containers with requests, 1 pod with pod-level requests".
+
+If your cluster uses Pod-level resource limits or requests, the values reported
+by these services may change after upgrading. If you have configured thresholds
+for _CPU resources_ or _Memory resources_, review them to ensure they still
+reflect your expectations.

packages/cmk-plugins/cmk/plugins/kube/agent_handlers/common.py

@@ -124,28 +124,65 @@ def kube_object_namespace_name(kube_object: KubeNamespacedObj) -> NamespaceName:
 
 
 def aggregate_resources(
-    resource_type: Literal["memory", "cpu"], containers: Collection[api.ContainerSpec]
+    resource_type: Literal["memory", "cpu"], pods: Sequence[api.Pod]
 ) -> section.Resources:
-    specified_requests = [
-        request
-        for c in containers
-        if (request := getattr(c.resources.requests, resource_type)) is not None
-    ]
-    specified_limits = [
-        limit
-        for c in containers
-        if (limit := getattr(c.resources.limits, resource_type)) is not None
-    ]
-
-    count_total = len(containers)
+    # requests
+    total_request = 0.0  # sum of all requests (pod-level or container-level)
+    count_unspecified_requests = 0  # containers without a request set
+    count_total_requests = 0  # containers from pods using container-level requests
+    count_pods_pod_level_request = 0  # pods with a non-zero pod-level request
+
+    # limits
+    total_limit = 0.0  # sum of all limits (pod-level or container-level)
+    count_unspecified_limits = 0  # containers without a limit set
+    count_zeroed_limits = 0  # containers with limit==0 (unlimited)
+    count_total_limits = 0  # containers from pods using container-level limits
+    count_pods_pod_level_limit = 0  # pods with a non-zero pod-level limit
+
+    for pod in pods:
+        pod_request = getattr(pod.spec.resources.requests, resource_type)
+        pod_limit = getattr(pod.spec.resources.limits, resource_type)
+        containers = pod.spec.containers
+
+        if pod_request is not None and pod_request:  # also handles: 0 -> unlimited
+            total_request += pod_request
+            count_pods_pod_level_request += 1
+        else:
+            specified_requests = [
+                request
+                for c in containers
+                if (request := getattr(c.resources.requests, resource_type)) is not None
+            ]
+            total_request += sum(specified_requests)
+            count_unspecified_requests += len(containers) - len(specified_requests)
+            count_total_requests += len(containers)
+
+        if pod_limit is not None and pod_limit:  # also handles: 0 -> unlimited
+            total_limit += pod_limit
+            count_pods_pod_level_limit += 1
+        else:
+            specified_limits = [
+                limit
+                for c in containers
+                if (limit := getattr(c.resources.limits, resource_type)) is not None
+            ]
+            total_limit += sum(specified_limits)
+            count_unspecified_limits += len(containers) - len(specified_limits)
+            count_zeroed_limits += sum(1 for x in specified_limits if x == 0)
+            count_total_limits += len(containers)
 
     return section.Resources(
-        request=sum(specified_requests),
-        limit=sum(specified_limits),
-        count_unspecified_requests=count_total - len(specified_requests),
-        count_unspecified_limits=count_total - len(specified_limits),
-        count_zeroed_limits=sum(1 for x in specified_limits if x == 0),
-        count_total=count_total,
+        request=total_request,
+        limit=total_limit,
+        # requests
+        count_unspecified_requests=count_unspecified_requests,
+        count_total_requests=count_total_requests,
+        count_pods_pod_level_request=count_pods_pod_level_request,
+        # limits
+        count_unspecified_limits=count_unspecified_limits,
+        count_zeroed_limits=count_zeroed_limits,
+        count_total_limits=count_total_limits,
+        count_pods_pod_level_limit=count_pods_pod_level_limit,
     )
 
 
@@ -161,11 +198,11 @@ def thin_containers(pods: Collection[api.Pod]) -> section.ThinContainers:
 
 
 def collect_memory_resources_from_api_pods(pods: Sequence[api.Pod]) -> section.Resources:
-    return aggregate_resources("memory", [c for pod in pods for c in pod.spec.containers])
+    return aggregate_resources("memory", pods)
 
 
 def collect_cpu_resources_from_api_pods(pods: Sequence[api.Pod]) -> section.Resources:
-    return aggregate_resources("cpu", [c for pod in pods for c in pod.spec.containers])
+    return aggregate_resources("cpu", pods)
 
 
 def pod_resources_from_api_pods(pods: Sequence[api.Pod]) -> section.PodResources:

packages/cmk-plugins/cmk/plugins/kube/checkman/kube_cpu

@@ -7,9 +7,11 @@ description:
  This check shows the CPU usage of a Kubernetes object, its requests utilization and its limits
  utilization.
 
- Requests utilization is the CPU usage divided by the sum of all configured container CPU
- requests. Limits utilization is computed in the same manner. The utilization percentage values are
- omitted if any of the associated request/limit value are not configured. Moreover, limits
+ Requests utilization is the CPU usage divided by the configured CPU requests of the Kubernetes
+ object. For pods without pod-level requests, the sum of container requests is used. For pods with
+ pod-level requests, the pod-level value takes precedence so long as the pod-level value is
+ non-zero. Limits utilization is computed in the same manner. The utilization percentage values
+ are omitted if any of the associated request/limit values are not configured. Moreover, limits
  utilization is omitted if any container has a limit of zero. The reason for omission is displayed
  in the check's details.
 

packages/cmk-plugins/cmk/plugins/kube/checkman/kube_memory

@@ -7,11 +7,13 @@ description:
  This check shows the memory usage of a Kubernetes object, its requests utilization and its limits
  utilization.
 
- Requests utilization is the memory usage divided by the sum of all configured container memory
- requests. Limit utilization is computed in the same manner. The utilization percentage values are
- omitted if any of the associated request/limit value are not configured. Moreover, limits
- utilization is omitted if any container has a limit of zero. The reason for omission is displayed
- in the check's details.
+ Requests utilization is the memory usage divided by the configured memory requests of the
+ Kubernetes object. For pods without pod-level requests, the sum of container requests is used.
+ For pods with pod-level requests, the pod-level value takes precedence so long as the pod-level
+ value is non-zero. Limits utilization is computed in the same manner. The utilization percentage
+ values are omitted if any of the  associated request/limit values are not configured. Moreover,
+ limits utilization is omitted if any container has a limit of zero. The reason for omission is
+ displayed in the check's details.
 
  The primary metric driving Kubernetes' memory-based restart decisions is
  container_memory_working_set_bytes. The plug-in reports this metric under the label "Usage".
@@ -33,7 +35,7 @@ description:
  The check requires that the special agent kube is configured.
 
  For the Checkmk Openshift monitoring, the reported metric values will differ from the values
- reported on the Openshift dashboard as as the included values describe a different quantity.
+ reported on the Openshift dashboard as the included values describe a different quantity.
 
 discovery:
  One service is created per Kubernetes object.

packages/cmk-plugins/cmk/plugins/kube/from_json/pod/pod_spec.py

@@ -6,6 +6,7 @@
 from typing import Literal, NotRequired, TypedDict
 
 from ...schemata import api
+from ..resources import JSONResourceRequirements
 from .container_spec import containers_spec, JSONContainerSpec
 from .volume import JSONPodVolume, parse_pod_volumes
 
@@ -20,6 +21,30 @@ class JSONPodSpec(TypedDict):
     priorityClassName: NotRequired[str]
     activeDeadlineSeconds: NotRequired[int]
     volumes: NotRequired[Sequence[JSONPodVolume]]
+    resources: NotRequired[JSONResourceRequirements]
+
+
+def pod_resources(pod_spec: JSONPodSpec) -> api.ResourceRequirements:
+    parsed_limits = api.ResourceRequirement()
+    parsed_requests = api.ResourceRequirement()
+    if resources := pod_spec.get("resources"):
+        if limits := resources.get("limits"):
+            parsed_limits = api.ResourceRequirement(
+                memory=api.parse_resource_value(limits["memory"]) if "memory" in limits else None,
+                cpu=api.parse_cpu_cores(limits["cpu"]) if "cpu" in limits else None,
+            )
+        if requests := resources.get("requests"):
+            parsed_requests = api.ResourceRequirement(
+                memory=api.parse_resource_value(requests["memory"])
+                if "memory" in requests
+                else None,
+                cpu=api.parse_cpu_cores(requests["cpu"]) if "cpu" in requests else None,
+            )
+
+    return api.ResourceRequirements(
+        limits=parsed_limits,
+        requests=parsed_requests,
+    )
 
 
 def pod_spec(spec: JSONPodSpec) -> api.PodSpec:
@@ -33,4 +58,5 @@ def pod_spec(spec: JSONPodSpec) -> api.PodSpec:
         priority_class_name=spec.get("priorityClassName"),
         active_deadline_seconds=spec.get("activeDeadlineSeconds"),
         volumes=parse_pod_volumes(volumes) if (volumes := spec.get("volumes")) else None,
+        resources=pod_resources(spec),
     )

packages/cmk-plugins/cmk/plugins/kube/kube_resources.py

@@ -37,25 +37,42 @@ def parse_performance_usage(string_table: StringTable) -> PerformanceUsage:
 
 
 def count_overview(resources: Resources, requirement: RequirementType) -> str:
-    ignored = (
-        resources.count_unspecified_requests
-        if requirement == "request"
-        else resources.count_unspecified_limits + resources.count_zeroed_limits
-    )
-    return (
-        f"{resources.count_total - ignored}/{resources.count_total} containers with {requirement}s"
-    )
+    if requirement == "request":
+        specified = resources.count_total_requests - resources.count_unspecified_requests
+        total = resources.count_total_requests
+        pod_level = resources.count_pods_pod_level_request
+    else:
+        specified = (
+            resources.count_total_limits
+            - resources.count_unspecified_limits
+            - resources.count_zeroed_limits
+        )
+        total = resources.count_total_limits
+        pod_level = resources.count_pods_pod_level_limit
+
+    container_part = f"{specified}/{total} containers with {requirement}s"
+    if pod_level == 0:
+        return container_part
+
+    pod_part = f"{pod_level} pod{'s' if pod_level > 1 else ''} with pod-level {requirement}s"
+    if total == 0:
+        return pod_part
+
+    return f"{container_part}, {pod_part}"
 
 
 def parse_resources(string_table: StringTable) -> Resources:
     """Parses limit and request values into Resources
     >>> parse_resources([['{"request": 209715200.0,'
     ... '"limit": 104857600.0,'
     ... '"count_unspecified_requests": 0,'
+    ... '"count_total_requests": 1,'
+    ... '"count_pods_pod_level_request": 0,'
     ... '"count_unspecified_limits": 0,'
     ... '"count_zeroed_limits": 1,'
-    ... '"count_total": 1}']])
-    Resources(request=209715200.0, limit=104857600.0, count_unspecified_requests=0, count_unspecified_limits=0, count_zeroed_limits=1, count_total=1)
+    ... '"count_total_limits": 1,'
+    ... '"count_pods_pod_level_limit": 0}']])
+    Resources(request=209715200.0, limit=104857600.0, count_unspecified_requests=0, count_total_requests=1, count_pods_pod_level_request=0, count_unspecified_limits=0, count_zeroed_limits=1, count_total_limits=1, count_pods_pod_level_limit=0)
     """
     return Resources.model_validate_json(string_table[0][0])
 

packages/cmk-plugins/cmk/plugins/kube/schemata/api.py

@@ -778,6 +778,7 @@ class PodSpec(BaseModel):
     priority_class_name: str | None = None
     active_deadline_seconds: int | None = None
     volumes: Sequence[Volume] | None = None
+    resources: ResourceRequirements
 
 
 @enum.unique

packages/cmk-plugins/cmk/plugins/kube/schemata/section.py

@@ -200,10 +200,17 @@ class Resources(Section):
 
     request: float
     limit: float
+
+    # requests
     count_unspecified_requests: int
+    count_total_requests: int
+    count_pods_pod_level_request: int
+
+    # limits
     count_unspecified_limits: int
     count_zeroed_limits: int
-    count_total: int
+    count_total_limits: int
+    count_pods_pod_level_limit: int
 
 
 class AllocatableResource(Section):

packages/cmk-plugins/tests/cmk/plugins/kube/agent_based/test_kube_cpu.py

@@ -46,10 +46,13 @@ class ResourcesFactory(ModelFactory):
 RESOURCES_SECTION = Resources(
     request=0.18,
     limit=0.36,
-    count_total=2,
     count_zeroed_limits=0,
     count_unspecified_limits=0,
     count_unspecified_requests=0,
+    count_total_requests=2,
+    count_total_limits=2,
+    count_pods_pod_level_request=0,
+    count_pods_pod_level_limit=0,
 )