Merge branch 'aws-cwa-dev' into aws-cwa-dev

Author: petruanica

internal/aws/containerinsight/const.go

@@ -146,6 +146,13 @@ const (
 	GpuRequest          = "gpu_request"
 	GpuReservedCapacity = "gpu_reserved_capacity"
 
+	NeuroncoreLimit              = "neuroncore_limit"
+	NeuroncoreUsageTotal         = "neuroncore_usage_total"
+	NeuroncoreRequest            = "neuroncore_request"
+	NeuroncoreReservedCapacity   = "neuroncore_reserved_capacity"
+	NeuroncoreUnreservedCapacity = "neuroncore_unreserved_capacity"
+	NeuroncoreAvailableCapacity  = "neuroncore_available_capacity"
+
 	HyperPodUnschedulablePendingReplacement = "unschedulable_pending_replacement"
 	HyperPodUnschedulablePendingReboot      = "unschedulable_pending_reboot"
 	HyperPodSchedulable                     = "schedulable"
@@ -367,6 +374,13 @@ func init() {
 		GpuRequest:          UnitCount,
 		GpuReservedCapacity: UnitPercent,
 
+		NeuroncoreLimit:              UnitCount,
+		NeuroncoreUsageTotal:         UnitCount,
+		NeuroncoreRequest:            UnitCount,
+		NeuroncoreReservedCapacity:   UnitPercent,
+		NeuroncoreUnreservedCapacity: UnitPercent,
+		NeuroncoreAvailableCapacity:  UnitCount,
+
 		HyperPodUnschedulablePendingReplacement: UnitCount,
 		HyperPodUnschedulablePendingReboot:      UnitCount,
 		HyperPodSchedulable:                     UnitCount,

receiver/awscontainerinsightreceiver/internal/stores/nodeinfo.go

@@ -15,12 +15,14 @@ import (
 )
 
 type nodeStats struct {
-	podCnt        int
-	containerCnt  int
-	cpuReq        uint64
-	memReq        uint64
-	gpuReq        uint64
-	gpuUsageTotal uint64
+	podCnt               int
+	containerCnt         int
+	cpuReq               uint64
+	memReq               uint64
+	gpuReq               uint64
+	gpuUsageTotal        uint64
+	neuroncoreReq        uint64
+	neuroncoreUsageTotal uint64
 }
 
 type nodeInfo struct {
@@ -118,6 +120,58 @@ func (n *nodeInfo) getNodeStatusCapacityGPUs() (uint64, bool) {
 	return forceConvertToInt64(gpus, n.logger), true
 }
 
+func (n *nodeInfo) getNeuronResourceCapacity(resourceKey v1.ResourceName) (uint64, bool) {
+	capacityResources, ok := n.provider.NodeToCapacityMap()[n.nodeName]
+	if !ok {
+		return 0, false
+	}
+	value := capacityResources.Name(resourceKey, resource.DecimalExponent).Value()
+	return forceConvertToInt64(value, n.logger), true
+}
+
+func (n *nodeInfo) getNodeStatusCapacityNeuron() (uint64, bool) {
+	return n.getNeuronResourceCapacity(resourceSpecNeuronKey)
+}
+
+func (n *nodeInfo) getNodeStatusCapacityNeuroncore() (uint64, bool) {
+	return n.getNeuronResourceCapacity(resourceSpecNeuroncoreKey)
+}
+
+func (n *nodeInfo) getNodeStatusCapacityNeurondevice() (uint64, bool) {
+	return n.getNeuronResourceCapacity(resourceSpecNeuronDeviceKey)
+}
+
+func (n *nodeInfo) getNeuronCoresPerDevice() (int, bool) {
+	devices, hasDevices := n.getNodeStatusCapacityNeuron()
+	cores, hasCores := n.getNodeStatusCapacityNeuroncore()
+
+	if hasDevices && hasCores && devices > 0 && cores > 0 {
+		return int(cores / devices), true
+	}
+
+	return 0, false
+}
+
+func (n *nodeInfo) getNodeStatusCapacityNeuronCores() (uint64, bool) {
+	if cores, ok := n.getNodeStatusCapacityNeuroncore(); ok {
+		return cores, true
+	}
+
+	coresPerDevice, hasRatio := n.getNeuronCoresPerDevice()
+	if !hasRatio {
+		return 0, false
+	}
+
+	if devices, ok := n.getNodeStatusCapacityNeuron(); ok {
+		return devices * uint64(coresPerDevice), true
+	}
+	if devices, ok := n.getNodeStatusCapacityNeurondevice(); ok {
+		return devices * uint64(coresPerDevice), true
+	}
+
+	return 0, false
+}
+
 func (n *nodeInfo) getNodeStatusCondition(conditionType v1.NodeConditionType) (uint64, bool) {
 	if nodeConditions, ok := n.provider.NodeToConditionsMap()[n.nodeName]; ok {
 		if conditionStatus, ok := nodeConditions[conditionType]; ok {

receiver/awscontainerinsightreceiver/internal/stores/podstore.go

@@ -29,6 +29,9 @@ const (
 	memoryKey          = "memory"
 	cpuKey             = "cpu"
 	gpuKey             = "nvidia.com/gpu"
+	neuronKey          = "aws.amazon.com/neuron"
+	neuroncoreKey      = "aws.amazon.com/neuroncore"
+	neuronDeviceKey    = "aws.amazon.com/neurondevice"
 	splitRegexStr      = "\\.|-"
 	kubeProxy          = "kube-proxy"
 )
@@ -247,6 +250,7 @@ func (p *PodStore) Decorate(ctx context.Context, metric CIMetric, kubernetesBlob
 		p.decorateCPU(metric, &entry.pod)
 		p.decorateMem(metric, &entry.pod)
 		p.decorateGPU(metric, &entry.pod)
+		p.decorateNeuron(metric, &entry.pod)
 		p.addStatus(metric, &entry.pod)
 		addContainerCount(metric, &entry.pod)
 		addContainerID(&entry.pod, metric, kubernetesBlob, p.logger)
@@ -298,6 +302,8 @@ func (p *PodStore) refreshInternal(now time.Time, podList []corev1.Pod) {
 	var memRequest uint64
 	var gpuRequest uint64
 	var gpuUsageTotal uint64
+	var neuroncoreRequest uint64
+	var neuroncoreUsageTotal uint64
 
 	for i := range podList {
 		pod := podList[i]
@@ -312,6 +318,12 @@ func (p *PodStore) refreshInternal(now time.Time, podList []corev1.Pod) {
 			cpuRequest += tmpCPUReq
 			tmpMemReq, _ := getResourceSettingForPod(&pod, p.nodeInfo.getMemCapacity(), memoryKey, getRequestForContainer)
 			memRequest += tmpMemReq
+			if coresLimit, coresReq, hasNeuron := p.getNeuronCoresFromPod(&pod); hasNeuron {
+				neuroncoreRequest += coresReq
+				if pod.Status.Phase == corev1.PodRunning {
+					neuroncoreUsageTotal += coresLimit
+				}
+			}
 			if tmpGpuLimit, ok := getResourceSettingForPod(&pod, 0, gpuKey, getLimitForContainer); ok {
 				tmpGpuReq, _ := getResourceSettingForPod(&pod, 0, gpuKey, getRequestForContainer)
 				gpuRequest += tmpGpuReq
@@ -337,12 +349,14 @@ func (p *PodStore) refreshInternal(now time.Time, podList []corev1.Pod) {
 	}
 
 	p.nodeInfo.setNodeStats(nodeStats{
-		podCnt:        podCount,
-		containerCnt:  containerCount,
-		memReq:        memRequest,
-		cpuReq:        cpuRequest,
-		gpuReq:        gpuRequest,
-		gpuUsageTotal: gpuUsageTotal,
+		podCnt:               podCount,
+		containerCnt:         containerCount,
+		memReq:               memRequest,
+		cpuReq:               cpuRequest,
+		gpuReq:               gpuRequest,
+		gpuUsageTotal:        gpuUsageTotal,
+		neuroncoreReq:        neuroncoreRequest,
+		neuroncoreUsageTotal: neuroncoreUsageTotal,
 	})
 }
 
@@ -408,6 +422,17 @@ func (p *PodStore) decorateNode(metric CIMetric) {
 				metric.AddField(ci.MetricName(ci.TypeNode, ci.GpuUsageTotal), nodeStats.gpuUsageTotal)
 				metric.AddField(ci.MetricName(ci.TypeNode, ci.GpuReservedCapacity), float64(nodeStats.gpuReq)/float64(nodeStatusCapacityGPUs)*100)
 			}
+
+			if nodeStatusCapacityNeuroncore, ok := p.nodeInfo.getNodeStatusCapacityNeuronCores(); ok && nodeStatusCapacityNeuroncore != 0 {
+				metric.AddField(ci.MetricName(ci.TypeNode, ci.NeuroncoreRequest), nodeStats.neuroncoreReq)
+				metric.AddField(ci.MetricName(ci.TypeNode, ci.NeuroncoreLimit), nodeStatusCapacityNeuroncore)
+				metric.AddField(ci.MetricName(ci.TypeNode, ci.NeuroncoreUsageTotal), nodeStats.neuroncoreUsageTotal)
+
+				reservedCapacity := float64(nodeStats.neuroncoreReq) / float64(nodeStatusCapacityNeuroncore) * 100
+				metric.AddField(ci.MetricName(ci.TypeNode, ci.NeuroncoreReservedCapacity), reservedCapacity)
+				metric.AddField(ci.MetricName(ci.TypeNode, ci.NeuroncoreUnreservedCapacity), 100.0-reservedCapacity)
+				metric.AddField(ci.MetricName(ci.TypeNode, ci.NeuroncoreAvailableCapacity), nodeStatusCapacityNeuroncore-nodeStats.neuroncoreReq)
+			}
 		}
 	}
 }
@@ -431,6 +456,56 @@ func (p *PodStore) decorateGPU(metric CIMetric, pod *corev1.Pod) {
 	}
 }
 
+func (p *PodStore) getCoresPerNeuronDevice() (int, bool) {
+	return p.nodeInfo.getNeuronCoresPerDevice()
+}
+
+func (p *PodStore) getNeuronCoresFromPod(pod *corev1.Pod) (limitCores, requestCores uint64, hasNeuron bool) {
+	if coreLimit, ok := getResourceSettingForPod(pod, 0, neuroncoreKey, getLimitForContainer); ok {
+		coreRequest, _ := getResourceSettingForPod(pod, 0, neuroncoreKey, getRequestForContainer)
+		return coreLimit, coreRequest, true
+	}
+
+	coresPerDevice, hasRatio := p.getCoresPerNeuronDevice()
+	if !hasRatio {
+		p.logger.Debug("Cannot determine neuron cores per device ratio, skipping device-based neuron metrics")
+		return 0, 0, false
+	}
+
+	deviceKeys := []corev1.ResourceName{neuronKey, neuronDeviceKey}
+	for _, deviceKey := range deviceKeys {
+		if deviceLimit, ok := getResourceSettingForPod(pod, 0, deviceKey, getLimitForContainer); ok {
+			deviceRequest, _ := getResourceSettingForPod(pod, 0, deviceKey, getRequestForContainer)
+			return deviceLimit * uint64(coresPerDevice), deviceRequest * uint64(coresPerDevice), true
+		}
+	}
+
+	return 0, 0, false
+}
+
+func (p *PodStore) decorateNeuron(metric CIMetric, pod *corev1.Pod) {
+	if p.includeEnhancedMetrics && p.enableAcceleratedComputeMetrics && metric.GetTag(ci.MetricType) == ci.TypePod &&
+		pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed {
+		coresLimit, coresRequest, hasNeuron := p.getNeuronCoresFromPod(pod)
+
+		if hasNeuron {
+			metric.AddField(ci.MetricName(ci.TypePod, ci.NeuroncoreRequest), coresRequest)
+			metric.AddField(ci.MetricName(ci.TypePod, ci.NeuroncoreLimit), coresLimit)
+
+			var podNeuroncoreUsageTotal uint64
+			if pod.Status.Phase == corev1.PodRunning {
+				podNeuroncoreUsageTotal = coresLimit
+			}
+			metric.AddField(ci.MetricName(ci.TypePod, ci.NeuroncoreUsageTotal), podNeuroncoreUsageTotal)
+
+			if nodeCapacityCores, ok := p.nodeInfo.getNodeStatusCapacityNeuronCores(); ok && nodeCapacityCores != 0 {
+				reservedCapacity := float64(coresLimit) / float64(nodeCapacityCores) * 100
+				metric.AddField(ci.MetricName(ci.TypePod, ci.NeuroncoreReservedCapacity), reservedCapacity)
+			}
+		}
+	}
+}
+
 func (p *PodStore) decorateCPU(metric CIMetric, pod *corev1.Pod) {
 	if metric.GetTag(ci.MetricType) == ci.TypePod {
 		// add cpu limit and request for pod cpu