Skip to content

Add UltraServer field to Neuron integ tests #477

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sky333999 merged 13 commits into from
Sep 11, 2025
Merged

Add UltraServer field to Neuron integ tests #477

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
67e1720
Add UltraServer field to Neuron integ tests
petruanica Mar 13, 2025
503a33e
Merge branch 'main' into ultraserver-integ-test
petruanica Aug 18, 2025
e14de3e
Add ultraserver_id to json output
petruanica Aug 18, 2025
5e0fe9e
Update expected metrics and dimensions
petruanica Aug 19, 2025
0eab296
Update dimensions format
petruanica Aug 19, 2025
ff4279c
Reoder dimensions
petruanica Aug 19, 2025
73ce503
Fix metric list
petruanica Aug 19, 2025
aefa482
Merge branch 'main' into ultraserver-integ-test
petruanica Aug 19, 2025
4218ac4
Add logs
petruanica Aug 19, 2025
7a01983
change timeout
petruanica Aug 19, 2025
914dc5c
Add more logs
petruanica Aug 19, 2025
608eec6
Update expected metrics
petruanica Aug 20, 2025
9a8b38e
Merge branch 'main' into ultraserver-integ-test
petruanica Aug 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/resources/dummy-neuron-monitor/dummy_neuron_monitor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ def get_instance_labels(instance_info):
'region': instance_info['instance_region'],
'subnet_id': instance_info['subnet_id']
}
if 'ultraserver_id' in instance_info:
Comment thread
sky333999 marked this conversation as resolved.
instance_labels['ultraserver_id'] = instance_info['ultraserver_id']
return instance_labels


Expand Down
1 change: 1 addition & 0 deletions docs/resources/dummy-neuron-monitor/neuron-monitor-output.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28155,6 +28155,7 @@
"instance_region": "us-east-1",
"ami_id": "ami-030686a4e905e98d3",
"subnet_id": "subnet-06a7754948e8a000f",
"ultraserver_id": "u-cafb07773a569a23",
"error": ""
},
"neuron_hardware_info": {
Expand Down
27 changes: 24 additions & 3 deletions test/awsneuron/neuron_metrics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,30 @@ const (

var expectedDimsToMetrics = map[string][]string{
"ClusterName": {
NodeNeuronCoreUtil, NodeNeuronCoreMemUsageConstants, NodeNeuronCoreMemUsageModel, NodeNeuronCoreMemUsageScratchpad,
NodeNeuronCoreMemUsageRuntime, NodeNeuronCoreMemUsageTensors, NodeNeuronCoreMemUsageTotal, NodeNeuronDeviceHwEccEvents,
NodeExecutionErrorsTotal, NodeNeuronDeviceRuntimeMemoryUsed, NodeNeuronExecutionLatency,
NodeNeuronCoreUtil, NodeNeuronCoreMemUsageTotal, NodeNeuronCoreMemUsageConstants,
NodeNeuronCoreMemUsageModel, NodeNeuronCoreMemUsageScratchpad, NodeNeuronCoreMemUsageRuntime,
NodeNeuronCoreMemUsageTensors, NodeExecutionErrorsTotal, NodeNeuronDeviceRuntimeMemoryUsed,
NodeNeuronExecutionLatency, NodeNeuronDeviceHwEccEvents,
},
"ClusterName-UltraServer": {
NodeNeuronCoreUtil, NodeNeuronCoreMemUsageTotal, NodeNeuronCoreMemUsageConstants,
NodeNeuronCoreMemUsageModel, NodeNeuronCoreMemUsageScratchpad, NodeNeuronCoreMemUsageRuntime,
NodeNeuronCoreMemUsageTensors, NodeExecutionErrorsTotal, NodeNeuronDeviceRuntimeMemoryUsed,
NodeNeuronExecutionLatency, NodeNeuronDeviceHwEccEvents,
},
"ClusterName-InstanceId-NodeName": {
NodeNeuronCoreUtil, NodeNeuronCoreMemUsageTotal, NodeNeuronCoreMemUsageConstants,
NodeNeuronCoreMemUsageModel, NodeNeuronCoreMemUsageScratchpad, NodeNeuronCoreMemUsageRuntime,
NodeNeuronCoreMemUsageTensors, NodeExecutionErrorsTotal, NodeNeuronDeviceRuntimeMemoryUsed,
NodeNeuronExecutionLatency, NodeNeuronDeviceHwEccEvents,
},
"ClusterName-InstanceId-InstanceType-NeuronCore-NeuronDevice-NodeName": {
NodeNeuronCoreUtil, NodeNeuronCoreMemUsageTotal, NodeNeuronCoreMemUsageConstants,
NodeNeuronCoreMemUsageModel, NodeNeuronCoreMemUsageScratchpad, NodeNeuronCoreMemUsageRuntime,
NodeNeuronCoreMemUsageTensors,
},
"ClusterName-InstanceId-NeuronDevice-NodeName": {
NodeNeuronDeviceHwEccEvents,
},
}

Expand Down
15 changes: 15 additions & 0 deletions test/metric/container_insights_util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,22 @@ type dimToMetrics struct {

func ValidateMetrics(env *environment.MetaData, metricFilter string, expectedDimsToMetrics map[string][]string) []status.TestResult {
var results []status.TestResult
log.Printf("Expected %d dimension groups", len(expectedDimsToMetrics))
for dims, metrics := range expectedDimsToMetrics {
log.Printf("Expected dimension group: %s with %d metrics", dims, len(metrics))
for _, metricName := range metrics {
log.Printf(" Expected metric: %s", metricName)
}
}

dimsToMetrics := getMetricsInClusterDimension(env, metricFilter)
log.Printf("Found %d dimension groups", len(dimsToMetrics))
for _, dtm := range dimsToMetrics {
log.Printf("Dimension group: %s with %d metrics", dtm.dimStr, len(dtm.metrics))
for metricName, dimSets := range dtm.metrics {
log.Printf(" Metric: %s has %d dimension sets", metricName, len(dimSets))
}
}
for dims, metrics := range expectedDimsToMetrics {
var actual map[string][][]types.Dimension
// find matching dim set from fetched and processed metric-dims groups
Expand Down
2 changes: 2 additions & 0 deletions test/metric_value_benchmark/eks_resources/test_schemas/container_neuroncore.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"Service": {},
"Timestamp": {},
"Type": {},
"UltraServer": {},
"Version": {},
"availability_zone": {},
"kubernetes": {},
Expand All @@ -44,6 +45,7 @@
"Service",
"Timestamp",
"Type",
"UltraServer",
"Version",
"CloudWatchMetrics"
]
Expand Down
2 changes: 2 additions & 0 deletions test/metric_value_benchmark/eks_resources/test_schemas/node_neuron.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
"NodeName": {},
"Timestamp": {},
"Type": {},
"UltraServer": {},
"Version": {},
"availability_zone": {},
"kubernetes": {},
Expand All @@ -27,6 +28,7 @@
"NodeName",
"Timestamp",
"Type",
"UltraServer",
"Version",
"CloudWatchMetrics"
]
Expand Down
2 changes: 2 additions & 0 deletions test/metric_value_benchmark/eks_resources/test_schemas/node_neuroncore.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"NodeName": {},
"Timestamp": {},
"Type": {},
"UltraServer": {},
"Version": {},
"availability_zone": {},
"kubernetes": {},
Expand All @@ -36,6 +37,7 @@
"NodeName",
"Timestamp",
"Type",
"UltraServer",
"Version",
"CloudWatchMetrics"
]
Expand Down
2 changes: 2 additions & 0 deletions test/metric_value_benchmark/eks_resources/test_schemas/node_neurondevice.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
"NodeName": {},
"Timestamp": {},
"Type": {},
"UltraServer": {},
"Version": {},
"availability_zone": {},
"kubernetes": {},
Expand All @@ -28,6 +29,7 @@
"NodeName",
"Timestamp",
"Type",
"UltraServer",
"Version",
"CloudWatchMetrics"
]
Expand Down
2 changes: 2 additions & 0 deletions test/metric_value_benchmark/eks_resources/test_schemas/pod_neuroncore.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
"Service": {},
"Timestamp": {},
"Type": {},
"UltraServer": {},
"Version": {},
"availability_zone": {},
"kubernetes": {},
Expand All @@ -43,6 +44,7 @@
"Service",
"Timestamp",
"Type",
"UltraServer",
"Version",
"CloudWatchMetrics"
]
Expand Down
Loading