improve pynvml add gpm for sm util

Author: ajcasagrande

src/aiperf/common/models/telemetry_models.py

@@ -28,7 +28,9 @@ class TelemetryMetrics(AIPerfBaseModel):
         default=None, description="Cumulative energy consumption in MJ"
     )
     gpu_utilization: float | None = Field(
-        default=None, description="GPU utilization percentage (0-100)"
+        default=None,
+        description="GPU utilization percentage (0-100). "
+        "Percent of time over the past sample period during which one or more kernels was executing on the GPU.",
     )
     gpu_memory_used: float | None = Field(
         default=None, description="GPU memory used in GB"
@@ -37,7 +39,9 @@ class TelemetryMetrics(AIPerfBaseModel):
         default=None, description="GPU temperature in °C"
     )
     mem_utilization: float | None = Field(
-        default=None, description="Memory bandwidth utilization percentage (0-100)"
+        default=None,
+        description="Memory bandwidth utilization percentage (0-100). "
+        "Percent of time over the past sample period during which global (device) memory was being read or written.",
     )
     sm_utilization: float | None = Field(
         default=None,

src/aiperf/gpu_telemetry/__init__.py

@@ -17,10 +17,8 @@
     GPUTelemetryAccumulator,
 )
 from aiperf.gpu_telemetry.constants import (
-    DCGM_SCALING_FACTORS,
     DCGM_TO_FIELD_MAPPING,
     GPU_TELEMETRY_METRICS_CONFIG,
-    PYNVML_SCALING_FACTORS,
     PYNVML_SOURCE_IDENTIFIER,
     get_gpu_telemetry_metrics_config,
 )
@@ -48,7 +46,6 @@
 
 __all__ = [
     "DCGMTelemetryCollector",
-    "DCGM_SCALING_FACTORS",
     "DCGM_TO_FIELD_MAPPING",
     "GPUTelemetryAccumulator",
     "GPUTelemetryCollectorFactory",
@@ -57,7 +54,6 @@
     "GPUTelemetryManager",
     "GPU_TELEMETRY_METRICS_CONFIG",
     "MetricsConfigLoader",
-    "PYNVML_SCALING_FACTORS",
     "PYNVML_SOURCE_IDENTIFIER",
     "PyNVMLTelemetryCollector",
     "TErrorCallback",

src/aiperf/gpu_telemetry/constants.py

@@ -16,22 +16,6 @@
 # Source identifier for pynvml collector (used in TelemetryRecord.dcgm_url field)
 PYNVML_SOURCE_IDENTIFIER = "pynvml://localhost"
 
-# DCGM unit conversion scaling factors
-DCGM_SCALING_FACTORS = {
-    "energy_consumption": 1e-9,  # mJ to MJ
-    "gpu_memory_used": 1.048576 * 1e-3,  # MiB to GB
-    "sm_utilization": 100,  # ratio (0-1) to percentage (0-100)
-    "power_violation": 1e-3,  # ns to µs
-}
-
-# PyNVML unit conversion scaling factors
-# NVML returns values in different units than what TelemetryMetrics expects
-PYNVML_SCALING_FACTORS = {
-    "gpu_power_usage": 1e-3,  # mW -> W
-    "energy_consumption": 1e-9,  # mJ -> MJ
-    "gpu_memory_used": 1e-9,  # bytes -> GB
-}
-
 # DCGM field mapping to telemetry record fields
 DCGM_TO_FIELD_MAPPING = {
     "DCGM_FI_DEV_POWER_USAGE": "gpu_power_usage",

src/aiperf/gpu_telemetry/dcgm_collector.py

@@ -14,17 +14,22 @@
     TRecordCallback,
 )
 from aiperf.common.models import GpuMetadata, TelemetryMetrics, TelemetryRecord
-from aiperf.gpu_telemetry.constants import (
-    DCGM_SCALING_FACTORS,
-    DCGM_TO_FIELD_MAPPING,
-)
+from aiperf.gpu_telemetry.constants import DCGM_TO_FIELD_MAPPING
 from aiperf.gpu_telemetry.factories import (
     GPUTelemetryCollectorFactory,
     GPUTelemetryCollectorProtocol,
 )
 
 __all__ = ["DCGMTelemetryCollector"]
 
+# Unit conversion scaling factors for DCGM metrics
+SCALING_FACTORS = {
+    "energy_consumption": 1e-9,  # mJ -> MJ
+    "gpu_memory_used": 1.048576e-3,  # MiB -> GB
+    "sm_utilization": 100,  # ratio (0-1) -> percentage (0-100)
+    "power_violation": 1e-3,  # ns -> µs
+}
+
 
 @implements_protocol(GPUTelemetryCollectorProtocol)
 @GPUTelemetryCollectorFactory.register(GPUTelemetryCollectorType.DCGM)
@@ -65,7 +70,7 @@ def __init__(
         error_callback: TErrorCallback | None = None,
         collector_id: str = "telemetry_collector",
     ) -> None:
-        self._scaling_factors = DCGM_SCALING_FACTORS
+        self._scaling_factors = SCALING_FACTORS
         super().__init__(
             endpoint_url=dcgm_url,
             collection_interval=collection_interval,