Add CPU breakdown min/max aggregations and cpu.nice metric for allotment ODS (#554)

Summary:

Extends the allotment-level CPU metrics to support MySQL and ZippyDB stacking migration. This is part of the broader effort to expose allotment-level ODS metrics as these workloads move from whole-host shapes to stacked allotments.

Changes:
- Add cpu.nice breakdown metric from /proc/stat per-CPU nice field (not available in cgroup v2 cpu.stat) via CPUTimeMonitor, with avg/min/max aggregations
- Add min.60/max.60 aggregations for cpu.iowait and cpu.saturation-pct from CPUTimeMonitor /proc/stat data
- Add min.60/max.60 aggregations for cpu.user and cpu.sys from the cgroup ring buffer (AllotmentsMonitor::getCgroupCounterMinMax), keeping the existing cgroup data source for consistency with the avg metric
- Add kCpuSaturationMin/Max key specs to AllotmentCpuOdsKeySpecs
- Extract granularityToLevel() helper to deduplicate 6 identical switch blocks in CPUTimeMonitor
- Fix readCgroupStat_ fallback in getMinCPUCoresUsage/getMaxCPUCoresUsage to match existing getStat() behavior

All new ODS keys are gated behind TargetedOdsFilter::shouldPublish() with defaultPublish=false, so they only emit when an explicit allow rule is configured for the workload.

New ODS keys (all require TargetedOdsFilter config to publish):
- dyno.allotment.cpu.nice (avg via ODS suffix)
- dyno.allotment.cpu.{nice,iowait,idle,softirq,hardirq}.min.60 / .max.60
- dyno.allotment.cpu.user.min.60 / .max.60
- dyno.allotment.cpu.sys.min.60 / .max.60
- dyno.allotment.cpu.saturation-pct.min.60 / .max.60

Design doc: https://docs.google.com/document/d/16ea49QzoL60PTtFQ0DXss4Bmq7aCnSGh4MuG9phMmO0/edit

Differential Revision: D104745222

Author: cmgrace

dynolog/src/CPUTimeMonitor.cpp

@@ -8,6 +8,7 @@
 #include "dynolog/src/CPUTimeMonitor.h"
 #include <dynolog/src/metric_frame/MetricFrameTsUnit.h>
 
+#include <algorithm>
 #include <utility>
 
 enum { IDX_MIN = 0, IDX_SEC = 1, IDX_HUNDRED_MS = 2 };
@@ -19,14 +20,16 @@ static const std::string kBreakdownIdle = ".idle";
 static const std::string kBreakdownSoftirq = ".softirq";
 static const std::string kBreakdownIowait = ".iowait";
 static const std::string kBreakdownHardirq = ".hardirq";
+static const std::string kBreakdownNice = ".nice";
 
 // All breakdown suffixes in a single list to avoid iterating in multiple
 // places.
 static const std::vector<std::string> kBreakdownSuffixes = {
     kBreakdownIdle,
     kBreakdownSoftirq,
     kBreakdownIowait,
-    kBreakdownHardirq};
+    kBreakdownHardirq,
+    kBreakdownNice};
 
 // Helper to add breakdown series for a target to a MetricFrameMap.
 static void addBreakdownSeries(
@@ -51,13 +54,27 @@ static const std::string& breakdownSuffix(
       return kBreakdownIowait;
     case CPUTimeMonitor::CpuBreakdown::HARDIRQ:
       return kBreakdownHardirq;
+    case CPUTimeMonitor::CpuBreakdown::NICE:
+      return kBreakdownNice;
   }
   return kBreakdownIdle; // unreachable
 }
 
 constexpr int kRingbufferSizeMinutes = 6;
 static const std::string kHostCgroupPath = "/sys/fs/cgroup";
 
+static std::optional<int> granularityToLevel(CPUTimeMonitor::Granularity gran) {
+  switch (gran) {
+    case CPUTimeMonitor::Granularity::MINUTE:
+      return IDX_MIN;
+    case CPUTimeMonitor::Granularity::SECOND:
+      return IDX_SEC;
+    case CPUTimeMonitor::Granularity::HUNDRED_MS:
+      return IDX_HUNDRED_MS;
+  }
+  return std::nullopt;
+}
+
 std::array<MetricFrameMap, 3> CPUTimeMonitor::createMetricFrameArray() {
   return {
       MetricFrameMap(
@@ -194,20 +211,11 @@ std::optional<double> CPUTimeMonitor::getCpuBreakdownAvg(
   TimePoint now = std::chrono::steady_clock::now();
   std::shared_lock lock(dataLock_);
 
-  int level = 0;
-  switch (gran) {
-    case Granularity::MINUTE:
-      level = IDX_MIN;
-      break;
-    case Granularity::SECOND:
-      level = IDX_SEC;
-      break;
-    case Granularity::HUNDRED_MS:
-      level = IDX_HUNDRED_MS;
-      break;
-    default:
-      return std::nullopt;
+  auto levelOpt = granularityToLevel(gran);
+  if (!levelOpt.has_value()) {
+    return std::nullopt;
   }
+  int level = *levelOpt;
 
   auto& frame = procUsageMetricFrames_[level];
   auto slice = frame.slice(now - std::chrono::seconds(seconds_ago), now);
@@ -248,6 +256,65 @@ const MetricFrameMap* CPUTimeMonitor::getMetricFrame(
       : &procUsageMetricFrames_[level];
 }
 
+// Min/max for CPU breakdown metrics. Reads per-second samples from the
+// /proc/stat ring buffer and returns the min (or max) value over the window.
+// MetricSeries doesn't have built-in min/max, so we use raw() + std algorithms.
+std::optional<double> CPUTimeMonitor::getCpuBreakdownMin(
+    Granularity gran,
+    uint64_t seconds_ago,
+    CpuBreakdown breakdown,
+    const std::optional<std::string>& targetId) {
+  TimePoint now = std::chrono::steady_clock::now();
+  std::shared_lock lock(dataLock_);
+
+  auto levelOpt = granularityToLevel(gran);
+  if (!levelOpt.has_value()) {
+    return std::nullopt;
+  }
+  int level = *levelOpt;
+
+  auto& frame = procUsageMetricFrames_[level];
+  auto slice = frame.slice(now - std::chrono::seconds(seconds_ago), now);
+  if (slice == std::nullopt) {
+    return std::nullopt;
+  }
+  std::string key = targetId.value_or("host") + breakdownSuffix(breakdown);
+  auto series = slice->series<double>(key);
+  if (series == std::nullopt || series->size() == 0) {
+    return std::nullopt;
+  }
+  auto data = series->raw();
+  return *std::min_element(data.begin(), data.end());
+}
+
+std::optional<double> CPUTimeMonitor::getCpuBreakdownMax(
+    Granularity gran,
+    uint64_t seconds_ago,
+    CpuBreakdown breakdown,
+    const std::optional<std::string>& targetId) {
+  TimePoint now = std::chrono::steady_clock::now();
+  std::shared_lock lock(dataLock_);
+
+  auto levelOpt = granularityToLevel(gran);
+  if (!levelOpt.has_value()) {
+    return std::nullopt;
+  }
+  int level = *levelOpt;
+
+  auto& frame = procUsageMetricFrames_[level];
+  auto slice = frame.slice(now - std::chrono::seconds(seconds_ago), now);
+  if (slice == std::nullopt) {
+    return std::nullopt;
+  }
+  std::string key = targetId.value_or("host") + breakdownSuffix(breakdown);
+  auto series = slice->series<double>(key);
+  if (series == std::nullopt || series->size() == 0) {
+    return std::nullopt;
+  }
+  auto data = series->raw();
+  return *std::max_element(data.begin(), data.end());
+}
+
 void CPUTimeMonitor::tick(TMask mask) {
   TimePoint tickTime = std::chrono::steady_clock::now();
   bool readPerCore = false;
@@ -589,8 +656,11 @@ void CPUTimeMonitor::processProcUsage(
     auto softirqDeltaOpt = safeDelta(newCt.y, lastCt.y);
     auto iowaitDeltaOpt = safeDelta(newCt.w, lastCt.w);
     auto hardirqDeltaOpt = safeDelta(newCt.x, lastCt.x);
+    // nice (CpuTime.n) from /proc/stat. Not available in cgroup v2 cpu.stat.
+    auto niceDeltaOpt = safeDelta(newCt.n, lastCt.n);
 
-    if (idleDeltaOpt && softirqDeltaOpt && iowaitDeltaOpt && hardirqDeltaOpt) {
+    if (idleDeltaOpt && softirqDeltaOpt && iowaitDeltaOpt && hardirqDeltaOpt &&
+        niceDeltaOpt) {
       double idleCores =
           static_cast<double>(*idleDeltaOpt) * msPerJiffy / wallDelta;
       double softirqCores =
@@ -599,11 +669,14 @@ void CPUTimeMonitor::processProcUsage(
           static_cast<double>(*iowaitDeltaOpt) * msPerJiffy / wallDelta;
       double hardirqCores =
           static_cast<double>(*hardirqDeltaOpt) * msPerJiffy / wallDelta;
+      double niceCores =
+          static_cast<double>(*niceDeltaOpt) * msPerJiffy / wallDelta;
 
       line.emplace_back(targetId + kBreakdownIdle, idleCores);
       line.emplace_back(targetId + kBreakdownSoftirq, softirqCores);
       line.emplace_back(targetId + kBreakdownIowait, iowaitCores);
       line.emplace_back(targetId + kBreakdownHardirq, hardirqCores);
+      line.emplace_back(targetId + kBreakdownNice, niceCores);
     }
   }
   if (!line.empty()) {

dynolog/src/CPUTimeMonitor.h

@@ -60,7 +60,7 @@ class CPUTimeMonitor : MonitorBase<Ticker<60000, 1000, 10, 3>> {
 
   enum class Granularity { MINUTE, SECOND, HUNDRED_MS };
   enum class DataSource { PROC_STAT, CGROUP_STAT };
-  enum class CpuBreakdown { IDLE, SOFTIRQ, IOWAIT, HARDIRQ };
+  enum class CpuBreakdown { IDLE, SOFTIRQ, IOWAIT, HARDIRQ, NICE };
 
   explicit CPUTimeMonitor(
       std::shared_ptr<TTicker> ticker,
@@ -97,14 +97,26 @@ class CPUTimeMonitor : MonitorBase<Ticker<60000, 1000, 10, 3>> {
       const std::optional<std::string>& targetId = std::nullopt,
       DataSource dataSource = DataSource::PROC_STAT);
 
-  // Get the average CPU breakdown value (idle, softirq, iowait, hardirq)
+  // Get the average CPU breakdown value (idle, softirq, iowait, hardirq, nice)
   // for a target over the specified time window. Returns raw cores used.
   std::optional<double> getCpuBreakdownAvg(
       Granularity gran,
       uint64_t seconds_ago,
       CpuBreakdown breakdown,
       const std::optional<std::string>& targetId = std::nullopt);
 
+  std::optional<double> getCpuBreakdownMin(
+      Granularity gran,
+      uint64_t seconds_ago,
+      CpuBreakdown breakdown,
+      const std::optional<std::string>& targetId = std::nullopt);
+
+  std::optional<double> getCpuBreakdownMax(
+      Granularity gran,
+      uint64_t seconds_ago,
+      CpuBreakdown breakdown,
+      const std::optional<std::string>& targetId = std::nullopt);
+
  private:
   // Reads CPU time data from /proc/stat
   // If read_per_core is true, reads per-core data in addition to the

dynolog/tests/CPUTimeMonitorTest.cpp

@@ -1048,5 +1048,74 @@ TEST_F(CPUTimeMonitorTest, testBreakdownSeriesKeyPattern) {
   monitor->deRegisterTarget("key_test");
 }
 
+TEST_F(CPUTimeMonitorTest, testNiceBreakdownType) {
+  monitor->registerTarget("target_a", {0, 1, 2, 3});
+  monitor->tick(major_tick_60s);
+  monitor->tick(major_tick_60s);
+
+  auto avg = monitor->getCpuBreakdownAvg(
+      CPUTimeMonitor::Granularity::MINUTE,
+      60,
+      CPUTimeMonitor::CpuBreakdown::NICE,
+      "target_a");
+  ASSERT_TRUE(avg.has_value());
+  EXPECT_DOUBLE_EQ(avg.value(), 0.0);
+
+  monitor->deRegisterTarget("target_a");
+
+  EXPECT_EQ(
+      monitor->getCpuBreakdownAvg(
+          CPUTimeMonitor::Granularity::MINUTE,
+          60,
+          CPUTimeMonitor::CpuBreakdown::NICE,
+          "target_a"),
+      std::nullopt);
+}
+
+TEST_F(CPUTimeMonitorTest, testBreakdownMinMax) {
+  monitor->registerTarget("minmax_test", {0, 1, 2, 3});
+  monitor->tick(major_tick_60s);
+  monitor->tick(major_tick_60s);
+
+  for (const auto& bd :
+       {CPUTimeMonitor::CpuBreakdown::IDLE,
+        CPUTimeMonitor::CpuBreakdown::SOFTIRQ,
+        CPUTimeMonitor::CpuBreakdown::IOWAIT,
+        CPUTimeMonitor::CpuBreakdown::HARDIRQ,
+        CPUTimeMonitor::CpuBreakdown::NICE}) {
+    auto minVal = monitor->getCpuBreakdownMin(
+        CPUTimeMonitor::Granularity::MINUTE, 60, bd, "minmax_test");
+    auto maxVal = monitor->getCpuBreakdownMax(
+        CPUTimeMonitor::Granularity::MINUTE, 60, bd, "minmax_test");
+    ASSERT_TRUE(minVal.has_value());
+    ASSERT_TRUE(maxVal.has_value());
+    EXPECT_LE(minVal.value(), maxVal.value());
+  }
+
+  monitor->deRegisterTarget("minmax_test");
+}
+
+TEST_F(CPUTimeMonitorTest, testBreakdownMinMaxNulloptAfterDeregister) {
+  monitor->registerTarget("dereg_test", {0, 1});
+  monitor->tick(major_tick_60s);
+  monitor->tick(major_tick_60s);
+
+  auto minBefore = monitor->getCpuBreakdownMin(
+      CPUTimeMonitor::Granularity::MINUTE,
+      60,
+      CPUTimeMonitor::CpuBreakdown::NICE,
+      "dereg_test");
+  ASSERT_TRUE(minBefore.has_value());
+
+  monitor->deRegisterTarget("dereg_test");
+
+  auto minAfter = monitor->getCpuBreakdownMin(
+      CPUTimeMonitor::Granularity::MINUTE,
+      60,
+      CPUTimeMonitor::CpuBreakdown::NICE,
+      "dereg_test");
+  EXPECT_EQ(minAfter, std::nullopt);
+}
+
 } // namespace dynolog
 } // namespace facebook