From 72a43fb854b5f5e50044f4a82f47d572c76dcae8 Mon Sep 17 00:00:00 2001 From: Grace Cheng Date: Thu, 28 May 2026 11:04:55 -0700 Subject: [PATCH 1/2] Add CPU breakdown min/max aggregations and cpu.nice metric for allotment ODS (#554) Summary: Extends the allotment-level CPU metrics to support MySQL and ZippyDB stacking migration. This is part of the broader effort to expose allotment-level ODS metrics as these workloads move from whole-host shapes to stacked allotments. Changes: - Add cpu.nice breakdown metric from /proc/stat per-CPU nice field (not available in cgroup v2 cpu.stat) via CPUTimeMonitor, with avg/min/max aggregations - Add min.60/max.60 aggregations for cpu.iowait and cpu.saturation-pct from CPUTimeMonitor /proc/stat data - Add min.60/max.60 aggregations for cpu.user and cpu.sys from the cgroup ring buffer (AllotmentsMonitor::getCgroupCounterMinMax), keeping the existing cgroup data source for consistency with the avg metric - Add kCpuSaturationMin/Max key specs to AllotmentCpuOdsKeySpecs - Extract granularityToLevel() helper to deduplicate 6 identical switch blocks in CPUTimeMonitor - Fix readCgroupStat_ fallback in getMinCPUCoresUsage/getMaxCPUCoresUsage to match existing getStat() behavior All new ODS keys are gated behind TargetedOdsFilter::shouldPublish() with defaultPublish=false, so they only emit when an explicit allow rule is configured for the workload. New ODS keys (all require TargetedOdsFilter config to publish): - dyno.allotment.cpu.nice (avg via ODS suffix) - dyno.allotment.cpu.{nice,iowait,idle,softirq,hardirq}.min.60 / .max.60 - dyno.allotment.cpu.user.min.60 / .max.60 - dyno.allotment.cpu.sys.min.60 / .max.60 - dyno.allotment.cpu.saturation-pct.min.60 / .max.60 Design doc: https://docs.google.com/document/d/16ea49QzoL60PTtFQ0DXss4Bmq7aCnSGh4MuG9phMmO0/edit Differential Revision: D104745222 --- dynolog/src/CPUTimeMonitor.cpp | 93 +++++++++++++++++++++----- dynolog/src/CPUTimeMonitor.h | 26 ++++++-- dynolog/tests/CPUTimeMonitorTest.cpp | 98 ++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+), 20 deletions(-) diff --git a/dynolog/src/CPUTimeMonitor.cpp b/dynolog/src/CPUTimeMonitor.cpp index ff306af4..e4ffdee9 100644 --- a/dynolog/src/CPUTimeMonitor.cpp +++ b/dynolog/src/CPUTimeMonitor.cpp @@ -8,6 +8,7 @@ #include "dynolog/src/CPUTimeMonitor.h" #include +#include #include enum { IDX_MIN = 0, IDX_SEC = 1, IDX_HUNDRED_MS = 2 }; @@ -19,6 +20,7 @@ static const std::string kBreakdownIdle = ".idle"; static const std::string kBreakdownSoftirq = ".softirq"; static const std::string kBreakdownIowait = ".iowait"; static const std::string kBreakdownHardirq = ".hardirq"; +static const std::string kBreakdownNice = ".nice"; // All breakdown suffixes in a single list to avoid iterating in multiple // places. @@ -26,7 +28,8 @@ static const std::vector kBreakdownSuffixes = { kBreakdownIdle, kBreakdownSoftirq, kBreakdownIowait, - kBreakdownHardirq}; + kBreakdownHardirq, + kBreakdownNice}; // Helper to add breakdown series for a target to a MetricFrameMap. static void addBreakdownSeries( @@ -51,6 +54,8 @@ static const std::string& breakdownSuffix( return kBreakdownIowait; case CPUTimeMonitor::CpuBreakdown::HARDIRQ: return kBreakdownHardirq; + case CPUTimeMonitor::CpuBreakdown::NICE: + return kBreakdownNice; } return kBreakdownIdle; // unreachable } @@ -58,6 +63,18 @@ static const std::string& breakdownSuffix( constexpr int kRingbufferSizeMinutes = 6; static const std::string kHostCgroupPath = "/sys/fs/cgroup"; +static std::optional granularityToLevel(CPUTimeMonitor::Granularity gran) { + switch (gran) { + case CPUTimeMonitor::Granularity::MINUTE: + return IDX_MIN; + case CPUTimeMonitor::Granularity::SECOND: + return IDX_SEC; + case CPUTimeMonitor::Granularity::HUNDRED_MS: + return IDX_HUNDRED_MS; + } + return std::nullopt; +} + std::array CPUTimeMonitor::createMetricFrameArray() { return { MetricFrameMap( @@ -194,20 +211,11 @@ std::optional CPUTimeMonitor::getCpuBreakdownAvg( TimePoint now = std::chrono::steady_clock::now(); std::shared_lock lock(dataLock_); - int level = 0; - switch (gran) { - case Granularity::MINUTE: - level = IDX_MIN; - break; - case Granularity::SECOND: - level = IDX_SEC; - break; - case Granularity::HUNDRED_MS: - level = IDX_HUNDRED_MS; - break; - default: - return std::nullopt; + auto levelOpt = granularityToLevel(gran); + if (!levelOpt.has_value()) { + return std::nullopt; } + int level = *levelOpt; auto& frame = procUsageMetricFrames_[level]; auto slice = frame.slice(now - std::chrono::seconds(seconds_ago), now); @@ -248,6 +256,56 @@ const MetricFrameMap* CPUTimeMonitor::getMetricFrame( : &procUsageMetricFrames_[level]; } +// Combined min+max in a single lock+slice+copy pass. Breakdown values are +// already stored as per-second rates (cores) by processProcUsage(), so we +// pick min/max directly from the raw samples using std::minmax_element. +std::optional> CPUTimeMonitor::getCpuBreakdownMinMax( + Granularity gran, + uint64_t seconds_ago, + CpuBreakdown breakdown, + const std::optional& targetId) { + TimePoint now = std::chrono::steady_clock::now(); + std::shared_lock lock(dataLock_); + + auto levelOpt = granularityToLevel(gran); + if (!levelOpt.has_value()) { + return std::nullopt; + } + int level = *levelOpt; + + auto& frame = procUsageMetricFrames_[level]; + auto slice = frame.slice(now - std::chrono::seconds(seconds_ago), now); + if (slice == std::nullopt) { + return std::nullopt; + } + std::string key = targetId.value_or("host") + breakdownSuffix(breakdown); + auto series = slice->series(key); + if (series == std::nullopt || series->size() == 0) { + return std::nullopt; + } + auto data = series->raw(); + auto [minIt, maxIt] = std::minmax_element(data.begin(), data.end()); + return std::make_pair(*minIt, *maxIt); +} + +std::optional CPUTimeMonitor::getCpuBreakdownMin( + Granularity gran, + uint64_t seconds_ago, + CpuBreakdown breakdown, + const std::optional& targetId) { + auto result = getCpuBreakdownMinMax(gran, seconds_ago, breakdown, targetId); + return result ? std::optional(result->first) : std::nullopt; +} + +std::optional CPUTimeMonitor::getCpuBreakdownMax( + Granularity gran, + uint64_t seconds_ago, + CpuBreakdown breakdown, + const std::optional& targetId) { + auto result = getCpuBreakdownMinMax(gran, seconds_ago, breakdown, targetId); + return result ? std::optional(result->second) : std::nullopt; +} + void CPUTimeMonitor::tick(TMask mask) { TimePoint tickTime = std::chrono::steady_clock::now(); bool readPerCore = false; @@ -589,8 +647,10 @@ void CPUTimeMonitor::processProcUsage( auto softirqDeltaOpt = safeDelta(newCt.y, lastCt.y); auto iowaitDeltaOpt = safeDelta(newCt.w, lastCt.w); auto hardirqDeltaOpt = safeDelta(newCt.x, lastCt.x); + auto niceDeltaOpt = safeDelta(newCt.n, lastCt.n); - if (idleDeltaOpt && softirqDeltaOpt && iowaitDeltaOpt && hardirqDeltaOpt) { + if (idleDeltaOpt && softirqDeltaOpt && iowaitDeltaOpt && hardirqDeltaOpt && + niceDeltaOpt) { double idleCores = static_cast(*idleDeltaOpt) * msPerJiffy / wallDelta; double softirqCores = @@ -599,11 +659,14 @@ void CPUTimeMonitor::processProcUsage( static_cast(*iowaitDeltaOpt) * msPerJiffy / wallDelta; double hardirqCores = static_cast(*hardirqDeltaOpt) * msPerJiffy / wallDelta; + double niceCores = + static_cast(*niceDeltaOpt) * msPerJiffy / wallDelta; line.emplace_back(targetId + kBreakdownIdle, idleCores); line.emplace_back(targetId + kBreakdownSoftirq, softirqCores); line.emplace_back(targetId + kBreakdownIowait, iowaitCores); line.emplace_back(targetId + kBreakdownHardirq, hardirqCores); + line.emplace_back(targetId + kBreakdownNice, niceCores); } } if (!line.empty()) { diff --git a/dynolog/src/CPUTimeMonitor.h b/dynolog/src/CPUTimeMonitor.h index 8fc59866..7de39ead 100644 --- a/dynolog/src/CPUTimeMonitor.h +++ b/dynolog/src/CPUTimeMonitor.h @@ -60,7 +60,7 @@ class CPUTimeMonitor : MonitorBase> { enum class Granularity { MINUTE, SECOND, HUNDRED_MS }; enum class DataSource { PROC_STAT, CGROUP_STAT }; - enum class CpuBreakdown { IDLE, SOFTIRQ, IOWAIT, HARDIRQ }; + enum class CpuBreakdown { IDLE, SOFTIRQ, IOWAIT, HARDIRQ, NICE }; explicit CPUTimeMonitor( std::shared_ptr ticker, @@ -97,7 +97,7 @@ class CPUTimeMonitor : MonitorBase> { const std::optional& targetId = std::nullopt, DataSource dataSource = DataSource::PROC_STAT); - // Get the average CPU breakdown value (idle, softirq, iowait, hardirq) + // Get the average CPU breakdown value (idle, softirq, iowait, hardirq, nice) // for a target over the specified time window. Returns raw cores used. std::optional getCpuBreakdownAvg( Granularity gran, @@ -105,10 +105,26 @@ class CPUTimeMonitor : MonitorBase> { CpuBreakdown breakdown, const std::optional& targetId = std::nullopt); + std::optional getCpuBreakdownMin( + Granularity gran, + uint64_t seconds_ago, + CpuBreakdown breakdown, + const std::optional& targetId = std::nullopt); + + std::optional getCpuBreakdownMax( + Granularity gran, + uint64_t seconds_ago, + CpuBreakdown breakdown, + const std::optional& targetId = std::nullopt); + + // Combined min+max in a single lock+slice+copy pass. + std::optional> getCpuBreakdownMinMax( + Granularity gran, + uint64_t seconds_ago, + CpuBreakdown breakdown, + const std::optional& targetId = std::nullopt); + private: - // Reads CPU time data from /proc/stat - // If read_per_core is true, reads per-core data in addition to the - // all-core data (first element) enum class Statistic { AVG, QUANTILE }; std::vector<::dynolog::CpuTime> readProcStat(bool read_per_core = false); diff --git a/dynolog/tests/CPUTimeMonitorTest.cpp b/dynolog/tests/CPUTimeMonitorTest.cpp index 34210382..03d294b6 100644 --- a/dynolog/tests/CPUTimeMonitorTest.cpp +++ b/dynolog/tests/CPUTimeMonitorTest.cpp @@ -1048,5 +1048,103 @@ TEST_F(CPUTimeMonitorTest, testBreakdownSeriesKeyPattern) { monitor->deRegisterTarget("key_test"); } +TEST_F(CPUTimeMonitorTest, testNiceBreakdownType) { + monitor->registerTarget("target_a", {0, 1, 2, 3}); + monitor->tick(major_tick_60s); + monitor->tick(major_tick_60s); + + auto avg = monitor->getCpuBreakdownAvg( + CPUTimeMonitor::Granularity::MINUTE, + 60, + CPUTimeMonitor::CpuBreakdown::NICE, + "target_a"); + ASSERT_TRUE(avg.has_value()); + EXPECT_DOUBLE_EQ(avg.value(), 0.0); + + monitor->deRegisterTarget("target_a"); + + EXPECT_EQ( + monitor->getCpuBreakdownAvg( + CPUTimeMonitor::Granularity::MINUTE, + 60, + CPUTimeMonitor::CpuBreakdown::NICE, + "target_a"), + std::nullopt); +} + +TEST_F(CPUTimeMonitorTest, testBreakdownMinMax) { + monitor->registerTarget("minmax_test", {0, 1, 2, 3}); + monitor->tick(major_tick_60s); + monitor->tick(major_tick_60s); + + for (const auto& bd : + {CPUTimeMonitor::CpuBreakdown::IDLE, + CPUTimeMonitor::CpuBreakdown::SOFTIRQ, + CPUTimeMonitor::CpuBreakdown::IOWAIT, + CPUTimeMonitor::CpuBreakdown::HARDIRQ, + CPUTimeMonitor::CpuBreakdown::NICE}) { + auto minVal = monitor->getCpuBreakdownMin( + CPUTimeMonitor::Granularity::MINUTE, 60, bd, "minmax_test"); + auto maxVal = monitor->getCpuBreakdownMax( + CPUTimeMonitor::Granularity::MINUTE, 60, bd, "minmax_test"); + ASSERT_TRUE(minVal.has_value()); + ASSERT_TRUE(maxVal.has_value()); + EXPECT_LE(minVal.value(), maxVal.value()); + } + + monitor->deRegisterTarget("minmax_test"); +} + +TEST_F(CPUTimeMonitorTest, testBreakdownMinMaxCombined) { + monitor->registerTarget("combined_test", {0, 1, 2, 3}); + monitor->tick(major_tick_60s); + monitor->tick(major_tick_60s); + + for (const auto& bd : + {CPUTimeMonitor::CpuBreakdown::IDLE, + CPUTimeMonitor::CpuBreakdown::SOFTIRQ, + CPUTimeMonitor::CpuBreakdown::IOWAIT, + CPUTimeMonitor::CpuBreakdown::HARDIRQ, + CPUTimeMonitor::CpuBreakdown::NICE}) { + auto minMax = monitor->getCpuBreakdownMinMax( + CPUTimeMonitor::Granularity::MINUTE, 60, bd, "combined_test"); + auto minVal = monitor->getCpuBreakdownMin( + CPUTimeMonitor::Granularity::MINUTE, 60, bd, "combined_test"); + auto maxVal = monitor->getCpuBreakdownMax( + CPUTimeMonitor::Granularity::MINUTE, 60, bd, "combined_test"); + + ASSERT_TRUE(minMax.has_value()); + ASSERT_TRUE(minVal.has_value()); + ASSERT_TRUE(maxVal.has_value()); + EXPECT_DOUBLE_EQ(minMax->first, minVal.value()); + EXPECT_DOUBLE_EQ(minMax->second, maxVal.value()); + EXPECT_LE(minMax->first, minMax->second); + } + + monitor->deRegisterTarget("combined_test"); +} + +TEST_F(CPUTimeMonitorTest, testBreakdownMinMaxNulloptAfterDeregister) { + monitor->registerTarget("dereg_test", {0, 1}); + monitor->tick(major_tick_60s); + monitor->tick(major_tick_60s); + + auto minBefore = monitor->getCpuBreakdownMin( + CPUTimeMonitor::Granularity::MINUTE, + 60, + CPUTimeMonitor::CpuBreakdown::NICE, + "dereg_test"); + ASSERT_TRUE(minBefore.has_value()); + + monitor->deRegisterTarget("dereg_test"); + + auto minAfter = monitor->getCpuBreakdownMin( + CPUTimeMonitor::Granularity::MINUTE, + 60, + CPUTimeMonitor::CpuBreakdown::NICE, + "dereg_test"); + EXPECT_EQ(minAfter, std::nullopt); +} + } // namespace dynolog } // namespace facebook From b3143b7698ef3e5194cdf6f7d84d2f09c4799b6b Mon Sep 17 00:00:00 2001 From: Grace Cheng Date: Thu, 28 May 2026 11:04:55 -0700 Subject: [PATCH 2/2] Add CPU saturation-pct min/max ODS metrics for allotment monitoring (#558) Summary: Adds min.60/max.60 aggregations for cpu.saturation-pct, extracted from D104745222 for a cleaner diff stack. New ODS keys (gated with TargetedOdsFilter defaultPublish=false): - dyno.allotment.cpu.saturation-pct.min.60 - dyno.allotment.cpu.saturation-pct.max.60 These use the same /proc/stat PROC_STAT data source as the existing saturation-pct avg and percentile metrics, with the readCgroupStat_ fallback honored. Values are normalized to percentage of the allotment cpuSaturationLimit. Adds getMinCPUCoresUsage() and getMaxCPUCoresUsage() to CPUTimeMonitor using raw() + std::min/max_element since MetricSeries lacks built-in min/max. Design doc: https://docs.google.com/document/d/16ea49QzoL60PTtFQ0DXss4Bmq7aCnSGh4MuG9phMmO0/edit Differential Revision: D105619438 --- dynolog/src/CPUTimeMonitor.cpp | 44 +++++++++++++++++++++++++ dynolog/src/CPUTimeMonitor.h | 18 ++++++++++ dynolog/tests/CPUTimeMonitorTest.cpp | 49 ++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+) diff --git a/dynolog/src/CPUTimeMonitor.cpp b/dynolog/src/CPUTimeMonitor.cpp index e4ffdee9..d2b6ef1c 100644 --- a/dynolog/src/CPUTimeMonitor.cpp +++ b/dynolog/src/CPUTimeMonitor.cpp @@ -306,6 +306,50 @@ std::optional CPUTimeMonitor::getCpuBreakdownMax( return result ? std::optional(result->second) : std::nullopt; } +std::optional> CPUTimeMonitor::getMinMaxCPUCoresUsage( + Granularity gran, + uint64_t seconds_ago, + const std::optional& targetId, + DataSource dataSource) { + TimePoint now = std::chrono::steady_clock::now(); + std::shared_lock lock(dataLock_); + + const auto* frame = getMetricFrame(gran, dataSource); + if (frame == nullptr) { + return std::nullopt; + } + auto slice = frame->slice(now - std::chrono::seconds(seconds_ago), now); + if (slice == std::nullopt) { + return std::nullopt; + } + std::string key = targetId.value_or("host"); + auto series = slice->series(key); + if (series == std::nullopt || series->size() == 0) { + return std::nullopt; + } + auto data = series->raw(); + auto [minIt, maxIt] = std::minmax_element(data.begin(), data.end()); + return std::make_pair(*minIt, *maxIt); +} + +std::optional CPUTimeMonitor::getMinCPUCoresUsage( + Granularity gran, + uint64_t seconds_ago, + const std::optional& targetId, + DataSource dataSource) { + auto result = getMinMaxCPUCoresUsage(gran, seconds_ago, targetId, dataSource); + return result ? std::optional(result->first) : std::nullopt; +} + +std::optional CPUTimeMonitor::getMaxCPUCoresUsage( + Granularity gran, + uint64_t seconds_ago, + const std::optional& targetId, + DataSource dataSource) { + auto result = getMinMaxCPUCoresUsage(gran, seconds_ago, targetId, dataSource); + return result ? std::optional(result->second) : std::nullopt; +} + void CPUTimeMonitor::tick(TMask mask) { TimePoint tickTime = std::chrono::steady_clock::now(); bool readPerCore = false; diff --git a/dynolog/src/CPUTimeMonitor.h b/dynolog/src/CPUTimeMonitor.h index 7de39ead..71f6adf1 100644 --- a/dynolog/src/CPUTimeMonitor.h +++ b/dynolog/src/CPUTimeMonitor.h @@ -124,6 +124,24 @@ class CPUTimeMonitor : MonitorBase> { CpuBreakdown breakdown, const std::optional& targetId = std::nullopt); + std::optional getMinCPUCoresUsage( + Granularity gran, + uint64_t seconds_ago, + const std::optional& targetId = std::nullopt, + DataSource dataSource = DataSource::PROC_STAT); + + std::optional getMaxCPUCoresUsage( + Granularity gran, + uint64_t seconds_ago, + const std::optional& targetId = std::nullopt, + DataSource dataSource = DataSource::PROC_STAT); + + std::optional> getMinMaxCPUCoresUsage( + Granularity gran, + uint64_t seconds_ago, + const std::optional& targetId = std::nullopt, + DataSource dataSource = DataSource::PROC_STAT); + private: enum class Statistic { AVG, QUANTILE }; diff --git a/dynolog/tests/CPUTimeMonitorTest.cpp b/dynolog/tests/CPUTimeMonitorTest.cpp index 03d294b6..87d9200f 100644 --- a/dynolog/tests/CPUTimeMonitorTest.cpp +++ b/dynolog/tests/CPUTimeMonitorTest.cpp @@ -1124,6 +1124,55 @@ TEST_F(CPUTimeMonitorTest, testBreakdownMinMaxCombined) { monitor->deRegisterTarget("combined_test"); } +TEST_F(CPUTimeMonitorTest, testMinMaxCPUCoresUsage) { + monitor->registerTarget("cores_test", {0, 1, 2, 3}); + monitor->tick(major_tick_60s); + monitor->tick(major_tick_60s); + + for (const auto& gran : + {CPUTimeMonitor::Granularity::MINUTE, + CPUTimeMonitor::Granularity::SECOND}) { + auto minVal = monitor->getMinCPUCoresUsage( + gran, 60, "cores_test", CPUTimeMonitor::DataSource::PROC_STAT); + auto maxVal = monitor->getMaxCPUCoresUsage( + gran, 60, "cores_test", CPUTimeMonitor::DataSource::PROC_STAT); + auto avgVal = monitor->getAvgCPUCoresUsage( + gran, 60, "cores_test", CPUTimeMonitor::DataSource::PROC_STAT); + + ASSERT_TRUE(minVal.has_value()); + ASSERT_TRUE(maxVal.has_value()); + ASSERT_TRUE(avgVal.has_value()); + EXPECT_LE(minVal.value(), avgVal.value()); + EXPECT_LE(avgVal.value(), maxVal.value()); + } + + monitor->deRegisterTarget("cores_test"); +} + +TEST(CPUTimeMonitorFallbackTest, testMinMaxCgroupStatFallback) { + // readCgroupStat=false: CGROUP_STAT requests fall back to PROC_STAT data + auto ticker = std::make_shared(); + auto mon = std::make_shared( + ticker, false, coreCount, getenv("TESTROOT"), true); + mon->tick(major_tick_60s); + mon->tick(major_tick_60s); + + auto minProc = mon->getMinCPUCoresUsage( + CPUTimeMonitor::Granularity::MINUTE, + 60, + std::nullopt, + CPUTimeMonitor::DataSource::PROC_STAT); + auto minCgroup = mon->getMinCPUCoresUsage( + CPUTimeMonitor::Granularity::MINUTE, + 60, + std::nullopt, + CPUTimeMonitor::DataSource::CGROUP_STAT); + + ASSERT_TRUE(minProc.has_value()); + ASSERT_TRUE(minCgroup.has_value()); + EXPECT_DOUBLE_EQ(minProc.value(), minCgroup.value()); +} + TEST_F(CPUTimeMonitorTest, testBreakdownMinMaxNulloptAfterDeregister) { monitor->registerTarget("dereg_test", {0, 1}); monitor->tick(major_tick_60s);