Skip to content

Commit 4eec6f5

Browse files
cmgracefacebook-github-bot
cmgrace
authored and
facebook-github-bot
committed
Add CPU breakdown min/max aggregations and cpu.nice metric for allotment ODS (#554)
Summary: Extends the allotment-level CPU metrics to support MySQL and ZippyDB stacking migration. This is part of the broader effort to expose allotment-level ODS metrics as these workloads move from whole-host shapes to stacked allotments. Changes: - Add cpu.nice breakdown metric from /proc/stat per-CPU nice field (not available in cgroup v2 cpu.stat) via CPUTimeMonitor, with avg/min/max aggregations - Add min.60/max.60 aggregations for cpu.iowait and cpu.saturation-pct from CPUTimeMonitor /proc/stat data - Add min.60/max.60 aggregations for cpu.user and cpu.sys from the cgroup ring buffer (AllotmentsMonitor::getCgroupCounterMinMax), keeping the existing cgroup data source for consistency with the avg metric - Add kCpuSaturationMin/Max key specs to AllotmentCpuOdsKeySpecs - Extract granularityToLevel() helper to deduplicate 6 identical switch blocks in CPUTimeMonitor - Fix readCgroupStat_ fallback in getMinCPUCoresUsage/getMaxCPUCoresUsage to match existing getStat() behavior All new ODS keys are gated behind TargetedOdsFilter::shouldPublish() with defaultPublish=false, so they only emit when an explicit allow rule is configured for the workload. New ODS keys (all require TargetedOdsFilter config to publish): - dyno.allotment.cpu.nice (avg via ODS suffix) - dyno.allotment.cpu.{nice,iowait,idle,softirq,hardirq}.min.60 / .max.60 - dyno.allotment.cpu.user.min.60 / .max.60 - dyno.allotment.cpu.sys.min.60 / .max.60 - dyno.allotment.cpu.saturation-pct.min.60 / .max.60 Design doc: https://docs.google.com/document/d/16ea49QzoL60PTtFQ0DXss4Bmq7aCnSGh4MuG9phMmO0/edit Differential Revision: D104745222
1 parent 1fe6a0b commit 4eec6f5

3 files changed

Lines changed: 197 additions & 20 deletions

File tree

‎dynolog/src/CPUTimeMonitor.cpp‎

Lines changed: 78 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include "dynolog/src/CPUTimeMonitor.h"
99
#include <dynolog/src/metric_frame/MetricFrameTsUnit.h>
1010

11+
#include <algorithm>
1112
#include <utility>
1213

1314
enum { IDX_MIN = 0, IDX_SEC = 1, IDX_HUNDRED_MS = 2 };
@@ -19,14 +20,16 @@ static const std::string kBreakdownIdle = ".idle";
1920
static const std::string kBreakdownSoftirq = ".softirq";
2021
static const std::string kBreakdownIowait = ".iowait";
2122
static const std::string kBreakdownHardirq = ".hardirq";
23+
static const std::string kBreakdownNice = ".nice";
2224

2325
// All breakdown suffixes in a single list to avoid iterating in multiple
2426
// places.
2527
static const std::vector<std::string> kBreakdownSuffixes = {
2628
kBreakdownIdle,
2729
kBreakdownSoftirq,
2830
kBreakdownIowait,
29-
kBreakdownHardirq};
31+
kBreakdownHardirq,
32+
kBreakdownNice};
3033

3134
// Helper to add breakdown series for a target to a MetricFrameMap.
3235
static void addBreakdownSeries(
@@ -51,13 +54,27 @@ static const std::string& breakdownSuffix(
5154
return kBreakdownIowait;
5255
case CPUTimeMonitor::CpuBreakdown::HARDIRQ:
5356
return kBreakdownHardirq;
57+
case CPUTimeMonitor::CpuBreakdown::NICE:
58+
return kBreakdownNice;
5459
}
5560
return kBreakdownIdle; // unreachable
5661
}
5762

5863
constexpr int kRingbufferSizeMinutes = 6;
5964
static const std::string kHostCgroupPath = "/sys/fs/cgroup";
6065

66+
static std::optional<int> granularityToLevel(CPUTimeMonitor::Granularity gran) {
67+
switch (gran) {
68+
case CPUTimeMonitor::Granularity::MINUTE:
69+
return IDX_MIN;
70+
case CPUTimeMonitor::Granularity::SECOND:
71+
return IDX_SEC;
72+
case CPUTimeMonitor::Granularity::HUNDRED_MS:
73+
return IDX_HUNDRED_MS;
74+
}
75+
return std::nullopt;
76+
}
77+
6178
std::array<MetricFrameMap, 3> CPUTimeMonitor::createMetricFrameArray() {
6279
return {
6380
MetricFrameMap(
@@ -194,20 +211,11 @@ std::optional<double> CPUTimeMonitor::getCpuBreakdownAvg(
194211
TimePoint now = std::chrono::steady_clock::now();
195212
std::shared_lock lock(dataLock_);
196213

197-
int level = 0;
198-
switch (gran) {
199-
case Granularity::MINUTE:
200-
level = IDX_MIN;
201-
break;
202-
case Granularity::SECOND:
203-
level = IDX_SEC;
204-
break;
205-
case Granularity::HUNDRED_MS:
206-
level = IDX_HUNDRED_MS;
207-
break;
208-
default:
209-
return std::nullopt;
214+
auto levelOpt = granularityToLevel(gran);
215+
if (!levelOpt.has_value()) {
216+
return std::nullopt;
210217
}
218+
int level = *levelOpt;
211219

212220
auto& frame = procUsageMetricFrames_[level];
213221
auto slice = frame.slice(now - std::chrono::seconds(seconds_ago), now);
@@ -248,6 +256,56 @@ const MetricFrameMap* CPUTimeMonitor::getMetricFrame(
248256
: &procUsageMetricFrames_[level];
249257
}
250258

259+
// Combined min+max in a single lock+slice+copy pass. Breakdown values are
260+
// already stored as per-second rates (cores) by processProcUsage(), so we
261+
// pick min/max directly from the raw samples using std::minmax_element.
262+
std::optional<std::pair<double, double>> CPUTimeMonitor::getCpuBreakdownMinMax(
263+
Granularity gran,
264+
uint64_t seconds_ago,
265+
CpuBreakdown breakdown,
266+
const std::optional<std::string>& targetId) {
267+
TimePoint now = std::chrono::steady_clock::now();
268+
std::shared_lock lock(dataLock_);
269+
270+
auto levelOpt = granularityToLevel(gran);
271+
if (!levelOpt.has_value()) {
272+
return std::nullopt;
273+
}
274+
int level = *levelOpt;
275+
276+
auto& frame = procUsageMetricFrames_[level];
277+
auto slice = frame.slice(now - std::chrono::seconds(seconds_ago), now);
278+
if (slice == std::nullopt) {
279+
return std::nullopt;
280+
}
281+
std::string key = targetId.value_or("host") + breakdownSuffix(breakdown);
282+
auto series = slice->series<double>(key);
283+
if (series == std::nullopt || series->size() == 0) {
284+
return std::nullopt;
285+
}
286+
auto data = series->raw();
287+
auto [minIt, maxIt] = std::minmax_element(data.begin(), data.end());
288+
return std::make_pair(*minIt, *maxIt);
289+
}
290+
291+
std::optional<double> CPUTimeMonitor::getCpuBreakdownMin(
292+
Granularity gran,
293+
uint64_t seconds_ago,
294+
CpuBreakdown breakdown,
295+
const std::optional<std::string>& targetId) {
296+
auto result = getCpuBreakdownMinMax(gran, seconds_ago, breakdown, targetId);
297+
return result ? std::optional(result->first) : std::nullopt;
298+
}
299+
300+
std::optional<double> CPUTimeMonitor::getCpuBreakdownMax(
301+
Granularity gran,
302+
uint64_t seconds_ago,
303+
CpuBreakdown breakdown,
304+
const std::optional<std::string>& targetId) {
305+
auto result = getCpuBreakdownMinMax(gran, seconds_ago, breakdown, targetId);
306+
return result ? std::optional(result->second) : std::nullopt;
307+
}
308+
251309
void CPUTimeMonitor::tick(TMask mask) {
252310
TimePoint tickTime = std::chrono::steady_clock::now();
253311
bool readPerCore = false;
@@ -589,8 +647,10 @@ void CPUTimeMonitor::processProcUsage(
589647
auto softirqDeltaOpt = safeDelta(newCt.y, lastCt.y);
590648
auto iowaitDeltaOpt = safeDelta(newCt.w, lastCt.w);
591649
auto hardirqDeltaOpt = safeDelta(newCt.x, lastCt.x);
650+
auto niceDeltaOpt = safeDelta(newCt.n, lastCt.n);
592651

593-
if (idleDeltaOpt && softirqDeltaOpt && iowaitDeltaOpt && hardirqDeltaOpt) {
652+
if (idleDeltaOpt && softirqDeltaOpt && iowaitDeltaOpt && hardirqDeltaOpt &&
653+
niceDeltaOpt) {
594654
double idleCores =
595655
static_cast<double>(*idleDeltaOpt) * msPerJiffy / wallDelta;
596656
double softirqCores =
@@ -599,11 +659,14 @@ void CPUTimeMonitor::processProcUsage(
599659
static_cast<double>(*iowaitDeltaOpt) * msPerJiffy / wallDelta;
600660
double hardirqCores =
601661
static_cast<double>(*hardirqDeltaOpt) * msPerJiffy / wallDelta;
662+
double niceCores =
663+
static_cast<double>(*niceDeltaOpt) * msPerJiffy / wallDelta;
602664

603665
line.emplace_back(targetId + kBreakdownIdle, idleCores);
604666
line.emplace_back(targetId + kBreakdownSoftirq, softirqCores);
605667
line.emplace_back(targetId + kBreakdownIowait, iowaitCores);
606668
line.emplace_back(targetId + kBreakdownHardirq, hardirqCores);
669+
line.emplace_back(targetId + kBreakdownNice, niceCores);
607670
}
608671
}
609672
if (!line.empty()) {

‎dynolog/src/CPUTimeMonitor.h‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ class CPUTimeMonitor : MonitorBase<Ticker<60000, 1000, 10, 3>> {
6060

6161
enum class Granularity { MINUTE, SECOND, HUNDRED_MS };
6262
enum class DataSource { PROC_STAT, CGROUP_STAT };
63-
enum class CpuBreakdown { IDLE, SOFTIRQ, IOWAIT, HARDIRQ };
63+
enum class CpuBreakdown { IDLE, SOFTIRQ, IOWAIT, HARDIRQ, NICE };
6464

6565
explicit CPUTimeMonitor(
6666
std::shared_ptr<TTicker> ticker,
@@ -97,18 +97,34 @@ class CPUTimeMonitor : MonitorBase<Ticker<60000, 1000, 10, 3>> {
9797
const std::optional<std::string>& targetId = std::nullopt,
9898
DataSource dataSource = DataSource::PROC_STAT);
9999

100-
// Get the average CPU breakdown value (idle, softirq, iowait, hardirq)
100+
// Get the average CPU breakdown value (idle, softirq, iowait, hardirq, nice)
101101
// for a target over the specified time window. Returns raw cores used.
102102
std::optional<double> getCpuBreakdownAvg(
103103
Granularity gran,
104104
uint64_t seconds_ago,
105105
CpuBreakdown breakdown,
106106
const std::optional<std::string>& targetId = std::nullopt);
107107

108+
std::optional<double> getCpuBreakdownMin(
109+
Granularity gran,
110+
uint64_t seconds_ago,
111+
CpuBreakdown breakdown,
112+
const std::optional<std::string>& targetId = std::nullopt);
113+
114+
std::optional<double> getCpuBreakdownMax(
115+
Granularity gran,
116+
uint64_t seconds_ago,
117+
CpuBreakdown breakdown,
118+
const std::optional<std::string>& targetId = std::nullopt);
119+
120+
// Combined min+max in a single lock+slice+copy pass.
121+
std::optional<std::pair<double, double>> getCpuBreakdownMinMax(
122+
Granularity gran,
123+
uint64_t seconds_ago,
124+
CpuBreakdown breakdown,
125+
const std::optional<std::string>& targetId = std::nullopt);
126+
108127
private:
109-
// Reads CPU time data from /proc/stat
110-
// If read_per_core is true, reads per-core data in addition to the
111-
// all-core data (first element)
112128
enum class Statistic { AVG, QUANTILE };
113129

114130
std::vector<::dynolog::CpuTime> readProcStat(bool read_per_core = false);

‎dynolog/tests/CPUTimeMonitorTest.cpp‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1048,5 +1048,103 @@ TEST_F(CPUTimeMonitorTest, testBreakdownSeriesKeyPattern) {
10481048
monitor->deRegisterTarget("key_test");
10491049
}
10501050

1051+
TEST_F(CPUTimeMonitorTest, testNiceBreakdownType) {
1052+
monitor->registerTarget("target_a", {0, 1, 2, 3});
1053+
monitor->tick(major_tick_60s);
1054+
monitor->tick(major_tick_60s);
1055+
1056+
auto avg = monitor->getCpuBreakdownAvg(
1057+
CPUTimeMonitor::Granularity::MINUTE,
1058+
60,
1059+
CPUTimeMonitor::CpuBreakdown::NICE,
1060+
"target_a");
1061+
ASSERT_TRUE(avg.has_value());
1062+
EXPECT_DOUBLE_EQ(avg.value(), 0.0);
1063+
1064+
monitor->deRegisterTarget("target_a");
1065+
1066+
EXPECT_EQ(
1067+
monitor->getCpuBreakdownAvg(
1068+
CPUTimeMonitor::Granularity::MINUTE,
1069+
60,
1070+
CPUTimeMonitor::CpuBreakdown::NICE,
1071+
"target_a"),
1072+
std::nullopt);
1073+
}
1074+
1075+
TEST_F(CPUTimeMonitorTest, testBreakdownMinMax) {
1076+
monitor->registerTarget("minmax_test", {0, 1, 2, 3});
1077+
monitor->tick(major_tick_60s);
1078+
monitor->tick(major_tick_60s);
1079+
1080+
for (const auto& bd :
1081+
{CPUTimeMonitor::CpuBreakdown::IDLE,
1082+
CPUTimeMonitor::CpuBreakdown::SOFTIRQ,
1083+
CPUTimeMonitor::CpuBreakdown::IOWAIT,
1084+
CPUTimeMonitor::CpuBreakdown::HARDIRQ,
1085+
CPUTimeMonitor::CpuBreakdown::NICE}) {
1086+
auto minVal = monitor->getCpuBreakdownMin(
1087+
CPUTimeMonitor::Granularity::MINUTE, 60, bd, "minmax_test");
1088+
auto maxVal = monitor->getCpuBreakdownMax(
1089+
CPUTimeMonitor::Granularity::MINUTE, 60, bd, "minmax_test");
1090+
ASSERT_TRUE(minVal.has_value());
1091+
ASSERT_TRUE(maxVal.has_value());
1092+
EXPECT_LE(minVal.value(), maxVal.value());
1093+
}
1094+
1095+
monitor->deRegisterTarget("minmax_test");
1096+
}
1097+
1098+
TEST_F(CPUTimeMonitorTest, testBreakdownMinMaxCombined) {
1099+
monitor->registerTarget("combined_test", {0, 1, 2, 3});
1100+
monitor->tick(major_tick_60s);
1101+
monitor->tick(major_tick_60s);
1102+
1103+
for (const auto& bd :
1104+
{CPUTimeMonitor::CpuBreakdown::IDLE,
1105+
CPUTimeMonitor::CpuBreakdown::SOFTIRQ,
1106+
CPUTimeMonitor::CpuBreakdown::IOWAIT,
1107+
CPUTimeMonitor::CpuBreakdown::HARDIRQ,
1108+
CPUTimeMonitor::CpuBreakdown::NICE}) {
1109+
auto minMax = monitor->getCpuBreakdownMinMax(
1110+
CPUTimeMonitor::Granularity::MINUTE, 60, bd, "combined_test");
1111+
auto minVal = monitor->getCpuBreakdownMin(
1112+
CPUTimeMonitor::Granularity::MINUTE, 60, bd, "combined_test");
1113+
auto maxVal = monitor->getCpuBreakdownMax(
1114+
CPUTimeMonitor::Granularity::MINUTE, 60, bd, "combined_test");
1115+
1116+
ASSERT_TRUE(minMax.has_value());
1117+
ASSERT_TRUE(minVal.has_value());
1118+
ASSERT_TRUE(maxVal.has_value());
1119+
EXPECT_DOUBLE_EQ(minMax->first, minVal.value());
1120+
EXPECT_DOUBLE_EQ(minMax->second, maxVal.value());
1121+
EXPECT_LE(minMax->first, minMax->second);
1122+
}
1123+
1124+
monitor->deRegisterTarget("combined_test");
1125+
}
1126+
1127+
TEST_F(CPUTimeMonitorTest, testBreakdownMinMaxNulloptAfterDeregister) {
1128+
monitor->registerTarget("dereg_test", {0, 1});
1129+
monitor->tick(major_tick_60s);
1130+
monitor->tick(major_tick_60s);
1131+
1132+
auto minBefore = monitor->getCpuBreakdownMin(
1133+
CPUTimeMonitor::Granularity::MINUTE,
1134+
60,
1135+
CPUTimeMonitor::CpuBreakdown::NICE,
1136+
"dereg_test");
1137+
ASSERT_TRUE(minBefore.has_value());
1138+
1139+
monitor->deRegisterTarget("dereg_test");
1140+
1141+
auto minAfter = monitor->getCpuBreakdownMin(
1142+
CPUTimeMonitor::Granularity::MINUTE,
1143+
60,
1144+
CPUTimeMonitor::CpuBreakdown::NICE,
1145+
"dereg_test");
1146+
EXPECT_EQ(minAfter, std::nullopt);
1147+
}
1148+
10511149
} // namespace dynolog
10521150
} // namespace facebook

0 commit comments

Comments
 (0)