diff --git a/dynolog/src/CPUTimeMonitor.cpp b/dynolog/src/CPUTimeMonitor.cpp index ff306af4..d2b6ef1c 100644 --- a/dynolog/src/CPUTimeMonitor.cpp +++ b/dynolog/src/CPUTimeMonitor.cpp @@ -8,6 +8,7 @@ #include "dynolog/src/CPUTimeMonitor.h" #include +#include #include enum { IDX_MIN = 0, IDX_SEC = 1, IDX_HUNDRED_MS = 2 }; @@ -19,6 +20,7 @@ static const std::string kBreakdownIdle = ".idle"; static const std::string kBreakdownSoftirq = ".softirq"; static const std::string kBreakdownIowait = ".iowait"; static const std::string kBreakdownHardirq = ".hardirq"; +static const std::string kBreakdownNice = ".nice"; // All breakdown suffixes in a single list to avoid iterating in multiple // places. @@ -26,7 +28,8 @@ static const std::vector kBreakdownSuffixes = { kBreakdownIdle, kBreakdownSoftirq, kBreakdownIowait, - kBreakdownHardirq}; + kBreakdownHardirq, + kBreakdownNice}; // Helper to add breakdown series for a target to a MetricFrameMap. static void addBreakdownSeries( @@ -51,6 +54,8 @@ static const std::string& breakdownSuffix( return kBreakdownIowait; case CPUTimeMonitor::CpuBreakdown::HARDIRQ: return kBreakdownHardirq; + case CPUTimeMonitor::CpuBreakdown::NICE: + return kBreakdownNice; } return kBreakdownIdle; // unreachable } @@ -58,6 +63,18 @@ static const std::string& breakdownSuffix( constexpr int kRingbufferSizeMinutes = 6; static const std::string kHostCgroupPath = "/sys/fs/cgroup"; +static std::optional granularityToLevel(CPUTimeMonitor::Granularity gran) { + switch (gran) { + case CPUTimeMonitor::Granularity::MINUTE: + return IDX_MIN; + case CPUTimeMonitor::Granularity::SECOND: + return IDX_SEC; + case CPUTimeMonitor::Granularity::HUNDRED_MS: + return IDX_HUNDRED_MS; + } + return std::nullopt; +} + std::array CPUTimeMonitor::createMetricFrameArray() { return { MetricFrameMap( @@ -194,20 +211,11 @@ std::optional CPUTimeMonitor::getCpuBreakdownAvg( TimePoint now = std::chrono::steady_clock::now(); std::shared_lock lock(dataLock_); - int level = 0; - switch (gran) { - case Granularity::MINUTE: - level = IDX_MIN; - break; - case Granularity::SECOND: - level = IDX_SEC; - break; - case Granularity::HUNDRED_MS: - level = IDX_HUNDRED_MS; - break; - default: - return std::nullopt; + auto levelOpt = granularityToLevel(gran); + if (!levelOpt.has_value()) { + return std::nullopt; } + int level = *levelOpt; auto& frame = procUsageMetricFrames_[level]; auto slice = frame.slice(now - std::chrono::seconds(seconds_ago), now); @@ -248,6 +256,100 @@ const MetricFrameMap* CPUTimeMonitor::getMetricFrame( : &procUsageMetricFrames_[level]; } +// Combined min+max in a single lock+slice+copy pass. Breakdown values are +// already stored as per-second rates (cores) by processProcUsage(), so we +// pick min/max directly from the raw samples using std::minmax_element. +std::optional> CPUTimeMonitor::getCpuBreakdownMinMax( + Granularity gran, + uint64_t seconds_ago, + CpuBreakdown breakdown, + const std::optional& targetId) { + TimePoint now = std::chrono::steady_clock::now(); + std::shared_lock lock(dataLock_); + + auto levelOpt = granularityToLevel(gran); + if (!levelOpt.has_value()) { + return std::nullopt; + } + int level = *levelOpt; + + auto& frame = procUsageMetricFrames_[level]; + auto slice = frame.slice(now - std::chrono::seconds(seconds_ago), now); + if (slice == std::nullopt) { + return std::nullopt; + } + std::string key = targetId.value_or("host") + breakdownSuffix(breakdown); + auto series = slice->series(key); + if (series == std::nullopt || series->size() == 0) { + return std::nullopt; + } + auto data = series->raw(); + auto [minIt, maxIt] = std::minmax_element(data.begin(), data.end()); + return std::make_pair(*minIt, *maxIt); +} + +std::optional CPUTimeMonitor::getCpuBreakdownMin( + Granularity gran, + uint64_t seconds_ago, + CpuBreakdown breakdown, + const std::optional& targetId) { + auto result = getCpuBreakdownMinMax(gran, seconds_ago, breakdown, targetId); + return result ? std::optional(result->first) : std::nullopt; +} + +std::optional CPUTimeMonitor::getCpuBreakdownMax( + Granularity gran, + uint64_t seconds_ago, + CpuBreakdown breakdown, + const std::optional& targetId) { + auto result = getCpuBreakdownMinMax(gran, seconds_ago, breakdown, targetId); + return result ? std::optional(result->second) : std::nullopt; +} + +std::optional> CPUTimeMonitor::getMinMaxCPUCoresUsage( + Granularity gran, + uint64_t seconds_ago, + const std::optional& targetId, + DataSource dataSource) { + TimePoint now = std::chrono::steady_clock::now(); + std::shared_lock lock(dataLock_); + + const auto* frame = getMetricFrame(gran, dataSource); + if (frame == nullptr) { + return std::nullopt; + } + auto slice = frame->slice(now - std::chrono::seconds(seconds_ago), now); + if (slice == std::nullopt) { + return std::nullopt; + } + std::string key = targetId.value_or("host"); + auto series = slice->series(key); + if (series == std::nullopt || series->size() == 0) { + return std::nullopt; + } + auto data = series->raw(); + auto [minIt, maxIt] = std::minmax_element(data.begin(), data.end()); + return std::make_pair(*minIt, *maxIt); +} + +std::optional CPUTimeMonitor::getMinCPUCoresUsage( + Granularity gran, + uint64_t seconds_ago, + const std::optional& targetId, + DataSource dataSource) { + auto result = getMinMaxCPUCoresUsage(gran, seconds_ago, targetId, dataSource); + return result ? std::optional(result->first) : std::nullopt; +} + +std::optional CPUTimeMonitor::getMaxCPUCoresUsage( + Granularity gran, + uint64_t seconds_ago, + const std::optional& targetId, + DataSource dataSource) { + auto result = getMinMaxCPUCoresUsage(gran, seconds_ago, targetId, dataSource); + return result ? std::optional(result->second) : std::nullopt; +} + void CPUTimeMonitor::tick(TMask mask) { TimePoint tickTime = std::chrono::steady_clock::now(); bool readPerCore = false; @@ -589,8 +691,10 @@ void CPUTimeMonitor::processProcUsage( auto softirqDeltaOpt = safeDelta(newCt.y, lastCt.y); auto iowaitDeltaOpt = safeDelta(newCt.w, lastCt.w); auto hardirqDeltaOpt = safeDelta(newCt.x, lastCt.x); + auto niceDeltaOpt = safeDelta(newCt.n, lastCt.n); - if (idleDeltaOpt && softirqDeltaOpt && iowaitDeltaOpt && hardirqDeltaOpt) { + if (idleDeltaOpt && softirqDeltaOpt && iowaitDeltaOpt && hardirqDeltaOpt && + niceDeltaOpt) { double idleCores = static_cast(*idleDeltaOpt) * msPerJiffy / wallDelta; double softirqCores = @@ -599,11 +703,14 @@ void CPUTimeMonitor::processProcUsage( static_cast(*iowaitDeltaOpt) * msPerJiffy / wallDelta; double hardirqCores = static_cast(*hardirqDeltaOpt) * msPerJiffy / wallDelta; + double niceCores = + static_cast(*niceDeltaOpt) * msPerJiffy / wallDelta; line.emplace_back(targetId + kBreakdownIdle, idleCores); line.emplace_back(targetId + kBreakdownSoftirq, softirqCores); line.emplace_back(targetId + kBreakdownIowait, iowaitCores); line.emplace_back(targetId + kBreakdownHardirq, hardirqCores); + line.emplace_back(targetId + kBreakdownNice, niceCores); } } if (!line.empty()) { diff --git a/dynolog/src/CPUTimeMonitor.h b/dynolog/src/CPUTimeMonitor.h index 8fc59866..71f6adf1 100644 --- a/dynolog/src/CPUTimeMonitor.h +++ b/dynolog/src/CPUTimeMonitor.h @@ -60,7 +60,7 @@ class CPUTimeMonitor : MonitorBase> { enum class Granularity { MINUTE, SECOND, HUNDRED_MS }; enum class DataSource { PROC_STAT, CGROUP_STAT }; - enum class CpuBreakdown { IDLE, SOFTIRQ, IOWAIT, HARDIRQ }; + enum class CpuBreakdown { IDLE, SOFTIRQ, IOWAIT, HARDIRQ, NICE }; explicit CPUTimeMonitor( std::shared_ptr ticker, @@ -97,7 +97,7 @@ class CPUTimeMonitor : MonitorBase> { const std::optional& targetId = std::nullopt, DataSource dataSource = DataSource::PROC_STAT); - // Get the average CPU breakdown value (idle, softirq, iowait, hardirq) + // Get the average CPU breakdown value (idle, softirq, iowait, hardirq, nice) // for a target over the specified time window. Returns raw cores used. std::optional getCpuBreakdownAvg( Granularity gran, @@ -105,10 +105,44 @@ class CPUTimeMonitor : MonitorBase> { CpuBreakdown breakdown, const std::optional& targetId = std::nullopt); + std::optional getCpuBreakdownMin( + Granularity gran, + uint64_t seconds_ago, + CpuBreakdown breakdown, + const std::optional& targetId = std::nullopt); + + std::optional getCpuBreakdownMax( + Granularity gran, + uint64_t seconds_ago, + CpuBreakdown breakdown, + const std::optional& targetId = std::nullopt); + + // Combined min+max in a single lock+slice+copy pass. + std::optional> getCpuBreakdownMinMax( + Granularity gran, + uint64_t seconds_ago, + CpuBreakdown breakdown, + const std::optional& targetId = std::nullopt); + + std::optional getMinCPUCoresUsage( + Granularity gran, + uint64_t seconds_ago, + const std::optional& targetId = std::nullopt, + DataSource dataSource = DataSource::PROC_STAT); + + std::optional getMaxCPUCoresUsage( + Granularity gran, + uint64_t seconds_ago, + const std::optional& targetId = std::nullopt, + DataSource dataSource = DataSource::PROC_STAT); + + std::optional> getMinMaxCPUCoresUsage( + Granularity gran, + uint64_t seconds_ago, + const std::optional& targetId = std::nullopt, + DataSource dataSource = DataSource::PROC_STAT); + private: - // Reads CPU time data from /proc/stat - // If read_per_core is true, reads per-core data in addition to the - // all-core data (first element) enum class Statistic { AVG, QUANTILE }; std::vector<::dynolog::CpuTime> readProcStat(bool read_per_core = false); diff --git a/dynolog/tests/CPUTimeMonitorTest.cpp b/dynolog/tests/CPUTimeMonitorTest.cpp index 34210382..87d9200f 100644 --- a/dynolog/tests/CPUTimeMonitorTest.cpp +++ b/dynolog/tests/CPUTimeMonitorTest.cpp @@ -1048,5 +1048,152 @@ TEST_F(CPUTimeMonitorTest, testBreakdownSeriesKeyPattern) { monitor->deRegisterTarget("key_test"); } +TEST_F(CPUTimeMonitorTest, testNiceBreakdownType) { + monitor->registerTarget("target_a", {0, 1, 2, 3}); + monitor->tick(major_tick_60s); + monitor->tick(major_tick_60s); + + auto avg = monitor->getCpuBreakdownAvg( + CPUTimeMonitor::Granularity::MINUTE, + 60, + CPUTimeMonitor::CpuBreakdown::NICE, + "target_a"); + ASSERT_TRUE(avg.has_value()); + EXPECT_DOUBLE_EQ(avg.value(), 0.0); + + monitor->deRegisterTarget("target_a"); + + EXPECT_EQ( + monitor->getCpuBreakdownAvg( + CPUTimeMonitor::Granularity::MINUTE, + 60, + CPUTimeMonitor::CpuBreakdown::NICE, + "target_a"), + std::nullopt); +} + +TEST_F(CPUTimeMonitorTest, testBreakdownMinMax) { + monitor->registerTarget("minmax_test", {0, 1, 2, 3}); + monitor->tick(major_tick_60s); + monitor->tick(major_tick_60s); + + for (const auto& bd : + {CPUTimeMonitor::CpuBreakdown::IDLE, + CPUTimeMonitor::CpuBreakdown::SOFTIRQ, + CPUTimeMonitor::CpuBreakdown::IOWAIT, + CPUTimeMonitor::CpuBreakdown::HARDIRQ, + CPUTimeMonitor::CpuBreakdown::NICE}) { + auto minVal = monitor->getCpuBreakdownMin( + CPUTimeMonitor::Granularity::MINUTE, 60, bd, "minmax_test"); + auto maxVal = monitor->getCpuBreakdownMax( + CPUTimeMonitor::Granularity::MINUTE, 60, bd, "minmax_test"); + ASSERT_TRUE(minVal.has_value()); + ASSERT_TRUE(maxVal.has_value()); + EXPECT_LE(minVal.value(), maxVal.value()); + } + + monitor->deRegisterTarget("minmax_test"); +} + +TEST_F(CPUTimeMonitorTest, testBreakdownMinMaxCombined) { + monitor->registerTarget("combined_test", {0, 1, 2, 3}); + monitor->tick(major_tick_60s); + monitor->tick(major_tick_60s); + + for (const auto& bd : + {CPUTimeMonitor::CpuBreakdown::IDLE, + CPUTimeMonitor::CpuBreakdown::SOFTIRQ, + CPUTimeMonitor::CpuBreakdown::IOWAIT, + CPUTimeMonitor::CpuBreakdown::HARDIRQ, + CPUTimeMonitor::CpuBreakdown::NICE}) { + auto minMax = monitor->getCpuBreakdownMinMax( + CPUTimeMonitor::Granularity::MINUTE, 60, bd, "combined_test"); + auto minVal = monitor->getCpuBreakdownMin( + CPUTimeMonitor::Granularity::MINUTE, 60, bd, "combined_test"); + auto maxVal = monitor->getCpuBreakdownMax( + CPUTimeMonitor::Granularity::MINUTE, 60, bd, "combined_test"); + + ASSERT_TRUE(minMax.has_value()); + ASSERT_TRUE(minVal.has_value()); + ASSERT_TRUE(maxVal.has_value()); + EXPECT_DOUBLE_EQ(minMax->first, minVal.value()); + EXPECT_DOUBLE_EQ(minMax->second, maxVal.value()); + EXPECT_LE(minMax->first, minMax->second); + } + + monitor->deRegisterTarget("combined_test"); +} + +TEST_F(CPUTimeMonitorTest, testMinMaxCPUCoresUsage) { + monitor->registerTarget("cores_test", {0, 1, 2, 3}); + monitor->tick(major_tick_60s); + monitor->tick(major_tick_60s); + + for (const auto& gran : + {CPUTimeMonitor::Granularity::MINUTE, + CPUTimeMonitor::Granularity::SECOND}) { + auto minVal = monitor->getMinCPUCoresUsage( + gran, 60, "cores_test", CPUTimeMonitor::DataSource::PROC_STAT); + auto maxVal = monitor->getMaxCPUCoresUsage( + gran, 60, "cores_test", CPUTimeMonitor::DataSource::PROC_STAT); + auto avgVal = monitor->getAvgCPUCoresUsage( + gran, 60, "cores_test", CPUTimeMonitor::DataSource::PROC_STAT); + + ASSERT_TRUE(minVal.has_value()); + ASSERT_TRUE(maxVal.has_value()); + ASSERT_TRUE(avgVal.has_value()); + EXPECT_LE(minVal.value(), avgVal.value()); + EXPECT_LE(avgVal.value(), maxVal.value()); + } + + monitor->deRegisterTarget("cores_test"); +} + +TEST(CPUTimeMonitorFallbackTest, testMinMaxCgroupStatFallback) { + // readCgroupStat=false: CGROUP_STAT requests fall back to PROC_STAT data + auto ticker = std::make_shared(); + auto mon = std::make_shared( + ticker, false, coreCount, getenv("TESTROOT"), true); + mon->tick(major_tick_60s); + mon->tick(major_tick_60s); + + auto minProc = mon->getMinCPUCoresUsage( + CPUTimeMonitor::Granularity::MINUTE, + 60, + std::nullopt, + CPUTimeMonitor::DataSource::PROC_STAT); + auto minCgroup = mon->getMinCPUCoresUsage( + CPUTimeMonitor::Granularity::MINUTE, + 60, + std::nullopt, + CPUTimeMonitor::DataSource::CGROUP_STAT); + + ASSERT_TRUE(minProc.has_value()); + ASSERT_TRUE(minCgroup.has_value()); + EXPECT_DOUBLE_EQ(minProc.value(), minCgroup.value()); +} + +TEST_F(CPUTimeMonitorTest, testBreakdownMinMaxNulloptAfterDeregister) { + monitor->registerTarget("dereg_test", {0, 1}); + monitor->tick(major_tick_60s); + monitor->tick(major_tick_60s); + + auto minBefore = monitor->getCpuBreakdownMin( + CPUTimeMonitor::Granularity::MINUTE, + 60, + CPUTimeMonitor::CpuBreakdown::NICE, + "dereg_test"); + ASSERT_TRUE(minBefore.has_value()); + + monitor->deRegisterTarget("dereg_test"); + + auto minAfter = monitor->getCpuBreakdownMin( + CPUTimeMonitor::Granularity::MINUTE, + 60, + CPUTimeMonitor::CpuBreakdown::NICE, + "dereg_test"); + EXPECT_EQ(minAfter, std::nullopt); +} + } // namespace dynolog } // namespace facebook