Introduce Checkable#scheduler_shuffle_cap

Author: Al2Klimov

lib/compat/checkresultreader.cpp

@@ -162,5 +162,5 @@ void CheckResultReader::ProcessCheckResultFile(const String& path) const
 	/* Reschedule the next check. The side effect of this is that for as long
 	 * as we receive check result files for a host/service we won't execute any
 	 * active checks. */
-	checkable->SetNextCheck(Utility::GetTime() + checkable->GetCheckInterval());
+	checkable->SetNextCheck(Utility::GetTime() + checkable->GetCheckInterval() * checkable->GetIntervalShuffleFactor());
 }

lib/icinga/checkable-check.cpp

@@ -14,6 +14,7 @@
 #include "base/convert.hpp"
 #include "base/utility.hpp"
 #include "base/context.hpp"
+#include <cstdlib>
 
 using namespace icinga;
 
@@ -67,7 +68,7 @@ void Checkable::UpdateNextCheck(const MessageOrigin::Ptr& origin)
 	if (adj != 0.0)
 		adj = std::min(0.5 + fmod(GetSchedulingOffset(), interval * 5) / 100.0, adj);
 
-	double nextCheck = now - adj + interval;
+	double nextCheck = now - adj + interval * GetIntervalShuffleFactor();
 	double lastCheck = GetLastCheck();
 
 	Log(LogDebug, "Checkable")
@@ -278,7 +279,7 @@ void Checkable::ProcessCheckResult(const CheckResult::Ptr& cr, const MessageOrig
 			if (!parent->GetEnableActiveChecks())
 				continue;
 
-			if (parent->GetNextCheck() >= now + parent->GetRetryInterval()) {
+			if (parent->GetNextCheck() >= now + parent->GetRetryInterval() * parent->GetIntervalShuffleFactor()) {
 				ObjectLock olock(parent);
 				parent->SetNextCheck(now);
 			}
@@ -382,7 +383,7 @@ void Checkable::ProcessCheckResult(const CheckResult::Ptr& cr, const MessageOrig
 		if (ttl > 0)
 			offset = ttl;
 		else
-			offset = GetCheckInterval();
+			offset = GetCheckInterval() * GetIntervalShuffleFactor();
 
 		SetNextCheck(Utility::GetTime() + offset, false, origin);
 	}
@@ -667,3 +668,20 @@ void Checkable::AquirePendingCheckSlot(int maxPendingChecks)
 
 	m_PendingChecks++;
 }
+
+/**
+ * Returns a random factor derived from scheduler_shuffle_cap to multiply the check interval with.
+ *
+ * E.g. if scheduler_shuffle_cap is 20 (%), this function returns [0.8, 1.2].
+ */
+double Checkable::GetIntervalShuffleFactor()
+{
+	if (!GetEnableActiveChecks()) {
+		// scheduler_shuffle_cap doesn't influence external checkers.
+		return 1;
+	}
+
+	return (GetSchedulerShuffleCap() / 100) // scheduler_shuffle_cap as non-%, i.e. 10 => 0.1
+		* (rand() / (double)RAND_MAX * 2 - 1) // random number [-1, 1]
+		+ 1;
+}

lib/icinga/checkable.cpp

@@ -9,6 +9,7 @@
 #include "base/exception.hpp"
 #include "base/timer.hpp"
 #include <boost/thread/once.hpp>
+#include <cmath>
 
 using namespace icinga;
 
@@ -72,9 +73,9 @@ void Checkable::Start(bool runtimeCreated)
 	}
 
 	if (GetNextCheck() < now + 60) {
-		double delta = std::min(GetCheckInterval(), 60.0);
+		double delta = std::min(GetCheckInterval() * GetIntervalShuffleFactor(), 60.0);
 		delta *= (double)std::rand() / RAND_MAX;
-		SetNextCheck(now + delta);
+		SetNextCheck(now + delta + GetCheckInterval() * fabs(GetIntervalShuffleFactor() - 1));
 	}
 
 	ObjectImpl<Checkable>::Start(runtimeCreated);

lib/icinga/checkable.hpp

@@ -173,6 +173,7 @@ class Checkable : public ObjectImpl<Checkable>
 
 	bool NotificationReasonApplies(NotificationType type);
 	bool IsLikelyToBeCheckedSoon();
+	double GetIntervalShuffleFactor();
 
 	static void IncreasePendingChecks();
 	static void DecreasePendingChecks();

lib/icinga/checkable.ti

@@ -47,6 +47,9 @@ abstract class Checkable : CustomVarObject
 	[config] double retry_interval {
 		default {{{ return 60; }}}
 	};
+	[config] double scheduler_shuffle_cap {
+		default {{{ return 0.0; }}}
+	};
 	[config, navigation] name(EventCommand) event_command (EventCommandRaw) {
 		navigate {{{
 			return EventCommand::GetByName(GetEventCommandRaw());