bp/Control: randomize the EXPIRE_TCACHE_TAG delay

This completes commit 3739088

Author: MaxKellermann

src/bp/Control.cxx

@@ -25,6 +25,7 @@
 #include "pool/pool.hxx"
 #include "net/SocketAddress.hxx"
 #include "io/Logger.hxx"
+#include "util/djb_hash.hxx"
 #include "util/SpanCast.hxx"
 #include "util/UnalignedBigEndian.hxx"
 #include "AllocatorPtr.hxx"
@@ -68,7 +69,8 @@ BpInstance::HandleTcacheInvalidate(std::span<const std::byte> payload) noexcept
 }
 
 inline void
-BpInstance::OnExpireTcacheRA(const ResourceAddress &address) noexcept
+BpInstance::OnExpireTcacheRA(const ResourceAddress &address,
+			     Event::TimePoint time) noexcept
 {
 	switch (address.type) {
 	case ResourceAddress::Type::NONE:
@@ -85,8 +87,7 @@ BpInstance::OnExpireTcacheRA(const ResourceAddress &address) noexcept
 			const TempPoolLease tpool;
 			const auto key = lhttp.GetChildId(*tpool);
 
-			// TODO implement randomized delay
-			lhttp_stock->FadeKey(key);
+			lhttp_stock->ExpireKey(key, time);
 		}
 
 		break;
@@ -97,8 +98,7 @@ BpInstance::OnExpireTcacheRA(const ResourceAddress &address) noexcept
 				const TempPoolLease tpool;
 				const auto key = cgi.GetChildId(*tpool);
 
-				// TODO implement randomized delay
-				fcgi_stock->FadeKey(key);
+				fcgi_stock->ExpireKey(key, time);
 			}
 		}
 
@@ -111,16 +111,14 @@ BpInstance::OnExpireTcacheRA(const ResourceAddress &address) noexcept
 				const TempPoolLease tpool;
 				const auto key = cgi.GetChildId(*tpool);
 
-				// TODO implement randomized delay
-				was_stock->FadeKey(key);
+				was_stock->ExpireKey(key, time);
 			}
 		} else if (cgi.address_list.empty()) {
 			if (multi_was_stock) {
 				const TempPoolLease tpool;
 				const auto key = cgi.GetChildId(*tpool);
 
-				// TODO implement randomized delay
-				multi_was_stock->FadeKey(key);
+				multi_was_stock->ExpireKey(key, time);
 			}
 		}
 #endif // HAVE_LIBWAS
@@ -129,12 +127,12 @@ BpInstance::OnExpireTcacheRA(const ResourceAddress &address) noexcept
 }
 
 inline void
-BpInstance::OnExpireTcache(const TranslateResponse &response) noexcept
+BpInstance::OnExpireTcache(const TranslateResponse &response, Event::TimePoint time) noexcept
 {
-	OnExpireTcacheRA(response.address);
+	OnExpireTcacheRA(response.address, time);
 
 	for (const auto &view : response.views) {
-		OnExpireTcacheRA(view.address);
+		OnExpireTcacheRA(view.address, time);
 
 		for (const auto &transformation : view.transformations) {
 			switch (transformation.type) {
@@ -144,7 +142,7 @@ BpInstance::OnExpireTcache(const TranslateResponse &response) noexcept
 				break;
 
 			case Transformation::Type::FILTER:
-				OnExpireTcacheRA(transformation.u.filter.address);
+				OnExpireTcacheRA(transformation.u.filter.address, time);
 				break;
 			}
 		}

src/bp/Instance.hxx

@@ -281,8 +281,10 @@ struct BpInstance final : PInstance, BengControl::Handler,
 	void HandleExpireTcacheTag(std::span<const std::byte> payload) noexcept;
 	void HandleDisableUring(std::span<const std::byte> payload) noexcept;
 
-	void OnExpireTcacheRA(const ResourceAddress &address) noexcept;
-	void OnExpireTcache(const TranslateResponse &response) noexcept;
+	void OnExpireTcacheRA(const ResourceAddress &address,
+			      Event::TimePoint time) noexcept;
+	void OnExpireTcache(const TranslateResponse &response,
+			    Event::TimePoint time) noexcept;
 
 	/* virtual methods from class BengControl::Handler */
 	void OnControlPacket(BengControl::Command command,

src/translation/Builder.hxx

@@ -4,6 +4,7 @@
 
 #pragma once
 
+#include "event/Chrono.hxx"
 #include "util/BindMethod.hxx"
 
 #include <cstdint>
@@ -56,7 +57,8 @@ class TranslationCacheBuilder final : public TranslationServiceBuilder {
 	std::map<SocketAddress, std::shared_ptr<TranslationCache>,
 		 SocketAddressCompare> m;
 
-	using ExpireCallback = BoundMethod<void(const TranslateResponse &response) noexcept>;
+	using ExpireCallback = BoundMethod<void(const TranslateResponse &response,
+						Event::TimePoint time) noexcept>;
 
 public:
 	TranslationCacheBuilder(TranslationStockBuilder &_builder,

src/translation/Cache.cxx

@@ -26,6 +26,7 @@
 #include "io/Logger.hxx"
 #include "util/djb_hash.hxx"
 #include "util/IntrusiveForwardList.hxx"
+#include "util/RoundPowerOfTwo.hxx"
 #include "util/SpanCast.hxx"
 #include "util/StringAPI.hxx"
 #include "util/StringSplit.hxx"
@@ -886,13 +887,60 @@ TranslationCache::Invalidate(const TranslateRequest &request,
 	LogConcat(4, "TranslationCache", "invalidated ", removed, " cache items");
 }
 
+/**
+ * Overload for RoundUpToPowerOfTwo() (from
+ * `util/RoundPowerOfTwo.hxx`) which takes/returns an Event::Duration.
+ */
+static constexpr Event::Duration
+RoundUpToPowerOfTwo(Event::Duration value) noexcept
+{
+	using rep = Event::Duration::rep;
+	using unsigned_rep = std::make_unsigned_t<rep>;
+
+	const auto scalar = RoundUpToPowerOfTwo(static_cast<unsigned_rep>(value.count()));
+	return Event::Duration{static_cast<rep>(scalar)};
+}
+
+/**
+ * Calculate an EXPIRE_TCACHE_TAG delay for the specified SITE string.
+ * All translation cache items and child processes for this site shall
+ * expire at the same time, but each site should get a different
+ * expiry.
+ */
+static constexpr Event::Duration
+CalculateExpireDelayForSite(const char *site) noexcept
+{
+	if (site == nullptr)
+		return {};
+
+	/**
+	 * An approximation for the maximum delay, the range over
+	 * which all sites are going to be distributed.
+	 */
+	static constexpr Event::Duration APPROXIMATE_MAX_DELAY = std::chrono::minutes{5};
+
+	/**
+	 * This is the real maximum delay, calculated as the next
+	 * power of two, which can then be used to derive a bit mask
+	 * to be applied to the site hash with bit-wise "and".
+	 */
+	static constexpr Event::Duration MAX_DELAY = RoundUpToPowerOfTwo(APPROXIMATE_MAX_DELAY);
+	static constexpr std::size_t MASK = MAX_DELAY.count() - 1;
+
+	const std::size_t hash = djb_hash_string(site);
+	return Event::Duration(hash & MASK);
+}
+
 void
 TranslationCache::ExpireTag(std::string_view tag, ExpireCallback callback) noexcept
 {
-	per_tag.remove_and_dispose_key(tag, [this, callback](TranslateCacheItemTag *item_tag){
-		auto &item = item_tag->parent;
-		callback(item.response);
-		cache.Remove(item);
+	const auto now = cache.SteadyNow();
+	per_tag.for_each_key(tag, [callback, now](TranslateCacheItemTag &item_tag){
+		auto &item = item_tag.parent;
+		const auto delay = CalculateExpireDelayForSite(item.response.site);
+		const auto expires = now + delay;
+		item.SetExpires(expires);
+		callback(item.response, expires);
 	});
 }
 

src/translation/Cache.hxx

@@ -9,6 +9,7 @@
 #include "cache/Handler.hxx"
 #include "pool/Ptr.hxx"
 #include "stats/CacheStats.hxx"
+#include "event/Chrono.hxx"
 #include "memory/SlicePool.hxx"
 #include "util/BindMethod.hxx"
 #include "util/IntrusiveHashSet.hxx"
@@ -112,7 +113,13 @@ class TranslationCache final : public TranslationService, CacheHandler {
 	 */
 	bool active;
 
-	using ExpireCallback = BoundMethod<void(const TranslateResponse &response) noexcept>;
+	/**
+	 * @param time the time point when resources (e.g. child
+	 * processes) described by this #TranslateResponse shall
+	 * expire
+	 */
+	using ExpireCallback = BoundMethod<void(const TranslateResponse &response,
+						Event::TimePoint time) noexcept>;
 
 public:
 	/**