Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions pdns/dnsdistdist/Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ EXTRA_DIST=COPYING \
kqueuemplexer.cc \
portsmplexer.cc \
cdb.cc cdb.hh \
redis.cc redis.hh \
standalone_fuzz_target_runner.cc \
ext/lmdb-safe/lmdb-safe.cc ext/lmdb-safe/lmdb-safe.hh \
ext/protozero/include/* \
Expand Down Expand Up @@ -298,6 +299,7 @@ dnsdist_SOURCES = \
protozero.cc protozero.hh \
proxy-protocol.cc proxy-protocol.hh \
qtype.cc qtype.hh \
redis.hh \
remote_logger.cc remote_logger.hh \
remote_logger_pool.cc remote_logger_pool.hh \
sanitizer.hh \
Expand Down Expand Up @@ -402,6 +404,7 @@ testrunner_SOURCES = \
protozero.cc protozero.hh \
proxy-protocol.cc proxy-protocol.hh \
qtype.cc qtype.hh \
redis.hh \
sholder.hh \
sstuff.hh \
stat_t.hh \
Expand Down Expand Up @@ -475,6 +478,7 @@ testrunner_LDFLAGS = \
testrunner_LDADD = \
$(BOOST_UNIT_TEST_FRAMEWORK_LIBS) \
$(FSTRM_LIBS) \
$(YAHTTP_LIBS) \
$(LIBSODIUM_LIBS) \
$(LUA_LIBS) \
$(RT_LIBS) \
Expand All @@ -489,6 +493,13 @@ dnsdist_SOURCES += cdb.cc cdb.hh
testrunner_SOURCES += cdb.cc cdb.hh
endif

if HAVE_REDIS
dnsdist_LDADD += $(REDIS_LDFLAGS) $(REDIS_LIBS)
testrunner_LDADD += $(REDIS_LDFLAGS) $(REDIS_LIBS)
dnsdist_SOURCES += redis.cc
testrunner_SOURCES += redis.cc
endif

if HAVE_RE2
dnsdist_LDADD += $(RE2_LIBS)
testrunner_LDADD += $(RE2_LIBS)
Expand Down Expand Up @@ -664,6 +675,7 @@ fuzz_target_dnsdistcache_SOURCES = \
packetcache.hh \
protozero-trace.cc protozero-trace.hh \
qtype.cc qtype.hh \
redis.hh \
svc-records.cc svc-records.hh

fuzz_target_dnsdistcache_DEPENDENCIES = $(fuzz_targets_deps)
Expand Down
6 changes: 6 additions & 0 deletions pdns/dnsdistdist/configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ AS_IF([test "x$LUAPC" = "xluajit"], [
PDNS_CHECK_LUA_HPP

AM_CONDITIONAL([HAVE_CDB], [false])
AM_CONDITIONAL([HAVE_REDIS], [false])
AM_CONDITIONAL([HAVE_GNUTLS], [false])
AM_CONDITIONAL([HAVE_LIBSSL], [false])
AM_CONDITIONAL([HAVE_LMDB], [false])
Expand Down Expand Up @@ -141,6 +142,7 @@ AS_IF([test "x$enable_yaml" != "xno"], [
])

DNSDIST_WITH_CDB
DNSDIST_WITH_REDIS
PDNS_CHECK_LMDB
PDNS_ENABLE_IPCIPHER
PDNS_ENABLE_IPCRYPT2
Expand Down Expand Up @@ -317,6 +319,10 @@ AS_IF([test "x$LMDB_LIBS" != "x"],
[AC_MSG_NOTICE([lmdb: yes])],
[AC_MSG_NOTICE([lmdb: no])]
)
AS_IF([test "x$REDIS_LIBS" != "x"],
[AC_MSG_NOTICE([redis: yes])],
[AC_MSG_NOTICE([redis: no])]
)
AS_IF([test "x$enable_yaml" != "xno"],
[AC_MSG_NOTICE([YAML configuration: yes])],
[AC_MSG_NOTICE([YAML configuration: no])]
Expand Down
58 changes: 51 additions & 7 deletions pdns/dnsdistdist/dnsdist-configuration-yaml.cc
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include <vector>

#include "dnsdist-configuration-yaml.hh"
#include "dnsdist-rust-bridge.hh"

#if defined(HAVE_YAML_CONFIGURATION)
#include "base64.hh"
Expand All @@ -43,6 +44,7 @@
#include "dnsdist-xsk.hh"
#include "fstrm_logger.hh"
#include "iputils.hh"
#include "redis.hh"
#include "remote_logger.hh"
#include "remote_logger_pool.hh"
#include "xsk.hh"
Expand All @@ -68,7 +70,7 @@ struct Context

using XSKMap = std::vector<std::shared_ptr<XskSocket>>;

using RegisteredTypes = std::variant<std::shared_ptr<DNSDistPacketCache>, std::shared_ptr<dnsdist::rust::settings::DNSSelector>, std::shared_ptr<dnsdist::rust::settings::DNSActionWrapper>, std::shared_ptr<dnsdist::rust::settings::DNSResponseActionWrapper>, std::shared_ptr<NetmaskGroup>, std::shared_ptr<KeyValueStore>, std::shared_ptr<KeyValueLookupKey>, std::shared_ptr<RemoteLoggerInterface>, std::shared_ptr<ServerPolicy>, std::shared_ptr<TimedIPSetRule>, std::shared_ptr<XSKMap>>;
using RegisteredTypes = std::variant<std::shared_ptr<DNSDistPacketCache>, std::shared_ptr<dnsdist::rust::settings::DNSSelector>, std::shared_ptr<dnsdist::rust::settings::DNSActionWrapper>, std::shared_ptr<dnsdist::rust::settings::DNSResponseActionWrapper>, std::shared_ptr<NetmaskGroup>, std::shared_ptr<KeyValueStore>, std::shared_ptr<KeyValueLookupKey>, std::shared_ptr<RemoteLoggerInterface>, std::shared_ptr<ServerPolicy>, std::shared_ptr<TimedIPSetRule>, std::shared_ptr<XSKMap>, std::shared_ptr<RedisClient>>;
static LockGuarded<std::unordered_map<std::string, RegisteredTypes>> s_registeredTypesMap;
static std::atomic<bool> s_inConfigCheckMode;
static std::atomic<bool> s_inClientMode;
Expand Down Expand Up @@ -1357,7 +1359,7 @@ bool loadConfigurationFromFile(const std::string& fileName, [[maybe_unused]] boo
void addLuaBindingsForYAMLObjects([[maybe_unused]] LuaContext& luaCtx)
{
#if defined(HAVE_YAML_CONFIGURATION)
using ReturnValue = std::optional<boost::variant<std::shared_ptr<DNSDistPacketCache>, std::shared_ptr<DNSRule>, std::shared_ptr<DNSAction>, std::shared_ptr<DNSResponseAction>, std::shared_ptr<NetmaskGroup>, std::shared_ptr<KeyValueStore>, std::shared_ptr<KeyValueLookupKey>, std::shared_ptr<RemoteLoggerInterface>, std::shared_ptr<ServerPolicy>, std::shared_ptr<TimedIPSetRule>, std::shared_ptr<XSKMap>>>;
using ReturnValue = std::optional<boost::variant<std::shared_ptr<DNSDistPacketCache>, std::shared_ptr<DNSRule>, std::shared_ptr<DNSAction>, std::shared_ptr<DNSResponseAction>, std::shared_ptr<NetmaskGroup>, std::shared_ptr<KeyValueStore>, std::shared_ptr<KeyValueLookupKey>, std::shared_ptr<RemoteLoggerInterface>, std::shared_ptr<ServerPolicy>, std::shared_ptr<TimedIPSetRule>, std::shared_ptr<XSKMap>, std::shared_ptr<RedisClient>>>;

luaCtx.writeFunction("getObjectFromYAMLConfiguration", [](const std::string& name) -> ReturnValue {
auto map = s_registeredTypesMap.lock();
Expand Down Expand Up @@ -1398,6 +1400,9 @@ void addLuaBindingsForYAMLObjects([[maybe_unused]] LuaContext& luaCtx)
if (auto* ptr = std::get_if<std::shared_ptr<XSKMap>>(&item->second)) {
return ReturnValue(*ptr);
}
if (auto* ptr = std::get_if<std::shared_ptr<RedisClient>>(&item->second)) {
return ReturnValue(*ptr);
}

return std::nullopt;
});
Expand Down Expand Up @@ -1661,7 +1666,7 @@ std::shared_ptr<DNSSelector> getNetmaskGroupSelector(const NetmaskGroupSelectorC

std::shared_ptr<DNSActionWrapper> getKeyValueStoreLookupAction([[maybe_unused]] const KeyValueStoreLookupActionConfiguration& config)
{
#if defined(HAVE_LMDB) || defined(HAVE_CDB)
#if defined(HAVE_LMDB) || defined(HAVE_CDB) || defined(HAVE_REDIS)
auto kvs = dnsdist::configuration::yaml::getRegisteredTypeByName<KeyValueStore>(std::string(config.kvs_name));
if (!kvs && !(dnsdist::configuration::yaml::s_inClientMode || dnsdist::configuration::yaml::s_inConfigCheckMode)) {
throw std::runtime_error("Unable to find the key-value store named '" + std::string(config.kvs_name) + "'");
Expand All @@ -1679,7 +1684,7 @@ std::shared_ptr<DNSActionWrapper> getKeyValueStoreLookupAction([[maybe_unused]]

std::shared_ptr<DNSActionWrapper> getKeyValueStoreRangeLookupAction([[maybe_unused]] const KeyValueStoreRangeLookupActionConfiguration& config)
{
#if defined(HAVE_LMDB) || defined(HAVE_CDB)
#if defined(HAVE_LMDB) || defined(HAVE_CDB) || defined(HAVE_REDIS)
auto kvs = dnsdist::configuration::yaml::getRegisteredTypeByName<KeyValueStore>(std::string(config.kvs_name));
if (!kvs && !(dnsdist::configuration::yaml::s_inClientMode || dnsdist::configuration::yaml::s_inConfigCheckMode)) {
throw std::runtime_error("Unable to find the key-value store named '" + std::string(config.kvs_name) + "'");
Expand All @@ -1697,7 +1702,7 @@ std::shared_ptr<DNSActionWrapper> getKeyValueStoreRangeLookupAction([[maybe_unus

std::shared_ptr<DNSSelector> getKeyValueStoreLookupSelector([[maybe_unused]] const KeyValueStoreLookupSelectorConfiguration& config)
{
#if defined(HAVE_LMDB) || defined(HAVE_CDB)
#if defined(HAVE_LMDB) || defined(HAVE_CDB) || defined(HAVE_REDIS)
auto kvs = dnsdist::configuration::yaml::getRegisteredTypeByName<KeyValueStore>(std::string(config.kvs_name));
if (!kvs && !(dnsdist::configuration::yaml::s_inClientMode || dnsdist::configuration::yaml::s_inConfigCheckMode)) {
throw std::runtime_error("Unable to find the key-value store named '" + std::string(config.kvs_name) + "'");
Expand All @@ -1715,7 +1720,7 @@ std::shared_ptr<DNSSelector> getKeyValueStoreLookupSelector([[maybe_unused]] con

std::shared_ptr<DNSSelector> getKeyValueStoreRangeLookupSelector([[maybe_unused]] const KeyValueStoreRangeLookupSelectorConfiguration& config)
{
#if defined(HAVE_LMDB) || defined(HAVE_CDB)
#if defined(HAVE_LMDB) || defined(HAVE_CDB) || defined(HAVE_REDIS)
auto kvs = dnsdist::configuration::yaml::getRegisteredTypeByName<KeyValueStore>(std::string(config.kvs_name));
if (!kvs && !(dnsdist::configuration::yaml::s_inClientMode || dnsdist::configuration::yaml::s_inConfigCheckMode)) {
throw std::runtime_error("Unable to find the key-value store named '" + std::string(config.kvs_name) + "'");
Expand Down Expand Up @@ -1968,7 +1973,7 @@ void registerDnstapLogger([[maybe_unused]] const DnstapLoggerConfiguration& conf

void registerKVSObjects([[maybe_unused]] const KeyValueStoresConfiguration& config)
{
#if defined(HAVE_LMDB) || defined(HAVE_CDB)
#if defined(HAVE_LMDB) || defined(HAVE_CDB) || defined(HAVE_REDIS)
bool createObjects = !dnsdist::configuration::yaml::s_inClientMode && !dnsdist::configuration::yaml::s_inConfigCheckMode;
#if defined(HAVE_LMDB)
for (const auto& lmdb : config.lmdb) {
Expand All @@ -1982,6 +1987,36 @@ void registerKVSObjects([[maybe_unused]] const KeyValueStoresConfiguration& conf
dnsdist::configuration::yaml::registerType<KeyValueStore>(store, cdb.name);
}
#endif /* defined(HAVE_CDB) */
#if defined(HAVE_REDIS)
for (const auto& redis : config.redis) {
auto definedRedis = dnsdist::configuration::yaml::getRegisteredTypeByName<RedisClient>(redis.redis_client);
if (!definedRedis) {
throw std::runtime_error("Unable to find a Redis client named " + std::string(redis.redis_client));
}
std::optional<std::string> lookupAction;
std::optional<std::string> dataName;

if (!redis.lookup_action.empty()) {
lookupAction = std::string(redis.lookup_action);
}
if (!redis.data_name.empty()) {
dataName = std::string(redis.data_name);
}

std::string uniqueId = "url=" + definedRedis->getUrl().to_string() + ",action=" + lookupAction.value_or("GET") + ",data-name=" + dataName.value_or("") + ",";
std::string labels = "redis-server=" + definedRedis->getUrl().host + ":" + std::to_string(definedRedis->getUrl().port) + ",redis-action=" + lookupAction.value_or("GET") + ",data-name=" + dataName.value_or("");
std::shared_ptr<RedisStats> stats = std::make_shared<RedisStats>(labels);

dnsdist::configuration::updateRuntimeConfiguration([uniqueId, &stats](dnsdist::configuration::RuntimeConfiguration& runtimeConfig) {
if (runtimeConfig.d_redisStats.count(uniqueId) > 0) {
throw std::runtime_error("Duplicate redis instance. Combination of arguments has to be unique!");
}
runtimeConfig.d_redisStats.emplace(uniqueId, std::shared_ptr(stats));
});
auto store = createObjects ? std::shared_ptr<KeyValueStore>(std::make_shared<RedisKVStore>(definedRedis, lookupAction, dataName, stats)) : std::shared_ptr<KeyValueStore>();
dnsdist::configuration::yaml::registerType<KeyValueStore>(store, redis.name);
}
#endif /* defined(HAVE_REDIS) */
for (const auto& key : config.lookup_keys.source_ip_keys) {
auto lookup = createObjects ? std::shared_ptr<KeyValueLookupKey>(std::make_shared<KeyValueLookupKeySourceIP>(key.v4_mask, key.v6_mask, key.include_port)) : std::shared_ptr<KeyValueLookupKey>();
dnsdist::configuration::yaml::registerType<KeyValueLookupKey>(lookup, key.name);
Expand Down Expand Up @@ -2020,6 +2055,15 @@ void registerNMGObjects(const ::rust::Vec<NetmaskGroupConfiguration>& nmgs)
}
}

void registerRedisClientObjects([[maybe_unused]] const ::rust::Vec<RedisClientConfiguration>& redisClients)
{
#ifdef HAVE_REDIS
for (const auto& redisClient : redisClients) {
dnsdist::configuration::yaml::registerType<RedisClient>(std::make_shared<RedisClient>(std::string(redisClient.url)), redisClient.name);
}
#endif
}

void registerTimedIPSetObjects(const ::rust::Vec<TimedIpSetConfiguration>& sets)
{
for (const auto& timedIPSet : sets) {
Expand Down
4 changes: 4 additions & 0 deletions pdns/dnsdistdist/dnsdist-configuration.hh
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
#include "dnsdist-rule-chains.hh"
#include "dnsdist-server-pool.hh"
#include "iputils.hh"
#include "redis-stats.hh"

class ServerPolicy;
struct ServerPool;
Expand Down Expand Up @@ -131,6 +132,9 @@ struct RuntimeConfiguration
std::vector<dnsdist::Carbon::Endpoint> d_carbonEndpoints;
#endif /* DISABLE_CARBON */
std::unordered_map<std::string, ServerPool> d_pools;
#ifdef HAVE_REDIS
std::unordered_map<std::string, std::shared_ptr<RedisStats>> d_redisStats;
#endif /* HAVE_REDIS */
std::shared_ptr<const CredentialsHolder> d_webPassword;
std::shared_ptr<const CredentialsHolder> d_webAPIKey;
std::optional<std::unordered_map<std::string, std::string>> d_webCustomHeaders;
Expand Down
59 changes: 59 additions & 0 deletions pdns/dnsdistdist/dnsdist-kvs.cc
Original file line number Diff line number Diff line change
Expand Up @@ -302,3 +302,62 @@ bool CDBKVStore::keyExists(const std::string& key)
}

#endif /* HAVE_CDB */

#ifdef HAVE_REDIS

RedisKVStore::RedisKVStore(const std::shared_ptr<RedisClient>& redisClient, std::optional<std::string> lookupAction, std::optional<std::string> dataName, std::shared_ptr<RedisStats> stats) :
d_stats(stats)
{
std::unique_ptr<RedisLookupAction> command;
if (lookupAction && !boost::iequals(lookupAction.value(), "get")) {
if (!dataName) {
throw std::runtime_error("Option 'dataName' is required for lookup action " + lookupAction.value());
}
if (boost::iequals(lookupAction.value(), "hget")) {
command = std::make_unique<RedisHGetLookupAction>(dataName.value());
}
else {
throw std::runtime_error("Unknown lookup action: " + lookupAction.value());
}
}
else {
if (dataName) {
command = std::make_unique<RedisGetLookupAction>(dataName.value());
}
else {
command = std::make_unique<RedisGetLookupAction>();
}
}
d_redis = std::make_unique<RedisKVClient>(redisClient, std::move(command), stats);
}

bool RedisKVStore::reload()
{
return true;
}

bool RedisKVStore::getValue(const std::string& key, std::string& value)
{
auto result = d_redis->getValue(key, value);
if (result) {
d_stats->d_successfulLookups += 1;
}
else {
d_stats->d_failedLookups += 1;
}
return result;
}

bool RedisKVStore::keyExists(const std::string& key)
{
auto result = d_redis->keyExists(key);
if (result) {
d_stats->d_successfulLookups += 1;
}
else {
d_stats->d_failedLookups += 1;
}
return result;
}

#endif // HAVE_REDIS
20 changes: 20 additions & 0 deletions pdns/dnsdistdist/dnsdist-kvs.hh
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include <memory>
#include "dnsdist.hh"
#include "logr.hh"
#include "iputils.hh"

class KeyValueLookupKey
{
Expand Down Expand Up @@ -231,3 +232,22 @@ private:
};

#endif /* HAVE_LMDB */

#ifdef HAVE_REDIS

#include "redis.hh"

class RedisKVStore : public KeyValueStore
{
public:
RedisKVStore(const std::shared_ptr<RedisClient>& redisClient, std::optional<std::string> lookupAction, std::optional<std::string> dataName, std::shared_ptr<RedisStats> stats);

bool keyExists(const std::string& key) override;
bool getValue(const std::string& key, std::string& value) override;
bool reload() override;

private:
std::unique_ptr<RedisKVClientInterface> d_redis{nullptr};
std::shared_ptr<RedisStats> d_stats;
};
#endif // HAVE_REDIS
32 changes: 30 additions & 2 deletions pdns/dnsdistdist/dnsdist-lua-bindings-kvs.cc
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,35 @@ void setupLuaBindingsKVS([[maybe_unused]] LuaContext& luaCtx, [[maybe_unused]] b
});
#endif /* HAVE_CDB */

#if defined(HAVE_LMDB) || defined(HAVE_CDB)
#ifdef HAVE_REDIS
luaCtx.writeFunction("newRedisKVStore", [client](const std::shared_ptr<RedisClient>& redisClient, std::optional<LuaAssociativeTable<boost::variant<std::string, bool, LuaArray<std::string>>>> vars) {
if (client) {
return std::shared_ptr<KeyValueStore>(nullptr);
}

std::optional<std::string> lookupAction;
std::optional<std::string> dataName;
getOptionalValue<std::string>(vars, "dataName", dataName);
getOptionalValue<std::string>(vars, "lookupAction", lookupAction);

checkAllParametersConsumed("newRedisKVStore", vars);

std::string uniqueId = "url=" + redisClient->getUrl().to_string() + ",action=" + lookupAction.value_or("GET") + ",data-name=" + dataName.value_or("") + ",";
std::string labels = "redis-server=" + redisClient->getUrl().host + ":" + std::to_string(redisClient->getUrl().port) + ",redis-action=" + lookupAction.value_or("GET") + ",data-name=" + dataName.value_or("");
std::shared_ptr<RedisStats> stats = std::make_shared<RedisStats>(labels);
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So the idea is to share metrics between all RedisKVStore instances using the same parameters, right? What's the use case for defining several of these?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm I wanted to prevent that case, to prevent confusion about the metrics. I thought the runtime error I threw below would prevent duplicate instances with the exact same params.

There is a use case for different parameters of course and that should be supported.


dnsdist::configuration::updateRuntimeConfiguration([uniqueId, &stats](dnsdist::configuration::RuntimeConfiguration& config) {
if (config.d_redisStats.count(uniqueId) > 0) {
throw std::runtime_error("Duplicate redis instance. Combination of arguments has to be unique!");
}
config.d_redisStats.emplace(uniqueId, std::shared_ptr(stats));
});

return std::shared_ptr<KeyValueStore>(new RedisKVStore(redisClient, lookupAction, dataName, stats));
});
#endif /* HAVE_REDIS */

#if defined(HAVE_LMDB) || defined(HAVE_CDB) || defined(HAVE_REDIS)
/* Key Value Store objects */
luaCtx.writeFunction("KeyValueLookupKeySourceIP", [](std::optional<uint8_t> v4Mask, std::optional<uint8_t> v6Mask, std::optional<bool> includePort) {
return std::shared_ptr<KeyValueLookupKey>(new KeyValueLookupKeySourceIP(v4Mask ? *v4Mask : 32, v6Mask ? *v6Mask : 128, includePort ? *includePort : false));
Expand Down Expand Up @@ -113,5 +141,5 @@ void setupLuaBindingsKVS([[maybe_unused]] LuaContext& luaCtx, [[maybe_unused]] b

return kvs->reload();
});
#endif /* defined(HAVE_LMDB) || defined(HAVE_CDB) */
#endif /* defined(HAVE_LMDB) || defined(HAVE_CDB) || defined(HAVE_REDIS) */
}
Loading
Loading