diff --git a/lib/base/CMakeLists.txt b/lib/base/CMakeLists.txt index d44254a35e..1bb700c3d3 100644 --- a/lib/base/CMakeLists.txt +++ b/lib/base/CMakeLists.txt @@ -37,6 +37,7 @@ set(base_SOURCES fifo.cpp fifo.hpp filelogger.cpp filelogger.hpp filelogger-ti.hpp function.cpp function.hpp function-ti.hpp function-script.cpp functionwrapper.hpp + generator.hpp initialize.cpp initialize.hpp intrusive-ptr.hpp io-engine.cpp io-engine.hpp diff --git a/lib/base/array.cpp b/lib/base/array.cpp index 08e06fad2d..ff982e2dcb 100644 --- a/lib/base/array.cpp +++ b/lib/base/array.cpp @@ -99,7 +99,9 @@ void Array::Add(Value value, bool overrideFrozen) */ Array::Iterator Array::Begin() { - ASSERT(OwnsLock()); + if (!Frozen()) { + ASSERT(OwnsLock()); + } return m_Data.begin(); } @@ -113,7 +115,9 @@ Array::Iterator Array::Begin() */ Array::Iterator Array::End() { - ASSERT(OwnsLock()); + if (!Frozen()) { + ASSERT(OwnsLock()); + } return m_Data.end(); } @@ -333,7 +337,12 @@ Array::Ptr Array::Unique() const void Array::Freeze() { ObjectLock olock(this); - m_Frozen = true; + m_Frozen.store(true); +} + +bool Array::Frozen() const +{ + return m_Frozen; } Value Array::GetFieldByName(const String& field, bool sandboxed, const DebugInfo& debugInfo) const diff --git a/lib/base/array.hpp b/lib/base/array.hpp index 2c9a9dda7f..4f199e2eb3 100644 --- a/lib/base/array.hpp +++ b/lib/base/array.hpp @@ -4,6 +4,7 @@ #define ARRAY_H #include "base/i2-base.hpp" +#include "base/atomic.hpp" #include "base/objectlock.hpp" #include "base/value.hpp" #include @@ -98,13 +99,14 @@ class Array final : public Object Array::Ptr Unique() const; void Freeze(); + bool Frozen() const; Value GetFieldByName(const String& field, bool sandboxed, const DebugInfo& debugInfo) const override; void SetFieldByName(const String& field, const Value& value, bool overrideFrozen, const DebugInfo& debugInfo) override; private: std::vector m_Data; /**< The data for the array. */ - bool m_Frozen{false}; + Atomic m_Frozen{false}; }; Array::Iterator begin(const Array::Ptr& x); diff --git a/lib/base/dictionary.cpp b/lib/base/dictionary.cpp index 43df4af8f0..3bb2d98a99 100644 --- a/lib/base/dictionary.cpp +++ b/lib/base/dictionary.cpp @@ -133,7 +133,9 @@ bool Dictionary::Contains(const String& key) const */ Dictionary::Iterator Dictionary::Begin() { - ASSERT(OwnsLock()); + if (!Frozen()) { + ASSERT(OwnsLock()); + } return m_Data.begin(); } @@ -147,7 +149,9 @@ Dictionary::Iterator Dictionary::Begin() */ Dictionary::Iterator Dictionary::End() { - ASSERT(OwnsLock()); + if (!Frozen()) { + ASSERT(OwnsLock()); + } return m_Data.end(); } @@ -277,7 +281,12 @@ String Dictionary::ToString() const void Dictionary::Freeze() { ObjectLock olock(this); - m_Frozen = true; + m_Frozen.store(true); +} + +bool Dictionary::Frozen() const +{ + return m_Frozen; } Value Dictionary::GetFieldByName(const String& field, bool, const DebugInfo& debugInfo) const diff --git a/lib/base/dictionary.hpp b/lib/base/dictionary.hpp index ffccd630f3..7f2eb0ead1 100644 --- a/lib/base/dictionary.hpp +++ b/lib/base/dictionary.hpp @@ -4,6 +4,7 @@ #define DICTIONARY_H #include "base/i2-base.hpp" +#include "base/atomic.hpp" #include "base/object.hpp" #include "base/value.hpp" #include @@ -69,6 +70,7 @@ class Dictionary final : public Object String ToString() const override; void Freeze(); + bool Frozen() const; Value GetFieldByName(const String& field, bool sandboxed, const DebugInfo& debugInfo) const override; void SetFieldByName(const String& field, const Value& value, bool overrideFrozen, const DebugInfo& debugInfo) override; @@ -78,7 +80,7 @@ class Dictionary final : public Object private: std::map m_Data; /**< The data for the dictionary. */ mutable std::shared_timed_mutex m_DataMutex; - bool m_Frozen{false}; + Atomic m_Frozen{false}; }; Dictionary::Iterator begin(const Dictionary::Ptr& x); diff --git a/lib/base/generator.hpp b/lib/base/generator.hpp new file mode 100644 index 0000000000..fa41e67fc1 --- /dev/null +++ b/lib/base/generator.hpp @@ -0,0 +1,48 @@ +/* Icinga 2 | (c) 2025 Icinga GmbH | GPLv2+ */ + +#pragma once + +#include "base/i2-base.hpp" +#include "base/value.hpp" +#include + +namespace icinga +{ + +/** + * ValueGenerator is a class that defines a generator function type for producing Values on demand. + * + * This class is used to create generator functions that can yield any values that can be represented by the + * Icinga Value type. The generator function is exhausted when it returns `std::nullopt`, indicating that there + * are no more values to produce. Subsequent calls to `Next()` will always return `std::nullopt` after exhaustion. + * + * @ingroup base + */ +class ValueGenerator final : public Object +{ +public: + DECLARE_PTR_TYPEDEFS(ValueGenerator); + + /** + * Generates a Value using the provided generator function. + * + * The generator function should return an `std::optional` which contains the produced Value or + * `std::nullopt` when there are no more values to produce. After the generator function returns `std::nullopt`, + * the generator is considered exhausted, and further calls to `Next()` will always return `std::nullopt`. + */ + using GenFunc = std::function()>; + + explicit ValueGenerator(GenFunc generator): m_Generator(std::move(generator)) + { + } + + std::optional Next() const + { + return m_Generator(); + } + +private: + GenFunc m_Generator; // The generator function that produces Values. +}; + +} diff --git a/lib/base/json.cpp b/lib/base/json.cpp index 56893308ae..cb4281ce9f 100644 --- a/lib/base/json.cpp +++ b/lib/base/json.cpp @@ -2,22 +2,200 @@ #include "base/json.hpp" #include "base/debug.hpp" -#include "base/namespace.hpp" -#include "base/dictionary.hpp" #include "base/array.hpp" -#include "base/objectlock.hpp" -#include "base/convert.hpp" -#include "base/utility.hpp" -#include -#include -#include -#include +#include "base/generator.hpp" #include #include #include using namespace icinga; +constexpr uint8_t JsonEncoder::m_IndentSize; + +JsonEncoder::JsonEncoder(std::string& output, bool prettify) + : JsonEncoder{nlohmann::detail::output_adapter(output), prettify} +{ +} + +JsonEncoder::JsonEncoder(std::basic_ostream& stream, bool prettify) + : JsonEncoder{nlohmann::detail::output_adapter(stream), prettify} +{ +} + +JsonEncoder::JsonEncoder(nlohmann::detail::output_adapter_t stream, bool prettify) + : m_Pretty(prettify), m_Indent{0}, m_IndentStr(32, ' '), m_Writer(std::move(stream)) +{ +} + +/** + * Encodes the given value into JSON and writes it to the configured output stream. + * + * This function is specialized for the @c Value type and recursively encodes each + * and every concrete type it represents. + * + * @param value The value to be JSON serialized. + */ +void JsonEncoder::EncodeImpl(const Value& value, std::optional& yc, bool parentOlockAcquired) +{ + switch (value.GetType()) { + case ValueEmpty: + Write("null"); + return; + case ValueBoolean: + if (value.ToBool()) { + Write("true"); + } else { + Write("false"); + } + return; + case ValueString: + EncodeValidatedJson(Utility::ValidateUTF8(value.Get())); + return; + case ValueNumber: + if (auto ll(static_cast(value)); ll == value) { + EncodeValidatedJson(ll); + } else { + EncodeValidatedJson(value.Get()); + } + return; + case ValueObject: { + const Object::Ptr& obj = value.Get(); + const auto& type = obj->GetReflectionType(); + if (type == Namespace::TypeInstance) { + static constexpr auto extractor = [](const NamespaceValue& v) -> const Value& { return v.Val; }; + EncodeObject(static_pointer_cast(obj), extractor, yc, parentOlockAcquired); + } else if (type == Dictionary::TypeInstance) { + static constexpr auto extractor = [](const Value& v) -> const Value& { return v; }; + EncodeObject(static_pointer_cast(obj), extractor, yc, parentOlockAcquired); + } else if (type == Array::TypeInstance) { + auto arr(static_pointer_cast(obj)); + BeginContainer('['); + ObjectLock olock(arr, std::defer_lock); + bool objectLockAcquired = parentOlockAcquired; + if (!arr->Frozen()) { + FlushIfSafe(yc, parentOlockAcquired); + olock.Lock(); + objectLockAcquired = true; + } + + int count = 0; + for (const auto& item : arr) { + WriteSeparatorAndIndentStrIfNeeded(count++); + EncodeImpl(item, yc, objectLockAcquired); + } + EndContainer(']', arr->GetLength() == 0); + } else if (auto gen(dynamic_pointer_cast(obj)); gen) { + BeginContainer('['); + bool isEmpty = true; + for (int i = 0; true; ++i) { + auto result = gen->Next(); + if (!result) { + break; // Stop when the generator is exhausted. + } + isEmpty = false; + WriteSeparatorAndIndentStrIfNeeded(i); + EncodeImpl(*result, yc, parentOlockAcquired); + } + EndContainer(']', isEmpty); + } else { + // Some other non-serializable object type! + EncodeValidatedJson(Utility::ValidateUTF8(obj->ToString())); + } + return; + } + default: + VERIFY(!"Invalid variant type."); + } +} + +/** + * Flushes the output stream if it is safe to do so. + * + * Safe flushing means that it only performs the flush operation if the @c JsonEncoder has not acquired + * any object lock so far. This is to ensure that the stream can safely perform asynchronous operations + * without risking undefined behaviour due to coroutines being suspended while the stream is being flushed. + * + * Additionally, if there is no yield context provided, it will not attempt to flush the stream at all. + * + * @param yc The yield context to use for asynchronous operations. + * @param objectLockAcquired Whether an object lock has been acquired. + */ +void JsonEncoder::FlushIfSafe(std::optional& yc, bool objectLockAcquired) const +{ + if (yc && !objectLockAcquired) { + if (auto ajw(dynamic_cast(m_Writer.get())); ajw) { + ajw->Flush(*yc); + } + } +} + +/** + * Writes a string to the underlying output stream. + * + * This function writes the provided string view directly to the output stream without any additional formatting. + * + * @param sv The string view to write to the output stream. + */ +void JsonEncoder::Write(const std::string_view& sv) const +{ + m_Writer->write_characters(sv.data(), sv.size()); +} + +/** + * Begins a JSON container (object or array) by writing the opening character and adjusting the + * indentation level if pretty-printing is enabled. + * + * @param openChar The character that opens the container (either '{' for objects or '[' for arrays). + */ +void JsonEncoder::BeginContainer(char openChar) +{ + Write(std::string(1, openChar)); + if (m_Pretty) { + m_Indent += m_IndentSize; + if (m_IndentStr.size() < m_Indent) { + m_IndentStr.resize(m_IndentStr.size() * 2, ' '); + } + } +} + +/** + * Ends a JSON container (object or array) by writing the closing character and adjusting the + * indentation level if pretty-printing is enabled. + * + * @param closeChar The character that closes the container (either '}' for objects or ']' for arrays). + * @param isContainerEmpty Whether the container is empty, used to determine if a newline should be written. + */ +void JsonEncoder::EndContainer(char closeChar, bool isContainerEmpty) +{ + if (m_Pretty) { + VERIFY(m_Indent >= m_IndentSize); // Ensure we don't underflow the indent size. + m_Indent -= m_IndentSize; + if (!isContainerEmpty) { + Write("\n"); + m_Writer->write_characters(m_IndentStr.c_str(), m_Indent); + } + } + Write(std::string(1, closeChar)); +} + +/** + * Writes a separator (comma) and an indentation string if pretty-printing is enabled. + * + * This function is used to separate items in a JSON array or object and to maintain the correct indentation level. + * + * @param count The current item count, used to determine if a separator should be written. + */ +void JsonEncoder::WriteSeparatorAndIndentStrIfNeeded(int count) const +{ + if (count > 0) { + Write(","); + } + if (m_Pretty) { + Write("\n"); + m_Writer->write_characters(m_IndentStr.c_str(), m_Indent); + } +} + class JsonSax : public nlohmann::json_sax { public: @@ -45,165 +223,25 @@ class JsonSax : public nlohmann::json_sax void FillCurrentTarget(Value value); }; -const char l_Null[] = "null"; -const char l_False[] = "false"; -const char l_True[] = "true"; -const char l_Indent[] = " "; - -// https://github.com/nlohmann/json/issues/1512 -template -class JsonEncoder -{ -public: - void Null(); - void Boolean(bool value); - void NumberFloat(double value); - void Strng(String value); - void StartObject(); - void Key(String value); - void EndObject(); - void StartArray(); - void EndArray(); - - String GetResult(); - -private: - std::vector m_Result; - String m_CurrentKey; - std::stack> m_CurrentSubtree; - - void AppendChar(char c); - - template - void AppendChars(Iterator begin, Iterator end); - - void AppendJson(nlohmann::json json); - - void BeforeItem(); - - void FinishContainer(char terminator); -}; - -template -void Encode(JsonEncoder& stateMachine, const Value& value); - -template -inline -void EncodeNamespace(JsonEncoder& stateMachine, const Namespace::Ptr& ns) -{ - stateMachine.StartObject(); - - ObjectLock olock(ns); - for (const Namespace::Pair& kv : ns) { - stateMachine.Key(Utility::ValidateUTF8(kv.first)); - Encode(stateMachine, kv.second.Val); - } - - stateMachine.EndObject(); -} - -template -inline -void EncodeDictionary(JsonEncoder& stateMachine, const Dictionary::Ptr& dict) +String icinga::JsonEncode(const Value& value, bool pretty_print) { - stateMachine.StartObject(); - - ObjectLock olock(dict); - for (const Dictionary::Pair& kv : dict) { - stateMachine.Key(Utility::ValidateUTF8(kv.first)); - Encode(stateMachine, kv.second); - } - - stateMachine.EndObject(); + std::string output; + JsonEncoder encoder(output, pretty_print); + encoder.Encode(value); + return String(std::move(output)); } -template -inline -void EncodeArray(JsonEncoder& stateMachine, const Array::Ptr& arr) +/** + * Serializes an Icinga Value into a JSON object and writes it to the given output stream. + * + * @param value The value to be JSON serialized. + * @param os The output stream to write the JSON data to. + * @param prettify Whether to pretty print the serialized JSON. + */ +void icinga::JsonEncode(const Value& value, std::ostream& os, bool prettify) { - stateMachine.StartArray(); - - ObjectLock olock(arr); - for (const Value& value : arr) { - Encode(stateMachine, value); - } - - stateMachine.EndArray(); -} - -template -void Encode(JsonEncoder& stateMachine, const Value& value) -{ - switch (value.GetType()) { - case ValueNumber: - stateMachine.NumberFloat(value.Get()); - break; - - case ValueBoolean: - stateMachine.Boolean(value.ToBool()); - break; - - case ValueString: - stateMachine.Strng(Utility::ValidateUTF8(value.Get())); - break; - - case ValueObject: - { - const Object::Ptr& obj = value.Get(); - - { - Namespace::Ptr ns = dynamic_pointer_cast(obj); - if (ns) { - EncodeNamespace(stateMachine, ns); - break; - } - } - - { - Dictionary::Ptr dict = dynamic_pointer_cast(obj); - if (dict) { - EncodeDictionary(stateMachine, dict); - break; - } - } - - { - Array::Ptr arr = dynamic_pointer_cast(obj); - if (arr) { - EncodeArray(stateMachine, arr); - break; - } - } - - // obj is most likely a function => "Object of type 'Function'" - Encode(stateMachine, obj->ToString()); - break; - } - - case ValueEmpty: - stateMachine.Null(); - break; - - default: - VERIFY(!"Invalid variant type."); - } -} - -String icinga::JsonEncode(const Value& value, bool pretty_print) -{ - if (pretty_print) { - JsonEncoder stateMachine; - - Encode(stateMachine, value); - - return stateMachine.GetResult() + "\n"; - } else { - JsonEncoder stateMachine; - - Encode(stateMachine, value); - - return stateMachine.GetResult(); - } + JsonEncoder encoder(os, prettify); + encoder.Encode(value); } Value icinga::JsonDecode(const String& data) @@ -349,177 +387,3 @@ void JsonSax::FillCurrentTarget(Value value) } } } - -template -inline -void JsonEncoder::Null() -{ - BeforeItem(); - AppendChars((const char*)l_Null, (const char*)l_Null + 4); -} - -template -inline -void JsonEncoder::Boolean(bool value) -{ - BeforeItem(); - - if (value) { - AppendChars((const char*)l_True, (const char*)l_True + 4); - } else { - AppendChars((const char*)l_False, (const char*)l_False + 5); - } -} - -template -inline -void JsonEncoder::NumberFloat(double value) -{ - BeforeItem(); - - // Make sure 0.0 is serialized as 0, so e.g. Icinga DB can parse it as int. - if (value < 0) { - long long i = value; - - if (i == value) { - AppendJson(i); - } else { - AppendJson(value); - } - } else { - unsigned long long i = value; - - if (i == value) { - AppendJson(i); - } else { - AppendJson(value); - } - } -} - -template -inline -void JsonEncoder::Strng(String value) -{ - BeforeItem(); - AppendJson(std::move(value)); -} - -template -inline -void JsonEncoder::StartObject() -{ - BeforeItem(); - AppendChar('{'); - - m_CurrentSubtree.push(2); -} - -template -inline -void JsonEncoder::Key(String value) -{ - m_CurrentKey = std::move(value); -} - -template -inline -void JsonEncoder::EndObject() -{ - FinishContainer('}'); -} - -template -inline -void JsonEncoder::StartArray() -{ - BeforeItem(); - AppendChar('['); - - m_CurrentSubtree.push(0); -} - -template -inline -void JsonEncoder::EndArray() -{ - FinishContainer(']'); -} - -template -inline -String JsonEncoder::GetResult() -{ - return String(m_Result.begin(), m_Result.end()); -} - -template -inline -void JsonEncoder::AppendChar(char c) -{ - m_Result.emplace_back(c); -} - -template -template -inline -void JsonEncoder::AppendChars(Iterator begin, Iterator end) -{ - m_Result.insert(m_Result.end(), begin, end); -} - -template -inline -void JsonEncoder::AppendJson(nlohmann::json json) -{ - nlohmann::detail::serializer(nlohmann::detail::output_adapter(m_Result), ' ').dump(std::move(json), prettyPrint, true, 0); -} - -template -inline -void JsonEncoder::BeforeItem() -{ - if (!m_CurrentSubtree.empty()) { - auto& node (m_CurrentSubtree.top()); - - if (node[0]) { - AppendChar(','); - } else { - node[0] = true; - } - - if (prettyPrint) { - AppendChar('\n'); - - for (auto i (m_CurrentSubtree.size()); i; --i) { - AppendChars((const char*)l_Indent, (const char*)l_Indent + 4); - } - } - - if (node[1]) { - AppendJson(std::move(m_CurrentKey)); - AppendChar(':'); - - if (prettyPrint) { - AppendChar(' '); - } - } - } -} - -template -inline -void JsonEncoder::FinishContainer(char terminator) -{ - if (prettyPrint && m_CurrentSubtree.top()[0]) { - AppendChar('\n'); - - for (auto i (m_CurrentSubtree.size() - 1u); i; --i) { - AppendChars((const char*)l_Indent, (const char*)l_Indent + 4); - } - } - - AppendChar(terminator); - - m_CurrentSubtree.pop(); -} diff --git a/lib/base/json.hpp b/lib/base/json.hpp index df0ea18a0b..0947c209b4 100644 --- a/lib/base/json.hpp +++ b/lib/base/json.hpp @@ -4,14 +4,283 @@ #define JSON_H #include "base/i2-base.hpp" +#include "base/dictionary.hpp" +#include "base/namespace.hpp" +#include "base/objectlock.hpp" +#include "base/utility.hpp" +#include +#include +#include namespace icinga { +/** + * AsyncJsonWriter allows writing JSON data to any output stream asynchronously. + * + * All users of this class must ensure that the underlying output stream will not perform any asynchronous I/O + * operations when the @c write_character() or @c write_characters() methods are called. They shall only perform + * such ops when the @c JsonEncoder allows them to do so by calling the @c Flush() method. + * + * @ingroup base + */ +class AsyncJsonWriter : public nlohmann::detail::output_adapter_protocol +{ +public: + /** + * Flush instructs the underlying output stream to write any buffered data to wherever it is supposed to go. + * + * The @c JsonEncoder allows the stream to even perform asynchronous operations in a safe manner by calling + * this method with a dedicated @c boost::asio::yield_context object. The stream must not perform any async + * I/O operations triggered by methods other than this one. Any attempt to do so will result in undefined behavior. + * + * However, this doesn't necessarily enforce the stream to really flush its data immediately, but it's up + * to the implementation to do whatever it needs to. The encoder just gives it a chance to do so by calling + * this method. + * + * @param yield The yield context to use for asynchronous operations. + */ + virtual void Flush(boost::asio::yield_context& yield) = 0; +}; + +/** + * Adapter class for Boost Beast HTTP messages body to be used with the @c JsonEncoder. + * + * This class implements the @c nlohmann::detail::output_adapter_protocol<> interface and provides + * a way to write JSON data directly into the body of a Boost Beast HTTP message. The adapter is designed + * to work with Boost Beast HTTP messages that conform to the Beast HTTP message interface and must provide + * a body type that has a publicly accessible `reader` type that satisfies the Beast BodyReader [^1] requirements. + * + * @ingroup base + * + * [^1]: https://www.boost.org/doc/libs/1_85_0/libs/beast/doc/html/beast/concepts/BodyReader.html + */ +template +class BeastHttpMessageAdapter : public AsyncJsonWriter +{ +public: + using BodyType = typename BeastHttpMessage::body_type; + using Reader = typename BeastHttpMessage::body_type::reader; + + explicit BeastHttpMessageAdapter(BeastHttpMessage& msg) : m_Reader(msg.base(), msg.body()), m_Message{msg} + { + boost::system::error_code ec; + // This never returns an actual error, except when overflowing the max + // buffer size, which we don't do here. + m_Reader.init(0, ec); + } + + ~BeastHttpMessageAdapter() override + { + boost::system::error_code ec; + // Same here as in the constructor, all the standard Beast HTTP message reader implementations + // never return an error here, it's just there to satisfy the interface requirements. + m_Reader.finish(ec); + } + + void write_character(char c) override + { + write_characters(&c, 1); + } + + void write_characters(const char* s, std::size_t length) override + { + boost::system::error_code ec; + m_Reader.put(boost::asio::const_buffer{s, length}, ec); + if (ec) { + BOOST_THROW_EXCEPTION(boost::system::system_error{ec}); + } + m_PendingBufferSize += length; + } + + void Flush(boost::asio::yield_context& yield) override + { + if (m_PendingBufferSize >= 4096) { + m_Message.Write(yield); + m_PendingBufferSize = 0; + } + } + +private: + Reader m_Reader; + BeastHttpMessage& m_Message; + std::uint64_t m_PendingBufferSize{0}; // Tracks the size of the pending buffer to avoid unnecessary writes. +}; + +/** + * Adapter class for writing JSON data to a std::string. + * + * This class implements the @c nlohmann::detail::output_adapter_protocol<> interface and provides + * a way to write JSON data directly into a std::string and overcomes the overhead of using a + * @c std::ostringstream for JSON serialization. + * + * The constructor takes a reference to a std::string, which is used as the output buffer for the JSON data, + * and you must ensure that reference remains valid for the lifetime of the StringOutputAdapter instance. + * + * @ingroup base + */ +class StringOutputAdapter final : public nlohmann::detail::output_adapter_protocol +{ +public: + explicit StringOutputAdapter(std::string& str) : m_OutStr(str) + {} + + void write_character(char c) override + { + m_OutStr.append(1, c); + } + + void write_characters(const char* s, std::size_t length) override + { + m_OutStr.append(s, length); + } + +private: + std::string& m_OutStr; +}; + class String; class Value; +/** + * JSON encoder. + * + * This class can be used to encode Icinga Value types into JSON format and write them to an output stream. + * The supported stream types include std::ostream and AsioStreamAdapter. The nlohmann/json library already + * provides a full support for the former stream type, while the latter is fully implemented by our own and + * satisfies the @c nlohmann::detail::output_adapter_protocol<> interface. It is used for writing the produced + * JSON directly to an Asio either TCP or TLS stream and doesn't require any additional buffering other than + * the one used by the Asio buffered_stream<> class internally. + * + * The JSON encoder generates most of the low level JSON tokens, but it still relies on the already existing + * @c nlohmann::detail::serializer<> class to dump numbers and ascii validated JSON strings. This means that the + * encoder doesn't perform any kind of JSON validation or escaping on its own, but simply delegates all this kind + * of work to serializer<>. However, Strings are UTF-8 validated beforehand using the @c Utility::ValidateUTF8() + * function and only the validated (copy of the original) String is passed to the serializer. + * + * The generated JSON can be either prettified or compact, depending on your needs. The prettified JSON object + * is indented with 4 spaces and grows linearly with the depth of the object tree. + * + * @ingroup base + */ +class JsonEncoder +{ +public: + explicit JsonEncoder(std::string& output, bool prettify = false); + explicit JsonEncoder(std::basic_ostream& stream, bool prettify = false); + explicit JsonEncoder(nlohmann::detail::output_adapter_t stream, bool prettify = false); + + /** + * Encodes a single value into JSON and writes it to the underlying output stream. + * + * This method is the main entry point for encoding JSON data. It takes a value of any type that can be + * represented by our @c Value class and encodes it into JSON in an efficient manner. If prettifying is + * enabled, the JSON output will be formatted with indentation and newlines for better readability, and + * the final JSON will also be terminated by a newline character. + * + * @note If the used output adapter performs asynchronous I/O operations (it's derived from @c AsyncJsonWriter), + * please provide a @c boost::asio::yield_context object to allow the encoder to flush the output stream in a + * safe manner. The encoder will try to regularly give the output stream a chance to flush its data when it is + * safe to do so, but for this to work, there must be a valid yield context provided. Otherwise, the encoder + * will not attempt to flush the output stream at all, which may lead huge memory consumption when encoding + * large JSON objects or arrays. + * + * @tparam T The type of the value to encode. + * + * @param value The value to encode into JSON format. + */ + template + void Encode(T&& value, std::optional yc = std::nullopt) + { + EncodeImpl(std::forward(value), yc, false); + // If we are at the top level of the JSON object and prettifying is enabled, we need + // to end the JSON with a newline character to ensure that the output is properly formatted. + if (m_Indent == 0 && m_Pretty) { + Write("\n"); + } + } + +private: + void EncodeImpl(const Value& value, std::optional& yc, bool parentOlockAcquired); + void FlushIfSafe(std::optional& yc, bool objectLockAcquired) const; + + void Write(const std::string_view& sv) const; + void BeginContainer(char openChar); + void EndContainer(char closeChar, bool isContainerEmpty = false); + void WriteSeparatorAndIndentStrIfNeeded(int count) const; + + /** + * Serializes a value into JSON using the serializer and writes it directly to @c m_Writer. + * + * @tparam T Type of the value to encode via the serializer. + * @param value The value to validate and encode using the serializer. + */ + template + void EncodeValidatedJson(T&& value) const + { + nlohmann::detail::serializer s(m_Writer, ' ', nlohmann::json::error_handler_t::strict); + s.dump(nlohmann::json(std::forward(value)), m_Pretty, true, 0, 0); + } + + /** + * Encodes an Icinga 2 object (Namespace or Dictionary) into JSON and writes it to @c m_Writer. + * + * @tparam Iterable Type of the container (Namespace or Dictionary). + * @tparam ValExtractor Type of the value extractor function used to extract values from the container's iterator. + * + * @param container The container to JSON serialize. + * @param extractor The value extractor function used to extract values from the container's iterator. + * @param yc The optional yield context for asynchronous operations. + * @param parentOlockAcquired Whether an object lock has been acquired on some of the parent containers. + */ + template + void EncodeObject(const Iterable& container, const ValExtractor& extractor, std::optional& yc, bool parentOlockAcquired) + { + static_assert(std::is_same_v || std::is_same_v, + "Container must be a Namespace or Dictionary"); + + BeginContainer('{'); + ObjectLock olock(container, std::defer_lock); + bool objectLockAcquired = parentOlockAcquired; + if (!container->Frozen()) { + FlushIfSafe(yc, parentOlockAcquired); + olock.Lock(); + objectLockAcquired = true; + } + + int count = 0; + for (const auto& [key, val] : container) { + WriteSeparatorAndIndentStrIfNeeded(count++); + EncodeValidatedJson(Utility::ValidateUTF8(key)); + Write(m_Pretty ? ": " : ":"); + + const auto& v = extractor(val); + EncodeImpl(v, yc, objectLockAcquired); + } + EndContainer('}', container->GetLength() == 0); + } + +private: + // The number of spaces to use for indentation in prettified JSON. + static constexpr uint8_t m_IndentSize = 4; + + bool m_Pretty; // Whether to pretty-print the JSON output. + unsigned m_Indent; // The current indentation level for pretty-printing. + /** + * The pre-allocated indent characters for pretty-printing. + * + * By default, 32 @c ' ' characters (bytes) are allocated which can be used to indent JSON object trees up to + * 8 levels deep (4 spaces per level) without reallocation. Otherwise, when encountering a deeper JSON tree, + * this will be resized to the required size on the fly and doubled each time. + */ + std::string m_IndentStr; + + // The output stream adapter for writing JSON data. This can be either a std::ostream or an Asio stream adapter. + nlohmann::detail::output_adapter_t m_Writer; +}; + String JsonEncode(const Value& value, bool pretty_print = false); +void JsonEncode(const Value& value, std::ostream& os, bool prettify = false); Value JsonDecode(const String& data); } diff --git a/lib/base/namespace.cpp b/lib/base/namespace.cpp index c3b859cff2..66968cc078 100644 --- a/lib/base/namespace.cpp +++ b/lib/base/namespace.cpp @@ -122,6 +122,11 @@ void Namespace::Freeze() { m_Frozen = true; } +bool Namespace::Frozen() const +{ + return m_Frozen; +} + std::shared_lock Namespace::ReadLockUnlessFrozen() const { if (m_Frozen.load(std::memory_order_relaxed)) { @@ -165,14 +170,18 @@ bool Namespace::GetOwnField(const String& field, Value *result) const Namespace::Iterator Namespace::Begin() { - ASSERT(OwnsLock()); + if (!Frozen()) { + ASSERT(OwnsLock()); + } return m_Data.begin(); } Namespace::Iterator Namespace::End() { - ASSERT(OwnsLock()); + if (!Frozen()) { + ASSERT(OwnsLock()); + } return m_Data.end(); } diff --git a/lib/base/namespace.hpp b/lib/base/namespace.hpp index 94f2055d37..f9a7acfa97 100644 --- a/lib/base/namespace.hpp +++ b/lib/base/namespace.hpp @@ -73,6 +73,7 @@ class Namespace final : public Object bool Contains(const String& field) const; void Remove(const String& field); void Freeze(); + bool Frozen() const; Iterator Begin(); Iterator End(); diff --git a/lib/base/objectlock.cpp b/lib/base/objectlock.cpp index fc0c7c6314..e0246907c0 100644 --- a/lib/base/objectlock.cpp +++ b/lib/base/objectlock.cpp @@ -18,6 +18,13 @@ ObjectLock::ObjectLock(const Object::Ptr& object) { } +ObjectLock::ObjectLock(const Object::Ptr& object, std::defer_lock_t) + : m_Object(object.get()), m_Locked(false) +{ + // This constructor is used to create a lock without immediately locking the object. + // The user must call Lock() explicitly when needed. +} + ObjectLock::ObjectLock(const Object *object) : m_Object(object), m_Locked(false) { diff --git a/lib/base/objectlock.hpp b/lib/base/objectlock.hpp index 8e98641db6..abd071c666 100644 --- a/lib/base/objectlock.hpp +++ b/lib/base/objectlock.hpp @@ -15,6 +15,7 @@ struct ObjectLock { public: ObjectLock(const Object::Ptr& object); + ObjectLock(const Object::Ptr& object, std::defer_lock_t); ObjectLock(const Object *object); ObjectLock(const ObjectLock&) = delete; diff --git a/test/base-json.cpp b/test/base-json.cpp index 02bbebb6d6..6a62de2c01 100644 --- a/test/base-json.cpp +++ b/test/base-json.cpp @@ -4,6 +4,7 @@ #include "base/function.hpp" #include "base/namespace.hpp" #include "base/array.hpp" +#include "base/generator.hpp" #include "base/objectlock.hpp" #include "base/json.hpp" #include @@ -15,6 +16,15 @@ BOOST_AUTO_TEST_SUITE(base_json) BOOST_AUTO_TEST_CASE(encode) { + auto generate = []() -> std::optional { + static int count = 0; + if (++count == 4) { + count = 0; + return std::nullopt; + } + return Value(count); + }; + Dictionary::Ptr input (new Dictionary({ { "array", new Array({ new Namespace() }) }, { "false", false }, @@ -24,16 +34,24 @@ BOOST_AUTO_TEST_CASE(encode) { "null", Value() }, { "string", "LF\nTAB\tAUml\xC3\xA4Ill\xC3" }, { "true", true }, - { "uint", 23u } + { "uint", 23u }, + { "generator", new ValueGenerator(generate) }, + { "empty_generator", new ValueGenerator([]() -> std::optional { return std::nullopt; }) } })); String output (R"EOF({ "array": [ {} ], + "empty_generator": [], "false": false, "float": -1.25, "fx": "Object of type 'Function'", + "generator": [ + 1, + 2, + 3 + ], "int": -42, "null": null, "string": "LF\nTAB\tAUml\u00e4Ill\ufffd", @@ -42,7 +60,12 @@ BOOST_AUTO_TEST_CASE(encode) } )EOF"); - BOOST_CHECK(JsonEncode(input, true) == output); + auto got(JsonEncode(input, true)); + BOOST_CHECK_MESSAGE(output == got, "expected=" << output << "\ngot=" << got); + + std::ostringstream oss; + JsonEncode(input, oss, true); + BOOST_CHECK_MESSAGE(oss.str() == output, "expected=" << output << "\ngot=" << oss.str()); boost::algorithm::replace_all(output, " ", ""); boost::algorithm::replace_all(output, "Objectoftype'Function'", "Object of type 'Function'");