diff --git a/lib/base/json.cpp b/lib/base/json.cpp index 56893308ae..56f5557e37 100644 --- a/lib/base/json.cpp +++ b/lib/base/json.cpp @@ -1,23 +1,111 @@ /* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */ #include "base/json.hpp" -#include "base/debug.hpp" -#include "base/namespace.hpp" -#include "base/dictionary.hpp" #include "base/array.hpp" -#include "base/objectlock.hpp" -#include "base/convert.hpp" -#include "base/utility.hpp" -#include -#include -#include -#include #include -#include -#include using namespace icinga; +constexpr uint8_t JsonEncoder::m_IndentSize; + +/** + * Encodes the given value into JSON and writes it to the configured output stream. + * + * This function is specialized for the @c Value type and recursively encodes each + * and every concrete type it represents. + * + * @param value The value to be JSON serialized. + * @param currentIndent The current indentation level. Defaults to 0. + */ +void JsonEncoder::Encode(const Value& value, unsigned currentIndent) +{ + switch (value.GetType()) { + case ValueEmpty: + m_Writer->write_characters("null", 4); + return; + case ValueBoolean: + if (value.ToBool()) { + m_Writer->write_characters("true", 4); + } else { + m_Writer->write_characters("false", 5); + } + return; + case ValueString: + EncodeValidatedJson(Utility::ValidateUTF8(value.Get())); + return; + case ValueNumber: + if (auto ll(static_cast(value)); ll == value) { + EncodeValidatedJson(ll); + } else { + EncodeValidatedJson(value.Get()); + } + return; + case ValueObject: { + const Object::Ptr& obj = value.Get(); + if (obj->GetReflectionType() == Namespace::TypeInstance) { + if (auto ns(static_pointer_cast(obj)); ns->GetLength() == 0) { + m_Writer->write_characters("{}", 2); + } else { + static constexpr auto extractor = [](const NamespaceValue& v) -> const Value& { return v.Val; }; + EncodeObject(ns, currentIndent, extractor); + } + } else if (obj->GetReflectionType() == Dictionary::TypeInstance) { + if (auto dict(static_pointer_cast(obj)); dict->GetLength() == 0) { + m_Writer->write_characters("{}", 2); + } else { + static constexpr auto extractor = [](const Value& v) -> const Value& { return v; }; + EncodeObject(dict, currentIndent, extractor); + } + } else if (obj->GetReflectionType() == Array::TypeInstance) { + auto arr(static_pointer_cast(obj)); + ObjectLock olock(arr); + auto begin(arr->Begin()); + auto end(arr->End()); + // Release the object lock if we're writing to an Asio stream, i.e. every write operation + // is asynchronous which may cause the coroutine to yield and later resume on another thread. + RELEASE_OLOCK_IF_ASYNC_WRITER(m_Writer, olock); + + if (arr->GetLength() == 0) { + m_Writer->write_characters("[]", 2); + } else if (m_Pretty) { + m_Writer->write_characters("[\n", 2); + + auto newIndent = currentIndent + m_IndentSize; + if (m_IndentStr.size() < newIndent) { + m_IndentStr.resize(m_IndentStr.size() * 2, ' '); + } + + for (auto it(begin); it != end; ++it) { + if (it != begin) { + m_Writer->write_characters(",\n", 2); + } + m_Writer->write_characters(m_IndentStr.c_str(), newIndent); + Encode(*it, newIndent); + } + m_Writer->write_character('\n'); + m_Writer->write_characters(m_IndentStr.c_str(), currentIndent); + m_Writer->write_character(']'); + } else { + m_Writer->write_character('['); + for (auto it(begin); it != end; ++it) { + if (it != begin) { + m_Writer->write_character(','); + } + Encode(*it, currentIndent); + } + m_Writer->write_character(']'); + } + } else { + // Some other non-serializable object type! + EncodeValidatedJson(Utility::ValidateUTF8(obj->ToString())); + } + return; + } + default: + VERIFY(!"Invalid variant type."); + } +} + class JsonSax : public nlohmann::json_sax { public: @@ -45,165 +133,24 @@ class JsonSax : public nlohmann::json_sax void FillCurrentTarget(Value value); }; -const char l_Null[] = "null"; -const char l_False[] = "false"; -const char l_True[] = "true"; -const char l_Indent[] = " "; - -// https://github.com/nlohmann/json/issues/1512 -template -class JsonEncoder -{ -public: - void Null(); - void Boolean(bool value); - void NumberFloat(double value); - void Strng(String value); - void StartObject(); - void Key(String value); - void EndObject(); - void StartArray(); - void EndArray(); - - String GetResult(); - -private: - std::vector m_Result; - String m_CurrentKey; - std::stack> m_CurrentSubtree; - - void AppendChar(char c); - - template - void AppendChars(Iterator begin, Iterator end); - - void AppendJson(nlohmann::json json); - - void BeforeItem(); - - void FinishContainer(char terminator); -}; - -template -void Encode(JsonEncoder& stateMachine, const Value& value); - -template -inline -void EncodeNamespace(JsonEncoder& stateMachine, const Namespace::Ptr& ns) -{ - stateMachine.StartObject(); - - ObjectLock olock(ns); - for (const Namespace::Pair& kv : ns) { - stateMachine.Key(Utility::ValidateUTF8(kv.first)); - Encode(stateMachine, kv.second.Val); - } - - stateMachine.EndObject(); -} - -template -inline -void EncodeDictionary(JsonEncoder& stateMachine, const Dictionary::Ptr& dict) -{ - stateMachine.StartObject(); - - ObjectLock olock(dict); - for (const Dictionary::Pair& kv : dict) { - stateMachine.Key(Utility::ValidateUTF8(kv.first)); - Encode(stateMachine, kv.second); - } - - stateMachine.EndObject(); -} - -template -inline -void EncodeArray(JsonEncoder& stateMachine, const Array::Ptr& arr) +String icinga::JsonEncode(const Value& value, bool pretty_print) { - stateMachine.StartArray(); - - ObjectLock olock(arr); - for (const Value& value : arr) { - Encode(stateMachine, value); - } - - stateMachine.EndArray(); + std::ostringstream oss; + JsonEncode(value, oss, pretty_print); + return oss.str(); } -template -void Encode(JsonEncoder& stateMachine, const Value& value) +/** + * Serializes an Icinga Value into a JSON object and writes it to the given output stream. + * + * @param value The value to be JSON serialized. + * @param os The output stream to write the JSON data to. + * @param prettify Whether to pretty print the serialized JSON. + */ +void icinga::JsonEncode(const Value& value, std::ostream& os, bool prettify) { - switch (value.GetType()) { - case ValueNumber: - stateMachine.NumberFloat(value.Get()); - break; - - case ValueBoolean: - stateMachine.Boolean(value.ToBool()); - break; - - case ValueString: - stateMachine.Strng(Utility::ValidateUTF8(value.Get())); - break; - - case ValueObject: - { - const Object::Ptr& obj = value.Get(); - - { - Namespace::Ptr ns = dynamic_pointer_cast(obj); - if (ns) { - EncodeNamespace(stateMachine, ns); - break; - } - } - - { - Dictionary::Ptr dict = dynamic_pointer_cast(obj); - if (dict) { - EncodeDictionary(stateMachine, dict); - break; - } - } - - { - Array::Ptr arr = dynamic_pointer_cast(obj); - if (arr) { - EncodeArray(stateMachine, arr); - break; - } - } - - // obj is most likely a function => "Object of type 'Function'" - Encode(stateMachine, obj->ToString()); - break; - } - - case ValueEmpty: - stateMachine.Null(); - break; - - default: - VERIFY(!"Invalid variant type."); - } -} - -String icinga::JsonEncode(const Value& value, bool pretty_print) -{ - if (pretty_print) { - JsonEncoder stateMachine; - - Encode(stateMachine, value); - - return stateMachine.GetResult() + "\n"; - } else { - JsonEncoder stateMachine; - - Encode(stateMachine, value); - - return stateMachine.GetResult(); - } + JsonEncoder encoder(os, prettify); + encoder.Encode(value); } Value icinga::JsonDecode(const String& data) @@ -349,177 +296,3 @@ void JsonSax::FillCurrentTarget(Value value) } } } - -template -inline -void JsonEncoder::Null() -{ - BeforeItem(); - AppendChars((const char*)l_Null, (const char*)l_Null + 4); -} - -template -inline -void JsonEncoder::Boolean(bool value) -{ - BeforeItem(); - - if (value) { - AppendChars((const char*)l_True, (const char*)l_True + 4); - } else { - AppendChars((const char*)l_False, (const char*)l_False + 5); - } -} - -template -inline -void JsonEncoder::NumberFloat(double value) -{ - BeforeItem(); - - // Make sure 0.0 is serialized as 0, so e.g. Icinga DB can parse it as int. - if (value < 0) { - long long i = value; - - if (i == value) { - AppendJson(i); - } else { - AppendJson(value); - } - } else { - unsigned long long i = value; - - if (i == value) { - AppendJson(i); - } else { - AppendJson(value); - } - } -} - -template -inline -void JsonEncoder::Strng(String value) -{ - BeforeItem(); - AppendJson(std::move(value)); -} - -template -inline -void JsonEncoder::StartObject() -{ - BeforeItem(); - AppendChar('{'); - - m_CurrentSubtree.push(2); -} - -template -inline -void JsonEncoder::Key(String value) -{ - m_CurrentKey = std::move(value); -} - -template -inline -void JsonEncoder::EndObject() -{ - FinishContainer('}'); -} - -template -inline -void JsonEncoder::StartArray() -{ - BeforeItem(); - AppendChar('['); - - m_CurrentSubtree.push(0); -} - -template -inline -void JsonEncoder::EndArray() -{ - FinishContainer(']'); -} - -template -inline -String JsonEncoder::GetResult() -{ - return String(m_Result.begin(), m_Result.end()); -} - -template -inline -void JsonEncoder::AppendChar(char c) -{ - m_Result.emplace_back(c); -} - -template -template -inline -void JsonEncoder::AppendChars(Iterator begin, Iterator end) -{ - m_Result.insert(m_Result.end(), begin, end); -} - -template -inline -void JsonEncoder::AppendJson(nlohmann::json json) -{ - nlohmann::detail::serializer(nlohmann::detail::output_adapter(m_Result), ' ').dump(std::move(json), prettyPrint, true, 0); -} - -template -inline -void JsonEncoder::BeforeItem() -{ - if (!m_CurrentSubtree.empty()) { - auto& node (m_CurrentSubtree.top()); - - if (node[0]) { - AppendChar(','); - } else { - node[0] = true; - } - - if (prettyPrint) { - AppendChar('\n'); - - for (auto i (m_CurrentSubtree.size()); i; --i) { - AppendChars((const char*)l_Indent, (const char*)l_Indent + 4); - } - } - - if (node[1]) { - AppendJson(std::move(m_CurrentKey)); - AppendChar(':'); - - if (prettyPrint) { - AppendChar(' '); - } - } - } -} - -template -inline -void JsonEncoder::FinishContainer(char terminator) -{ - if (prettyPrint && m_CurrentSubtree.top()[0]) { - AppendChar('\n'); - - for (auto i (m_CurrentSubtree.size() - 1u); i; --i) { - AppendChars((const char*)l_Indent, (const char*)l_Indent + 4); - } - } - - AppendChar(terminator); - - m_CurrentSubtree.pop(); -} diff --git a/lib/base/json.hpp b/lib/base/json.hpp index df0ea18a0b..42d2c6a1ef 100644 --- a/lib/base/json.hpp +++ b/lib/base/json.hpp @@ -4,14 +4,222 @@ #define JSON_H #include "base/i2-base.hpp" +#include "base/dictionary.hpp" +#include "base/namespace.hpp" +#include "base/objectlock.hpp" +#include "base/tlsstream.hpp" +#include "base/utility.hpp" +#include namespace icinga { +/** + * Asynchronous JSON writer interface. + * + * This interface is used to write JSON data asynchronously into all kind of streams. It is intended to be used with + * the @c JsonEncoder class, but it's not limited to. It extends the @c nlohmann::detail::output_adapter_protocol<> + * interface, thus any class implementing this can be used as an output adapter for the @c nlohmann/json library. + * + * @ingroup base + */ +class AsyncJsonWriter : public nlohmann::detail::output_adapter_protocol +{ + virtual void async_put(char ch) = 0; + virtual void async_write(const char* data, std::size_t size) = 0; +}; + +/** + * Asio stream adapter for JSON serialization. + * + * This class implements the @c AsyncJsonWriter interface and provides an adapter for writing JSON data to an + * Asio stream. Similar to the @c nlohmann::detail::output_stream_adapter<> class, this class provides methods + * for writing characters and strings to an Asio stream asynchronously. + * + * In order to satisfy the @c nlohmann::detail::output_adapter_protocol<> interface, this class provides a proper + * implementation of the @c write_character() and @c write_characters() methods, which just forward the request to + * the @c async_put() and @c async_write() methods respectively. The type of the underlying Asio stream should be + * either @c AsioTcpStream or @c AsioTlsStream, or any other type derived from the @c boost::asio::buffered_stream<> + * class, since every write operation (even with a single char) is directly written to the stream without any extra + * buffering. + * + * @ingroup base + */ +template +class AsioStreamAdapter : public AsyncJsonWriter +{ +public: + AsioStreamAdapter(Stream& stream, boost::asio::yield_context& yc): m_Stream(stream), m_Yield{yc} + { + } + + void write_character(char c) override + { + async_put(c); + } + + void write_characters(const char* s, std::size_t length) override + { + async_write(s, length); + } + + void async_put(char ch) override + { + m_Stream.async_write_some(boost::asio::buffer(&ch, 1), m_Yield); + } + + void async_write(const char* data, std::size_t size) override + { + boost::asio::async_write(m_Stream, boost::asio::buffer(data, size), m_Yield); + } + +private: + Stream& m_Stream; + boost::asio::yield_context& m_Yield; +}; + class String; class Value; +/** + * JSON encoder. + * + * This class can be used to encode Icinga Value types into JSON format and write them to an output stream. + * The supported stream types include std::ostream and AsioStreamAdapter. The nlohmann/json library already + * provides a full support for the former stream type, while the latter is fully implemented by our own and + * satisfies the @c nlohmann::detail::output_adapter_protocol<> interface. It is used for writing the produced + * JSON directly to an Asio either TCP or TLS stream and doesn't require any additional buffering other than + * the one used by the Asio buffered_stream<> class internally. + * + * The JSON encoder generates most of the low level JSON tokens, but it still relies on the already existing + * @c nlohmann::detail::serializer<> class to dump numbers and ascii validated JSON strings. This means that the + * encoder doesn't perform any kind of JSON validation or escaping on its own, but simply delegates all this kind + * of work to serializer<>. However, Strings are UTF-8 validated beforehand using the @c Utility::ValidateUTF8() + * function and only the validated (copy of the original) String is passed to the serializer. + * + * The generated JSON can be either prettified or compact, depending on your needs. The prettified JSON object + * is indented with 4 spaces and grows linearly with the depth of the object tree. + * + * @note The JSON serialization logic of this encoder is heavily inspired by the @c nlohmann::detail::serializer<>. + * + * @ingroup base + */ +class JsonEncoder +{ +#define RELEASE_OLOCK_IF_ASYNC_WRITER(writer, olock) \ + if (std::is_base_of::value) { \ + olock.Unlock(); \ + } + +public: + explicit JsonEncoder(std::basic_ostream& stream, bool prettify = false) + : JsonEncoder{nlohmann::detail::output_adapter(stream), prettify} + { + } + + explicit JsonEncoder(const std::shared_ptr>& writer, bool prettify = false) + : JsonEncoder{nlohmann::detail::output_adapter_t(writer), prettify} + { + } + + void Encode(const Value& value, unsigned currentIndent = 0); + +private: + JsonEncoder(nlohmann::detail::output_adapter_t stream, bool pretty) + : m_Pretty(pretty), m_IndentStr(32, ' '), m_Writer(std::move(stream)) + {} + + /** + * Serializes a value into JSON using the serializer and writes it directly to @c m_Writer. + * + * @tparam T Type of the value to encode via the serializer. + * @param value The value to validate and encode using the serializer. + */ + template + void EncodeValidatedJson(T&& value) const + { + nlohmann::detail::serializer s(m_Writer, ' ', nlohmann::json::error_handler_t::strict); + s.dump(nlohmann::json(std::forward(value)), m_Pretty, true, 0, 0); + } + + /** + * Encodes an Icinga 2 object (Namespace or Dictionary) into JSON and writes it to @c m_Writer. + * + * @tparam Iterable Type of the container (Namespace or Dictionary). + * @tparam ValExtractor Type of the value extractor function used to extract values from the container's iterator. + * + * @param container The container to JSON serialize. + * @param currentIndent The current indentation level. + * @param extractor The value extractor function used to extract values from the container's iterator. + */ + template + void EncodeObject(const Iterable& container, unsigned currentIndent, const ValExtractor& extractor) + { + static_assert(std::is_same_v || std::is_same_v, + "Container must be a Namespace or Dictionary"); + + ObjectLock olock(container); + auto begin(container->Begin()); + auto end(container->End()); + // Release the object lock if we're writing to an Asio stream, i.e. every write operation + // is asynchronous which may cause the coroutine to yield and later resume on another thread. + RELEASE_OLOCK_IF_ASYNC_WRITER(*m_Writer, olock); + + if (m_Pretty) { + m_Writer->write_characters("{\n", 2); + + auto newIndent = currentIndent + m_IndentSize; + if (m_IndentStr.size() < newIndent) { + m_IndentStr.resize(m_IndentStr.size() * 2, ' '); + } + + for (auto it(begin); it != end; ++it) { + // Add a comma before the next key-value pair, but only if it's not the first one. + if (it != begin) { + m_Writer->write_characters(",\n", 2); + } + m_Writer->write_characters(m_IndentStr.c_str(), newIndent); + EncodeValidatedJson(Utility::ValidateUTF8(it->first)); + m_Writer->write_characters(": ", 2); + Encode(extractor(it->second), newIndent); + } + + m_Writer->write_character('\n'); + m_Writer->write_characters(m_IndentStr.c_str(), currentIndent); + m_Writer->write_character('}'); + } else { + m_Writer->write_character('{'); + for (auto it(begin); it != end; ++it) { + // Add a comma before the next key-value pair, but not before the first one. + if (it != begin) { + m_Writer->write_character(','); + } + EncodeValidatedJson(Utility::ValidateUTF8(it->first)); + m_Writer->write_character(':'); + Encode(extractor(it->second), currentIndent); + } + m_Writer->write_character('}'); + } + } + +private: + // The number of spaces to use for indentation in prettified JSON. + static constexpr uint8_t m_IndentSize = 4; + + bool m_Pretty; // Whether to pretty-print the JSON output. + // The pre-allocated indent characters for pretty-printing. + // By default, 32 ' ' characters (bytes) are allocated which can be used to indent JSON object trees up to + // 8 levels deep (4 spaces per level) without reallocation. Otherwise, when encountering a deeper JSON tree, + // this will be resized to the required size on the fly and doubled each time. + std::string m_IndentStr; + + // The output stream adapter for writing JSON data. + // This can be either a std::ostream or an Asio stream adapter. + nlohmann::detail::output_adapter_t m_Writer; +}; + String JsonEncode(const Value& value, bool pretty_print = false); +void JsonEncode(const Value& value, std::ostream& os, bool prettify = false); Value JsonDecode(const String& data); } diff --git a/lib/remote/eventshandler.cpp b/lib/remote/eventshandler.cpp index bdda714611..b2c808f8f6 100644 --- a/lib/remote/eventshandler.cpp +++ b/lib/remote/eventshandler.cpp @@ -9,9 +9,9 @@ #include "base/io-engine.hpp" #include "base/objectlock.hpp" #include "base/json.hpp" -#include #include #include +#include #include #include @@ -110,20 +110,16 @@ bool EventsHandler::HandleRequest( http::async_write(stream, response, yc); stream.async_flush(yc); - asio::const_buffer newLine ("\n", 1); + auto adapter(std::make_shared>(stream, yc)); + JsonEncoder encoder(adapter); for (;;) { - auto event (subscriber.GetInbox()->Shift(yc)); - - if (event) { - String body = JsonEncode(event); - - boost::algorithm::replace_all(body, "\n", ""); - - asio::const_buffer payload (body.CStr(), body.GetLength()); - - asio::async_write(stream, payload, yc); - asio::async_write(stream, newLine, yc); + if (auto event(subscriber.GetInbox()->Shift(yc)); event) { + encoder.Encode(event); + // Put a newline at the end of each event to render them on a separate line. + adapter->async_put('\n'); + // Since shifting the next event may cause the coroutine to yield, we need to flush the + // stream after each event to ensure that the client receives it immediately. stream.async_flush(yc); } else if (server.Disconnected()) { return true; diff --git a/test/base-json.cpp b/test/base-json.cpp index 02bbebb6d6..b4c1a0e3db 100644 --- a/test/base-json.cpp +++ b/test/base-json.cpp @@ -39,10 +39,14 @@ BOOST_AUTO_TEST_CASE(encode) "string": "LF\nTAB\tAUml\u00e4Ill\ufffd", "true": true, "uint": 23 -} -)EOF"); +})EOF"); + + auto got(JsonEncode(input, true)); + BOOST_CHECK_MESSAGE(output == got, "expected=" << output << "\ngot=" << got); - BOOST_CHECK(JsonEncode(input, true) == output); + std::ostringstream oss; + JsonEncode(input, oss, true); + BOOST_CHECK_MESSAGE(oss.str() == output, "expected=" << output << "\ngot=" << oss.str()); boost::algorithm::replace_all(output, " ", ""); boost::algorithm::replace_all(output, "Objectoftype'Function'", "Object of type 'Function'");