diff --git a/.github/workflows/long-test.yml b/.github/workflows/long-test.yml index 05555fc20f2..ba711fe64d5 100644 --- a/.github/workflows/long-test.yml +++ b/.github/workflows/long-test.yml @@ -180,7 +180,7 @@ jobs: cd build ./tests.sh --output-on-failure --timeout 1600 -LE "benchmark" - - name: "Run CBOR fuzz test" + - name: "Run fuzz tests" run: | set -o pipefail set -ex @@ -188,9 +188,17 @@ jobs: cd build_fuzz rm -f CMakeCache.txt cmake -GNinja -DFUZZING=ON -DSAN=ON -DUSE_SNMALLOC=OFF .. - ninja cbor_fuzz_test - mkdir -p /tmp/cbor_fuzz_live - ./cbor_fuzz_test /tmp/cbor_fuzz_live ../src/crypto/test/cbor_fuzz_corpus -max_total_time=60 + ninja cbor_fuzz_test msgpack_fuzz_test + mkdir -p fuzz-live/cbor fuzz-live/msgpack fuzz-artifacts/cbor fuzz-artifacts/msgpack + ./cbor_fuzz_test fuzz-live/cbor ../src/crypto/test/cbor_fuzz_corpus \ + -max_total_time=60 \ + -artifact_prefix="$PWD/fuzz-artifacts/cbor/" + ./msgpack_fuzz_test fuzz-live/msgpack ../src/msgpack/test/msgpack_fuzz_corpus \ + -max_total_time=60 \ + -max_len=4096 \ + -timeout=2 \ + -rss_limit_mb=1024 \ + -artifact_prefix="$PWD/fuzz-artifacts/msgpack/" - name: "Upload logs" if: success() || failure() @@ -204,6 +212,7 @@ jobs: build/workspace/*/*.ledger/* build/workspace/*/stack_trace build/workspace/**/openapi_coverage.json + build_fuzz/fuzz-artifacts/** if-no-files-found: ignore # All e2e tests in release mode (same as release build). diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f075dd6a70..b9590436b9c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -750,6 +750,18 @@ if(BUILD_TESTS) target_link_libraries(cbor_fuzz_test PRIVATE evercbor) endif() + add_unit_test( + msgpack_test + ${CMAKE_CURRENT_SOURCE_DIR}/src/msgpack/test/msgpack_unit.cpp + ) + + if(FUZZING) + add_fuzz_test( + msgpack_fuzz_test + ${CMAKE_CURRENT_SOURCE_DIR}/src/msgpack/test/msgpack_fuzz.cpp + ) + endif() + add_unit_test( sharing_test ${CMAKE_CURRENT_SOURCE_DIR}/src/crypto/test/secret_sharing.cpp @@ -1060,6 +1072,7 @@ if(BUILD_TESTS) add_picobench(map_bench SRCS src/ds/test/map_bench.cpp) add_picobench(logger_bench SRCS src/ds/test/logger_bench.cpp) add_picobench(json_bench SRCS src/ds/test/json_bench.cpp) + add_picobench(msgpack_bench SRCS src/msgpack/test/msgpack_bench.cpp) add_picobench(ring_buffer_bench SRCS src/ds/test/ring_buffer_bench.cpp) add_picobench(ledger_bench SRCS src/host/test/ledger_bench.cpp) add_picobench(crypto_bench SRCS src/crypto/test/bench.cpp LINK_LIBS) diff --git a/cmake/preproject.cmake b/cmake/preproject.cmake index 289929b6bcb..1e5151b68bf 100644 --- a/cmake/preproject.cmake +++ b/cmake/preproject.cmake @@ -83,4 +83,4 @@ function(add_warning_checks name) ) endfunction() -set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD 23) diff --git a/src/msgpack/encode.h b/src/msgpack/encode.h new file mode 100644 index 00000000000..0275891a78d --- /dev/null +++ b/src/msgpack/encode.h @@ -0,0 +1,434 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once + +// Header-only msgpack encoder. +// +// Spec: https://github.com/msgpack/msgpack/blob/master/spec.md +// +// Encoder-only. Decoding is out of scope; CCF currently decodes via +// nlohmann::json::from_msgpack. The encoder writes the smallest format +// family that fits each value (the spec's recommended canonical form). +// +// Supported subset: +// - All msgpack scalar types (nil, bool, int, uint, float64, +// str fixstr/str8/str16/str32, bin bin8/16/32). +// - Arrays (fixarray/array16/array32) and maps (fixmap/map16/map32). +// Out of scope: +// - float32 (write_float always emits float64). +// +// Failure modes that may escape ANY write_* function: +// - MsgpackEncodeError on encoder-defined limits (see Error enum). +// - std::bad_alloc from the underlying std::vector if buffer growth +// fails. The encoder offers no special handling - callers that +// might recover from OOM should treat the buffer as undefined-but- +// well-typed. +// +// For repeated records, reserve the expected maximum size once and reuse the +// same vector with clear(). This keeps its allocation while resetting its size. + +#include "msgpack/endian.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ccf::msgpack +{ + // ===== Errors as data, throw at the boundary ===== + + enum class Error : uint8_t + { + STRING_TOO_LARGE = 1, // > 2^32-1 bytes + BIN_TOO_LARGE = 2, // > 2^32-1 bytes + INVALID_EVENT_TIME = 4, // outside Fluentd EventTime's representable range + }; + + // Every error knows how to describe itself. The returned string_view + // refers to a function-local string literal (static storage duration); + // it is safe to retain indefinitely. + // + // Tests should match on MsgpackEncodeError::error_code(), not what(): + // what() messages are not part of the API contract and may be + // reformatted at any time. + [[nodiscard]] inline std::string_view to_string(Error e) + { + switch (e) + { + case Error::STRING_TOO_LARGE: + return "STRING_TOO_LARGE"; + case Error::BIN_TOO_LARGE: + return "BIN_TOO_LARGE"; + case Error::INVALID_EVENT_TIME: + return "INVALID_EVENT_TIME"; + default: + return "UNKNOWN_MSGPACK_ERROR"; + } + } + + // Thrown by encoder boundary functions. + // + // API contract: + // - error_code() identifies the failure as a stable enum value. + // - what() returns a human-readable diagnostic that includes the + // offending value where applicable. The exact format is NOT + // part of the API; do not parse it. Tests asserting on a + // specific failure mode must match on error_code(). + class MsgpackEncodeError : public std::runtime_error + { + public: + explicit MsgpackEncodeError(Error err, const std::string& what) : + std::runtime_error(what), + error(err) + {} + + // Convenience constructor: composes the standard ": " + // shape used at every throw site, ensuring every diagnostic + // includes the error code's name without each call site having to + // remember the convention. + [[nodiscard]] static MsgpackEncodeError make( + Error err, std::string_view detail) + { + return MsgpackEncodeError( + err, std::string{to_string(err)} + ": " + std::string{detail}); + } + + [[nodiscard]] Error error_code() const + { + return error; + } + + private: + Error error; + }; + + // ===== Format byte constants ===== + // Named per the msgpack spec so the write_* bodies read as direct + // transcriptions rather than magic numbers. Fix-family values are + // prefixes that get OR'd with a small N. + namespace fmt_byte + { + // Variable-length families. + constexpr uint8_t NIL = 0xC0; + constexpr uint8_t FALSE_ = 0xC2; + constexpr uint8_t TRUE_ = 0xC3; + constexpr uint8_t BIN_8 = 0xC4; + constexpr uint8_t BIN_16 = 0xC5; + constexpr uint8_t BIN_32 = 0xC6; + constexpr uint8_t FLOAT_64 = 0xCB; + constexpr uint8_t UINT_8 = 0xCC; + constexpr uint8_t UINT_16 = 0xCD; + constexpr uint8_t UINT_32 = 0xCE; + constexpr uint8_t UINT_64 = 0xCF; + constexpr uint8_t INT_8 = 0xD0; + constexpr uint8_t INT_16 = 0xD1; + constexpr uint8_t INT_32 = 0xD2; + constexpr uint8_t INT_64 = 0xD3; + constexpr uint8_t FIXEXT_8 = 0xD7; + constexpr uint8_t STR_8 = 0xD9; + constexpr uint8_t STR_16 = 0xDA; + constexpr uint8_t STR_32 = 0xDB; + constexpr uint8_t ARRAY_16 = 0xDC; + constexpr uint8_t ARRAY_32 = 0xDD; + constexpr uint8_t MAP_16 = 0xDE; + constexpr uint8_t MAP_32 = 0xDF; + + // Fix-family prefixes (OR with the 4- or 5-bit count). + constexpr uint8_t FIXSTR_PREFIX = 0xA0; // 0b101XXXXX (0xA0..0xBF) + constexpr uint8_t FIXARRAY_PREFIX = 0x90; // 0b1001XXXX (0x90..0x9F) + constexpr uint8_t FIXMAP_PREFIX = 0x80; // 0b1000XXXX (0x80..0x8F) + // positive fixint: 0b0XXXXXXX (0x00..0x7F) - emitted as the value itself. + // negative fixint: 0b111XXXXX (0xE0..0xFF) - emitted as the int8 bit + // pattern. + + } // namespace fmt_byte + + // ===== Scalar encoders ===== + + inline void write_nil(std::vector& buf) + { + buf.push_back(fmt_byte::NIL); + } + + inline void write_bool(std::vector& buf, bool v) + { + buf.push_back(v ? fmt_byte::TRUE_ : fmt_byte::FALSE_); + } + + // Smallest-format-wins: + // [0, 127] -> positive fixint (1 byte) + // [128, 255] -> uint 8 (2 bytes) + // [256, 65535] -> uint 16 (3 bytes) + // [65536, 2^32-1] -> uint 32 (5 bytes) + // [2^32, 2^64-1] -> uint 64 (9 bytes) + inline void write_uint(std::vector& buf, uint64_t v) + { + if (v <= 0x7FU) + { + buf.push_back(static_cast(v)); + } + else if (v <= 0xFFU) + { + utils::append_tagged_be( + buf, fmt_byte::UINT_8, static_cast(v)); + } + else if (v <= 0xFFFFU) + { + utils::append_tagged_be( + buf, fmt_byte::UINT_16, static_cast(v)); + } + else if (v <= 0xFFFFFFFFU) + { + utils::append_tagged_be( + buf, fmt_byte::UINT_32, static_cast(v)); + } + else + { + utils::append_tagged_be(buf, fmt_byte::UINT_64, v); + } + } + + // Smallest-format-wins for signed values. + // For non-negative inputs we delegate to write_uint, so write_int(5) + // produces one byte 0x05 (positive fixint), not the wider int 8 form + // 0xD0 0x05. This is the spec's canonical form (smallest fitting + // family across the unsigned and signed numeric ranges). + // + // For negative values: + // [-32, -1] -> negative fixint (1 byte) + // [-128, -33] -> int 8 (2 bytes) + // [-32768, -129] -> int 16 (3 bytes) + // [-2^31, -32769] -> int 32 (5 bytes) + // [INT64_MIN, -2^31 - 1] -> int 64 (9 bytes) + inline void write_int(std::vector& buf, int64_t v) + { + if (v >= 0) + { + write_uint(buf, static_cast(v)); + return; + } + + if (v >= -32) + { + // negative fixint: 0b111XXXXX, value is the 5-bit two's-complement. + // Equivalently: byte = 0xE0 | (v & 0x1F), but the cleanest formulation + // is to take the unsigned bit-pattern of the int8. + buf.push_back(static_cast(static_cast(v))); + } + else if (v >= std::numeric_limits::min()) + { + utils::append_tagged_be( + buf, fmt_byte::INT_8, static_cast(static_cast(v))); + } + else if (v >= std::numeric_limits::min()) + { + utils::append_tagged_be( + buf, fmt_byte::INT_16, static_cast(static_cast(v))); + } + else if (v >= std::numeric_limits::min()) + { + utils::append_tagged_be( + buf, fmt_byte::INT_32, static_cast(static_cast(v))); + } + else + { + utils::append_tagged_be( + buf, fmt_byte::INT_64, static_cast(v)); + } + } + + // Always emits float64 (0xCB ...). float32 narrowing is not + // supported; callers wanting it can add a separate write_float32. + // + // NaN and infinity bit-patterns are passed through unchanged: the + // function performs no canonicalisation. A signalling NaN stays a + // signalling NaN; -inf stays -inf. If the caller needs canonical + // NaN encoding, normalise before calling. + inline void write_float(std::vector& buf, double v) + { + static_assert( + sizeof(double) == 8, "ccf::msgpack assumes IEEE-754 binary64 doubles"); + uint64_t bits = 0; + std::memcpy(&bits, &v, sizeof(bits)); + utils::append_tagged_be(buf, fmt_byte::FLOAT_64, bits); + } + + // ===== str ===== + // + // Smallest-format-wins: + // [0, 31] -> fixstr (1-byte header) + // [32, 255] -> str 8 (2-byte header) + // [256, 65535] -> str 16 (3-byte header) + // [65536, 2^32-1] -> str 32 (5-byte header) + // Throws MsgpackEncodeError(STRING_TOO_LARGE) for sizes >= 2^32. + // + // The payload is copied verbatim. The msgpack spec defines str as + // UTF-8, but this encoder does not validate it. + inline void write_str(std::vector& buf, std::string_view s) + { + // The reinterpret_cast below from `const char*` to `const uint8_t*` + // is well-defined only if uint8_t IS unsigned char (so the access + // is "an unsigned char or std::byte" per [basic.lval]). Hold this + // invariant explicitly. + static_assert( + std::is_same_v, + "ccf::msgpack assumes uint8_t == unsigned char"); + + std::string aliased_s; + if (!buf.empty() && !s.empty()) + { + const auto less = std::less{}; + const auto* const buf_begin = buf.data(); + const auto* const buf_end = buf_begin + buf.size(); + const auto* const s_begin = reinterpret_cast(s.data()); + const auto* const s_end = s_begin + s.size(); + if (less(s_begin, buf_end) && less(buf_begin, s_end)) + { + aliased_s.assign(s); + s = aliased_s; + } + } + + const auto n = s.size(); + if (n <= 31U) + { + buf.push_back(static_cast(fmt_byte::FIXSTR_PREFIX | n)); + } + else if (n <= 0xFFU) + { + utils::append_tagged_be( + buf, fmt_byte::STR_8, static_cast(n)); + } + else if (n <= 0xFFFFU) + { + utils::append_tagged_be( + buf, fmt_byte::STR_16, static_cast(n)); + } + else if (n <= 0xFFFFFFFFULL) + { + utils::append_tagged_be( + buf, fmt_byte::STR_32, static_cast(n)); + } + else + { + throw MsgpackEncodeError::make( + Error::STRING_TOO_LARGE, + "string length " + std::to_string(n) + " exceeds 2^32 - 1"); + } + if (!s.empty()) + { + buf.insert( + buf.end(), + reinterpret_cast(s.data()), + reinterpret_cast(s.data()) + n); + } + } + + // ===== bin ===== + // + // Smallest-format-wins: + // [0, 255] -> bin 8 (2-byte header) + // [256, 65535] -> bin 16 (3-byte header) + // [65536, 2^32-1] -> bin 32 (5-byte header) + // Throws MsgpackEncodeError(BIN_TOO_LARGE) for sizes >= 2^32. + inline void write_bin( + std::vector& buf, std::span data) + { + std::vector aliased_data; + if (!buf.empty() && !data.empty()) + { + const auto less = std::less{}; + const auto* const buf_begin = buf.data(); + const auto* const buf_end = buf_begin + buf.size(); + const auto* const data_begin = data.data(); + const auto* const data_end = data_begin + data.size(); + if (less(data_begin, buf_end) && less(buf_begin, data_end)) + { + aliased_data.assign(data.begin(), data.end()); + data = aliased_data; + } + } + + const auto n = data.size(); + if (n <= 0xFFU) + { + utils::append_tagged_be( + buf, fmt_byte::BIN_8, static_cast(n)); + } + else if (n <= 0xFFFFU) + { + utils::append_tagged_be( + buf, fmt_byte::BIN_16, static_cast(n)); + } + else if (n <= 0xFFFFFFFFULL) + { + utils::append_tagged_be( + buf, fmt_byte::BIN_32, static_cast(n)); + } + else + { + throw MsgpackEncodeError::make( + Error::BIN_TOO_LARGE, + "bin length " + std::to_string(n) + " exceeds 2^32 - 1"); + } + buf.insert(buf.end(), data.begin(), data.end()); + } + + // ===== container headers ===== + // + // Coupling: the wire format requires the element count up front, so + // the caller must subsequently emit exactly `n` values (or `n` + // key/value pairs for a map). A wrong `n` produces malformed msgpack + // output silently - the encoder cannot check this at the header + // call site. + + // Smallest-format-wins: + // [0, 15] -> fixarray (1-byte header) + // [16, 65535] -> array_16 (3-byte header) + // [65536, 2^32-1] -> array_32 (5-byte header) + // Cannot throw MsgpackEncodeError: the input is uint32_t, so every + // value fits one of the above families. + inline void write_array_header(std::vector& buf, uint32_t n) + { + if (n <= 15U) + { + buf.push_back(static_cast(fmt_byte::FIXARRAY_PREFIX | n)); + } + else if (n <= 0xFFFFU) + { + utils::append_tagged_be( + buf, fmt_byte::ARRAY_16, static_cast(n)); + } + else + { + utils::append_tagged_be(buf, fmt_byte::ARRAY_32, n); + } + } + + // Smallest-format-wins: + // [0, 15] -> fixmap (1-byte header) + // [16, 65535] -> map_16 (3-byte header) + // [65536, 2^32-1] -> map_32 (5-byte header) + inline void write_map_header(std::vector& buf, uint32_t n) + { + if (n <= 15U) + { + buf.push_back(static_cast(fmt_byte::FIXMAP_PREFIX | n)); + } + else if (n <= 0xFFFFU) + { + utils::append_tagged_be( + buf, fmt_byte::MAP_16, static_cast(n)); + } + else + { + utils::append_tagged_be(buf, fmt_byte::MAP_32, n); + } + } +} // namespace ccf::msgpack diff --git a/src/msgpack/endian.h b/src/msgpack/endian.h new file mode 100644 index 00000000000..8a8c6849c43 --- /dev/null +++ b/src/msgpack/endian.h @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace ccf::msgpack::utils +{ + // The msgpack wire format is big-endian. The byte-swap below assumes + // a little-endian host; on a big-endian host it would silently no-op + // and produce wrong output. The static_assert fires loudly if that + // changes. + static_assert( + std::endian::native == std::endian::little, + "ccf::msgpack::utils::write_be assumes a little-endian host; " + "rework the byte-swap to support a big-endian platform."); + + // Write `value` to `out` in big-endian byte order. Only unsigned integer + // widths are accepted; callers wanting to write a signed value reinterpret + // it through the matching unsigned type at the call site. + template + void write_be(uint8_t* out, T value) + { + static_assert(std::is_unsigned_v, "write_be expects an unsigned type"); + static_assert( + sizeof(T) == 1 || sizeof(T) == 2 || sizeof(T) == 4 || sizeof(T) == 8, + "write_be supports 1/2/4/8-byte unsigned integers"); + + if constexpr (sizeof(T) == 1) + { + *out = static_cast(value); + } + else + { + const auto swapped = std::byteswap(value); + std::memcpy(out, &swapped, sizeof(T)); + } + } + + inline uint8_t* append_space(std::vector& buf, size_t size) + { + const auto offset = buf.size(); + buf.resize(offset + size); + return buf.data() + offset; + } + + template + void append_tagged_be(std::vector& buf, uint8_t tag, T value) + { + auto* out = append_space(buf, 1 + sizeof(T)); + *out = tag; + write_be(out + 1, value); + } +} // namespace ccf::msgpack::utils diff --git a/src/msgpack/fluentd_event_time.h b/src/msgpack/fluentd_event_time.h new file mode 100644 index 00000000000..da05471ca07 --- /dev/null +++ b/src/msgpack/fluentd_event_time.h @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once + +#include "msgpack/encode.h" + +#include +#include +#include +#include +#include + +namespace ccf::msgpack +{ + // Validated wrapper for Fluentd's application-defined EventTime extension + // type, not the MessagePack Timestamp extension type (-1). The two have + // different layouts and should remain separate types. + // + // Wire format (fixext8): 0xD7 0x00 . + // + // Construction takes a system_clock::time_point so callers cannot swap the + // seconds and nanoseconds operands, and callers with a time_point do not + // have to decompose it by hand. + // + // Range limitations enforced by make(): + // - Seconds since the epoch must fit in uint32_t. The range ends at + // 2106-02-07 06:28:15 UTC. + // - The time_point must not predate the epoch. + // + // If timestamps past 2106 are needed, add the MessagePack Timestamp 64 form + // as a sibling type. + class FluentdEventTime + { + public: + // Throws MsgpackEncodeError(INVALID_EVENT_TIME) when tp is outside + // EventTime's uint32 seconds range. The diagnostic includes the offending + // epoch value. + // + // The wire format carries 32-bit nanoseconds. The encoded precision is + // limited by system_clock's resolution. + [[nodiscard]] static FluentdEventTime make( + std::chrono::system_clock::time_point tp) + { + const auto since_epoch = tp.time_since_epoch(); + + // Checking the original duration catches negative fractions of a second, + // which duration_cast would truncate to zero. + if (since_epoch < std::chrono::system_clock::duration::zero()) + { + const auto ns_signed = + std::chrono::duration_cast(since_epoch) + .count(); + throw MsgpackEncodeError::make( + Error::INVALID_EVENT_TIME, + "time_point predates the epoch (since_epoch_ns=" + + std::to_string(ns_signed) + ")"); + } + + const auto secs = + std::chrono::duration_cast(since_epoch); + const auto secs_count = secs.count(); + if ( + secs_count > static_cast(std::numeric_limits::max())) + { + throw MsgpackEncodeError::make( + Error::INVALID_EVENT_TIME, + "time_point beyond 2106-02-07 06:28:15 UTC (seconds=" + + std::to_string(secs_count) + ")"); + } + + const auto ns_count = + std::chrono::duration_cast(since_epoch - secs) + .count(); + return FluentdEventTime{ + static_cast(secs_count), static_cast(ns_count)}; + } + + [[nodiscard]] uint32_t seconds() const + { + return s_; + } + + [[nodiscard]] uint32_t nanoseconds() const + { + return ns_; + } + + bool operator==(const FluentdEventTime&) const = default; + + private: + FluentdEventTime(uint32_t s, uint32_t ns) : s_(s), ns_(ns) {} + + uint32_t s_; + uint32_t ns_; + }; + + // Fluentd EventTime always uses ext type 0 in fixext8 form. The 12-byte ext + // form is unnecessary because the seconds field is limited to uint32_t. + inline void write_fluentd_event_time( + std::vector& buf, FluentdEventTime t) + { + constexpr uint8_t FLUENTD_EVENT_TIME_EXT_TYPE = 0x00; + auto* out = utils::append_space(buf, 10); + *out++ = fmt_byte::FIXEXT_8; + *out++ = FLUENTD_EVENT_TIME_EXT_TYPE; + utils::write_be(out, t.seconds()); + utils::write_be(out + sizeof(uint32_t), t.nanoseconds()); + } +} // namespace ccf::msgpack diff --git a/src/msgpack/test/format_introspect.h b/src/msgpack/test/format_introspect.h new file mode 100644 index 00000000000..571cb97f7c2 --- /dev/null +++ b/src/msgpack/test/format_introspect.h @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once + +// Test-only helper: classify the first byte of a msgpack-encoded value +// into its format family. Used by smallest-format-wins boundary tests +// to assert the encoder picked the narrowest fitting form. +// +// The MessagePack-defined names, bit masks, and byte values below come from +// the format table: +// https://github.com/msgpack/msgpack/blob/9aa092d6ca81f12005bd7dcbeb6488ad319e5133/spec.md#L98-L136 +// +// The hex `case` labels intentionally cross-check the `fmt_byte::*` +// constants used by the encoder. A bug that swaps, for example, 0xCD and +// 0xCE in either place is caught when the boundary tests run. + +#include + +namespace ccf::msgpack::test +{ + enum class FormatFamily : uint8_t + { + POSITIVE_FIXINT, + NEGATIVE_FIXINT, + FIXSTR, + FIXARRAY, + FIXMAP, + NIL, + FALSE_, + TRUE_, + BIN_8, + BIN_16, + BIN_32, + FLOAT_64, + UINT_8, + UINT_16, + UINT_32, + UINT_64, + INT_8, + INT_16, + INT_32, + INT_64, + FIXEXT_8, + STR_8, + STR_16, + STR_32, + ARRAY_16, + ARRAY_32, + MAP_16, + MAP_32, + NEVER_USED, // 0xC1, must never appear in valid encoded output + UNRECOGNISED, // bytes the encoder cannot emit (ext families other + // than fixext8, the never-used 0xC1, etc.) + }; + + [[nodiscard]] inline FormatFamily classify_first_byte(uint8_t b) + { + // Fixed-prefix families first. + if ((b & 0x80U) == 0x00U) // 0b0XXXXXXX + { + return FormatFamily::POSITIVE_FIXINT; + } + if ((b & 0xE0U) == 0xE0U) // 0b111XXXXX + { + return FormatFamily::NEGATIVE_FIXINT; + } + if ((b & 0xE0U) == 0xA0U) // 0b101XXXXX + { + return FormatFamily::FIXSTR; + } + if ((b & 0xF0U) == 0x90U) // 0b1001XXXX + { + return FormatFamily::FIXARRAY; + } + if ((b & 0xF0U) == 0x80U) // 0b1000XXXX + { + return FormatFamily::FIXMAP; + } + switch (b) + { + case 0xC0: + return FormatFamily::NIL; + case 0xC1: + return FormatFamily::NEVER_USED; + case 0xC2: + return FormatFamily::FALSE_; + case 0xC3: + return FormatFamily::TRUE_; + case 0xC4: + return FormatFamily::BIN_8; + case 0xC5: + return FormatFamily::BIN_16; + case 0xC6: + return FormatFamily::BIN_32; + case 0xCB: + return FormatFamily::FLOAT_64; + case 0xCC: + return FormatFamily::UINT_8; + case 0xCD: + return FormatFamily::UINT_16; + case 0xCE: + return FormatFamily::UINT_32; + case 0xCF: + return FormatFamily::UINT_64; + case 0xD0: + return FormatFamily::INT_8; + case 0xD1: + return FormatFamily::INT_16; + case 0xD2: + return FormatFamily::INT_32; + case 0xD3: + return FormatFamily::INT_64; + case 0xD7: + return FormatFamily::FIXEXT_8; + case 0xD9: + return FormatFamily::STR_8; + case 0xDA: + return FormatFamily::STR_16; + case 0xDB: + return FormatFamily::STR_32; + case 0xDC: + return FormatFamily::ARRAY_16; + case 0xDD: + return FormatFamily::ARRAY_32; + case 0xDE: + return FormatFamily::MAP_16; + case 0xDF: + return FormatFamily::MAP_32; + default: + return FormatFamily::UNRECOGNISED; + } + } +} // namespace ccf::msgpack::test diff --git a/src/msgpack/test/json.h b/src/msgpack/test/json.h new file mode 100644 index 00000000000..32784d6430f --- /dev/null +++ b/src/msgpack/test/json.h @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once + +#include "msgpack/encode.h" + +#include +#include +#include +#include +#include + +namespace ccf::msgpack::test +{ + inline void encode_json( + std::vector& buf, const nlohmann::json& value) + { + struct Work + { + const nlohmann::json* value = nullptr; + const std::string* key = nullptr; + }; + + std::vector pending = {{&value, nullptr}}; + while (!pending.empty()) + { + const auto work = pending.back(); + pending.pop_back(); + + if (work.key != nullptr) + { + write_str(buf, *work.key); + continue; + } + + const auto& current = *work.value; + if (current.is_null()) + { + write_nil(buf); + } + else if (current.is_boolean()) + { + write_bool(buf, current.get()); + } + else if (current.is_number_unsigned()) + { + write_uint(buf, current.get()); + } + else if (current.is_number_integer()) + { + write_int(buf, current.get()); + } + else if (current.is_number_float()) + { + write_float(buf, current.get()); + } + else if (current.is_string()) + { + write_str(buf, current.get_ref()); + } + else if (current.is_binary()) + { + const auto& binary = current.get_binary(); + if (binary.has_subtype()) + { + throw std::logic_error( + "MessagePack extension values are unsupported"); + } + write_bin(buf, binary); + } + else if (current.is_array()) + { + write_array_header(buf, static_cast(current.size())); + const auto& array = current.get_ref(); + for (auto it = array.rbegin(); it != array.rend(); ++it) + { + pending.push_back({&*it, nullptr}); + } + } + else if (current.is_object()) + { + write_map_header(buf, static_cast(current.size())); + const auto& object = current.get_ref(); + for (auto it = object.rbegin(); it != object.rend(); ++it) + { + pending.push_back({&it->second, nullptr}); + pending.push_back({nullptr, &it->first}); + } + } + else + { + throw std::logic_error("Unsupported JSON value"); + } + } + } +} diff --git a/src/msgpack/test/msgpack_bench.cpp b/src/msgpack/test/msgpack_bench.cpp new file mode 100644 index 00000000000..61ff18ef222 --- /dev/null +++ b/src/msgpack/test/msgpack_bench.cpp @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#include "msgpack/encode.h" + +#define PICOBENCH_IMPLEMENT_WITH_MAIN +#include +#include +#include +#include +#include +#include + +namespace +{ + constexpr uint64_t uint_value = 1234567890ULL; + constexpr int64_t int_value = -123456789LL; + constexpr double float_value = 12.5; + constexpr std::string_view string_value = "hello msgpack"; + constexpr int64_t nested_map_value = 42; + constexpr int64_t array_value = -7; + constexpr size_t reserved_capacity = 128; + + template + inline void do_not_optimize(const T& value) + { + asm volatile("" : : "r,m"(value) : "memory"); + } + + inline void clobber_memory() + { + asm volatile("" : : : "memory"); + } + + void write_msgpack_object(std::vector& out) + { + ccf::msgpack::write_map_header(out, 8); + + ccf::msgpack::write_str(out, "nil"); + ccf::msgpack::write_nil(out); + + ccf::msgpack::write_str(out, "bool"); + ccf::msgpack::write_bool(out, true); + + ccf::msgpack::write_str(out, "uint"); + ccf::msgpack::write_uint(out, uint_value); + + ccf::msgpack::write_str(out, "int"); + ccf::msgpack::write_int(out, int_value); + + ccf::msgpack::write_str(out, "float"); + ccf::msgpack::write_float(out, float_value); + + ccf::msgpack::write_str(out, "string"); + ccf::msgpack::write_str(out, string_value); + + ccf::msgpack::write_str(out, "map"); + ccf::msgpack::write_map_header(out, 1); + ccf::msgpack::write_str(out, "value"); + ccf::msgpack::write_int(out, nested_map_value); + + ccf::msgpack::write_str(out, "array"); + ccf::msgpack::write_array_header(out, 1); + ccf::msgpack::write_int(out, array_value); + } + + void verify_reserved_capacity() + { + std::vector out; + write_msgpack_object(out); + if (out.size() > reserved_capacity) + { + throw std::logic_error("reserved capacity no longer covers benchmark"); + } + } + + void encode_fresh(picobench::state& state) + { + verify_reserved_capacity(); + + clobber_memory(); + picobench::scope scope(state); + + for (int i = 0; i < state.iterations(); ++i) + { + std::vector out; + write_msgpack_object(out); + do_not_optimize(out); + clobber_memory(); + } + } + + void encode_reserved(picobench::state& state) + { + verify_reserved_capacity(); + + clobber_memory(); + picobench::scope scope(state); + + for (int i = 0; i < state.iterations(); ++i) + { + std::vector out; + out.reserve(reserved_capacity); + write_msgpack_object(out); + do_not_optimize(out); + clobber_memory(); + } + } + + void encode_reused(picobench::state& state) + { + verify_reserved_capacity(); + + std::vector out; + out.reserve(reserved_capacity); + + clobber_memory(); + picobench::scope scope(state); + + for (int i = 0; i < state.iterations(); ++i) + { + out.clear(); + write_msgpack_object(out); + do_not_optimize(out); + clobber_memory(); + } + } +} + +const std::vector sizes = {100, 1'000, 10'000}; + +PICOBENCH_SUITE("msgpack serialise"); +PICOBENCH(encode_fresh).iterations(sizes).baseline(); +PICOBENCH(encode_reserved).iterations(sizes); +PICOBENCH(encode_reused).iterations(sizes); diff --git a/src/msgpack/test/msgpack_fuzz.cpp b/src/msgpack/test/msgpack_fuzz.cpp new file mode 100644 index 00000000000..eeb0df0972f --- /dev/null +++ b/src/msgpack/test/msgpack_fuzz.cpp @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. + +#include "msgpack/test/json.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + using nlohmann::json; + + // Translate every fuzz input into one supported JSON value. Payloads share + // a fixed budget so nested values cannot cause unbounded allocations. + class Input + { + private: + const uint8_t* data; + size_t size; + size_t offset = 0; + size_t payload_budget = 65'536; + + public: + Input(const uint8_t* data_, size_t size_) : data(data_), size(size_) {} + + uint8_t byte() + { + return offset < size ? data[offset++] : 0; + } + + uint64_t uint64() + { + uint64_t value = 0; + for (size_t i = 0; i < sizeof(value); ++i) + { + value = (value << 8) | byte(); + } + return value; + } + + size_t payload_size() + { + // Most selectors produce a small payload. Four sentinel values exercise + // the str/bin format boundaries without requiring large corpus files. + const auto selector = byte(); + size_t requested = selector; + switch (selector) + { + case 0: + case 1: + case 31: + case 32: + break; + case 252: + requested = 65'536; + break; + case 253: + requested = 65'535; + break; + case 254: + requested = 256; + break; + case 255: + requested = 255; + break; + default: + requested %= 64; + break; + } + const auto granted = std::min(requested, payload_budget); + payload_budget -= granted; + return granted; + } + + std::string string() + { + const auto length = payload_size(); + const auto pattern = byte(); + std::string value(length, '\0'); + for (size_t i = 0; i < length; ++i) + { + value[i] = static_cast(32 + ((pattern + i) % 95)); + } + return value; + } + + json::binary_t binary() + { + const auto length = payload_size(); + const auto pattern = byte(); + json::binary_t value; + value.resize(length); + for (size_t i = 0; i < length; ++i) + { + value[i] = static_cast(pattern + i); + } + return value; + } + }; + + json generate_value(Input& input) + { + constexpr uint8_t VARIANTS = 9; + + json result; + std::vector pending = {&result}; + + while (!pending.empty()) + { + auto* current = pending.back(); + pending.pop_back(); + + switch (input.byte() % VARIANTS) + { + case 0: // Null + *current = nullptr; + break; + case 1: // Boolean + *current = (input.byte() & 1) != 0; + break; + case 2: // Unsigned integer + *current = input.uint64(); + break; + case 3: // Signed integer + { + const auto value = std::bit_cast(input.uint64()); + *current = + value < 0 ? json(value) : json(static_cast(value)); + break; + } + case 4: // Floating-point number + { + auto value = std::bit_cast(input.uint64()); + if (!std::isfinite(value)) + { + value = 0; + } + *current = value; + break; + } + case 5: // String + *current = input.string(); + break; + case 6: // Binary data + *current = json::binary(input.binary()); + break; + case 7: // Array + { + *current = json::array(); + auto& array = current->get_ref(); + array.resize(input.byte() % 5); + for (auto it = array.rbegin(); it != array.rend(); ++it) + { + pending.push_back(&*it); + } + break; + } + case 8: // Map + { + *current = json::object(); + auto& object = current->get_ref(); + const size_t size = input.byte() % 5; + for (size_t i = 0; i < size; ++i) + { + const auto key = + std::to_string(i) + static_cast('a' + input.byte() % 26); + object[key] = nullptr; + } + for (auto it = object.rbegin(); it != object.rend(); ++it) + { + pending.push_back(&it->second); + } + break; + } + default: + __builtin_unreachable(); + } + } + + return result; + } +} + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) +{ + Input input(data, size); + const auto expected = generate_value(input); + + std::vector encoded; + ccf::msgpack::test::encode_json(encoded, expected); + + if (json::from_msgpack(encoded) != expected) + { + __builtin_trap(); + } + + return 0; +} diff --git a/src/msgpack/test/msgpack_fuzz_corpus/array b/src/msgpack/test/msgpack_fuzz_corpus/array new file mode 100644 index 00000000000..c7780627226 --- /dev/null +++ b/src/msgpack/test/msgpack_fuzz_corpus/array @@ -0,0 +1 @@ +j4cdefghijklmnop diff --git a/src/msgpack/test/msgpack_fuzz_corpus/binary b/src/msgpack/test/msgpack_fuzz_corpus/binary new file mode 100644 index 00000000000..1fe0ddeb1be --- /dev/null +++ b/src/msgpack/test/msgpack_fuzz_corpus/binary @@ -0,0 +1 @@ +i boundary diff --git a/src/msgpack/test/msgpack_fuzz_corpus/bool b/src/msgpack/test/msgpack_fuzz_corpus/bool new file mode 100644 index 00000000000..6f1852975b9 --- /dev/null +++ b/src/msgpack/test/msgpack_fuzz_corpus/bool @@ -0,0 +1 @@ +d1 diff --git a/src/msgpack/test/msgpack_fuzz_corpus/float b/src/msgpack/test/msgpack_fuzz_corpus/float new file mode 100644 index 00000000000..815d72e623f --- /dev/null +++ b/src/msgpack/test/msgpack_fuzz_corpus/float @@ -0,0 +1 @@ +gabcdefgh diff --git a/src/msgpack/test/msgpack_fuzz_corpus/int b/src/msgpack/test/msgpack_fuzz_corpus/int new file mode 100644 index 00000000000..d4edc15a94e --- /dev/null +++ b/src/msgpack/test/msgpack_fuzz_corpus/int @@ -0,0 +1 @@ +fabcdefgh diff --git a/src/msgpack/test/msgpack_fuzz_corpus/map b/src/msgpack/test/msgpack_fuzz_corpus/map new file mode 100644 index 00000000000..a6b5d756584 --- /dev/null +++ b/src/msgpack/test/msgpack_fuzz_corpus/map @@ -0,0 +1 @@ +k4abcdefghijklmnopqrstuvwxyz diff --git a/src/msgpack/test/msgpack_fuzz_corpus/null b/src/msgpack/test/msgpack_fuzz_corpus/null new file mode 100644 index 00000000000..f2ad6c76f01 --- /dev/null +++ b/src/msgpack/test/msgpack_fuzz_corpus/null @@ -0,0 +1 @@ +c diff --git a/src/msgpack/test/msgpack_fuzz_corpus/string b/src/msgpack/test/msgpack_fuzz_corpus/string new file mode 100644 index 00000000000..436d7eb4d38 --- /dev/null +++ b/src/msgpack/test/msgpack_fuzz_corpus/string @@ -0,0 +1 @@ +h boundary diff --git a/src/msgpack/test/msgpack_fuzz_corpus/uint b/src/msgpack/test/msgpack_fuzz_corpus/uint new file mode 100644 index 00000000000..ddcb8e72e20 --- /dev/null +++ b/src/msgpack/test/msgpack_fuzz_corpus/uint @@ -0,0 +1 @@ +eabcdefgh diff --git a/src/msgpack/test/msgpack_unit.cpp b/src/msgpack/test/msgpack_unit.cpp new file mode 100644 index 00000000000..727c9c556e3 --- /dev/null +++ b/src/msgpack/test/msgpack_unit.cpp @@ -0,0 +1,606 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN +#include "msgpack/encode.h" +#include "msgpack/fluentd_event_time.h" +#include "msgpack/test/format_introspect.h" +#include "msgpack/test/json.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ccf::msgpack; +using ccf::msgpack::test::classify_first_byte; +using ccf::msgpack::test::encode_json; +using ccf::msgpack::test::FormatFamily; +using nlohmann::json; + +namespace +{ + // Decode a single big-endian integer from buf at offset. Accumulate into + // uint64_t and narrow at the end so the shift never executes at the result + // type's width. + template + T decode_be(const std::vector& buf, size_t offset) + { + static_assert(std::is_unsigned_v); + uint64_t acc = 0; + for (size_t i = 0; i < sizeof(T); ++i) + { + acc = (acc << 8) | static_cast(buf[offset + i]); + } + return static_cast(acc); + } +} + +// ===== write_uint: smallest-format-wins ===== + +TEST_CASE("write_uint boundary table") +{ + struct Row + { + uint64_t v; + std::vector expected; + FormatFamily family; + }; + const Row rows[] = { + {0, {0x00}, FormatFamily::POSITIVE_FIXINT}, + {127, {0x7F}, FormatFamily::POSITIVE_FIXINT}, + {128, {0xCC, 0x80}, FormatFamily::UINT_8}, + {255, {0xCC, 0xFF}, FormatFamily::UINT_8}, + {256, {0xCD, 0x01, 0x00}, FormatFamily::UINT_16}, + {65535, {0xCD, 0xFF, 0xFF}, FormatFamily::UINT_16}, + {65536, {0xCE, 0x00, 0x01, 0x00, 0x00}, FormatFamily::UINT_32}, + {0xFFFFFFFFULL, {0xCE, 0xFF, 0xFF, 0xFF, 0xFF}, FormatFamily::UINT_32}, + {0x100000000ULL, + {0xCF, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00}, + FormatFamily::UINT_64}, + {std::numeric_limits::max(), + {0xCF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + FormatFamily::UINT_64}, + }; + for (const auto& r : rows) + { + CAPTURE(r.v); + std::vector buf; + write_uint(buf, r.v); + CHECK(buf == r.expected); + CHECK(classify_first_byte(buf[0]) == r.family); + } +} + +TEST_CASE("tagged writes append after existing bytes") +{ + std::vector buf = {0xAA, 0xBB}; + + write_uint(buf, 0x0102); + write_array_header(buf, 16); + write_map_header(buf, 16); + + CHECK( + buf == + std::vector{ + 0xAA, 0xBB, 0xCD, 0x01, 0x02, 0xDC, 0x00, 0x10, 0xDE, 0x00, 0x10}); +} + +// ===== write_int: smallest-format-wins, non-negative delegates ===== + +TEST_CASE("write_int delegates to write_uint for non-negative") +{ + std::vector a; + std::vector b; + write_int(a, 5); + write_uint(b, 5); + CHECK(a == b); + + std::vector c; + std::vector d; + write_int(c, 0); + write_uint(d, 0); + CHECK(c == d); + + std::vector e; + std::vector f; + write_int(e, 1234567); + write_uint(f, 1234567); + CHECK(e == f); +} + +TEST_CASE("write_int negative boundary table") +{ + struct Row + { + int64_t v; + uint8_t first; + size_t size; + FormatFamily family; + }; + const Row rows[] = { + {-1, 0xFF, 1, FormatFamily::NEGATIVE_FIXINT}, + {-32, 0xE0, 1, FormatFamily::NEGATIVE_FIXINT}, + {-33, 0xD0, 2, FormatFamily::INT_8}, + {-128, 0xD0, 2, FormatFamily::INT_8}, + {-129, 0xD1, 3, FormatFamily::INT_16}, + {-32768, 0xD1, 3, FormatFamily::INT_16}, + {-32769, 0xD2, 5, FormatFamily::INT_32}, + {std::numeric_limits::min(), 0xD2, 5, FormatFamily::INT_32}, + {static_cast(std::numeric_limits::min()) - 1, + 0xD3, + 9, + FormatFamily::INT_64}, + {std::numeric_limits::min(), 0xD3, 9, FormatFamily::INT_64}, + }; + for (const auto& r : rows) + { + CAPTURE(r.v); + std::vector buf; + write_int(buf, r.v); + CHECK(buf.size() == r.size); + CHECK(buf[0] == r.first); + CHECK(classify_first_byte(buf[0]) == r.family); + + // Decode the payload back to int64_t and check round-trip equality. + // This catches wrong-width writes (e.g. zero-extending a negative + // value) and absolute-value bugs that the family + size checks + // alone miss. + int64_t decoded = 0; + switch (r.size) + { + case 1: + // fixint: the byte itself is the int8 bit pattern (negative + // fixint range is 0xE0..0xFF, which sign-extends correctly). + decoded = static_cast(buf[0]); + break; + case 2: + decoded = static_cast(buf[1]); + break; + case 3: + decoded = static_cast(decode_be(buf, 1)); + break; + case 5: + decoded = static_cast(decode_be(buf, 1)); + break; + case 9: + decoded = static_cast(decode_be(buf, 1)); + break; + default: + FAIL("unexpected encoded size for write_int row"); + } + CHECK(decoded == r.v); + } +} + +// ===== write_str ===== + +TEST_CASE("write_str boundary table") +{ + // Each row exercises both sides of a format-family boundary. + struct Row + { + size_t n; + uint8_t first; + size_t header_size; + FormatFamily family; + }; + const Row rows[] = { + {0, 0xA0, 1, FormatFamily::FIXSTR}, + {31, 0xBF, 1, FormatFamily::FIXSTR}, + {32, 0xD9, 2, FormatFamily::STR_8}, + {255, 0xD9, 2, FormatFamily::STR_8}, + {256, 0xDA, 3, FormatFamily::STR_16}, + {65535, 0xDA, 3, FormatFamily::STR_16}, + {65536, 0xDB, 5, FormatFamily::STR_32}, + {70000, 0xDB, 5, FormatFamily::STR_32}, + }; + for (const auto& r : rows) + { + CAPTURE(r.n); + // Position-dependent fill so any payload corruption (bit flip, + // zeroing, off-by-one) shows up as a byte-compare mismatch. + std::string s(r.n, '\0'); + for (size_t i = 0; i < r.n; ++i) + { + s[i] = static_cast((i * 7 + 13) & 0xFF); + } + std::vector buf; + write_str(buf, s); + CHECK(buf.size() == r.header_size + r.n); + CHECK(buf[0] == r.first); + CHECK(classify_first_byte(buf[0]) == r.family); + if (r.n > 0) + { + CHECK(std::memcmp(buf.data() + r.header_size, s.data(), r.n) == 0); + } + } +} + +TEST_CASE("write_str supports payloads stored in the destination buffer") +{ + std::vector buf = {'t', 'e', 's', 't'}; + buf.shrink_to_fit(); + buf.resize(buf.capacity(), 0xCC); + auto expected = buf; + expected.insert(expected.end(), {0xA4, 't', 'e', 's', 't'}); + const auto* const original_data = buf.data(); + const std::string_view s{reinterpret_cast(buf.data()), 4}; + + write_str(buf, s); + + CHECK(buf.data() != original_data); + CHECK(buf == expected); +} + +// ===== write_bool, write_nil ===== + +TEST_CASE("write_bool and write_nil produce single byte") +{ + std::vector buf; + write_nil(buf); + CHECK(buf == std::vector{0xC0}); + + buf.clear(); + write_bool(buf, true); + CHECK(buf == std::vector{0xC3}); + + buf.clear(); + write_bool(buf, false); + CHECK(buf == std::vector{0xC2}); +} + +// ===== write_float ===== + +// ===== write_bin ===== + +TEST_CASE("write_bin length prefix and family") +{ + // Boundary table only - generator coverage overlaps with str. + struct Row + { + size_t n; + uint8_t first; + size_t header_size; + FormatFamily family; + }; + const Row rows[] = { + {0, 0xC4, 2, FormatFamily::BIN_8}, + {1, 0xC4, 2, FormatFamily::BIN_8}, + {255, 0xC4, 2, FormatFamily::BIN_8}, + {256, 0xC5, 3, FormatFamily::BIN_16}, + {65535, 0xC5, 3, FormatFamily::BIN_16}, + {65536, 0xC6, 5, FormatFamily::BIN_32}, + {70000, 0xC6, 5, FormatFamily::BIN_32}, + }; + for (const auto& r : rows) + { + CAPTURE(r.n); + // Position-dependent fill so any payload corruption (bit flip, + // zeroing, off-by-one) shows up as a byte-compare mismatch. + std::vector data(r.n); + for (size_t i = 0; i < r.n; ++i) + { + data[i] = static_cast((i * 13 + 7) & 0xFF); + } + std::vector buf; + write_bin(buf, data); + CHECK(buf.size() == r.header_size + r.n); + CHECK(buf[0] == r.first); + CHECK(classify_first_byte(buf[0]) == r.family); + if (r.n > 0) + { + CHECK(std::memcmp(buf.data() + r.header_size, data.data(), r.n) == 0); + } + } +} + +TEST_CASE("write_bin supports payloads stored in the destination buffer") +{ + std::vector buf = {0xDE, 0xAD, 0xBE, 0xEF}; + buf.shrink_to_fit(); + + write_bin(buf, std::span{buf}); + + CHECK( + buf == + std::vector{ + 0xDE, 0xAD, 0xBE, 0xEF, 0xC4, 0x04, 0xDE, 0xAD, 0xBE, 0xEF}); +} + +// ===== container headers ===== + +TEST_CASE("write_array_header boundary table") +{ + struct Row + { + uint32_t n; + uint8_t first; + size_t size; + FormatFamily family; + }; + const Row rows[] = { + {0, 0x90, 1, FormatFamily::FIXARRAY}, + {15, 0x9F, 1, FormatFamily::FIXARRAY}, + {16, 0xDC, 3, FormatFamily::ARRAY_16}, + {65535, 0xDC, 3, FormatFamily::ARRAY_16}, + {65536, 0xDD, 5, FormatFamily::ARRAY_32}, + {std::numeric_limits::max(), 0xDD, 5, FormatFamily::ARRAY_32}, + }; + for (const auto& r : rows) + { + CAPTURE(r.n); + std::vector buf; + write_array_header(buf, r.n); + CHECK(buf.size() == r.size); + CHECK(buf[0] == r.first); + CHECK(classify_first_byte(buf[0]) == r.family); + } +} + +TEST_CASE("write_map_header boundary table") +{ + struct Row + { + uint32_t n; + std::vector expected; + FormatFamily family; + }; + const Row rows[] = { + {0, {0x80}, FormatFamily::FIXMAP}, + {15, {0x8F}, FormatFamily::FIXMAP}, + {16, {0xDE, 0x00, 0x10}, FormatFamily::MAP_16}, + {65535, {0xDE, 0xFF, 0xFF}, FormatFamily::MAP_16}, + {65536, {0xDF, 0x00, 0x01, 0x00, 0x00}, FormatFamily::MAP_32}, + {std::numeric_limits::max(), + {0xDF, 0xFF, 0xFF, 0xFF, 0xFF}, + FormatFamily::MAP_32}, + }; + for (const auto& r : rows) + { + CAPTURE(r.n); + std::vector buf; + write_map_header(buf, r.n); + CHECK(buf == r.expected); + CHECK(classify_first_byte(buf[0]) == r.family); + if (r.family == FormatFamily::MAP_16) + { + CHECK(decode_be(buf, 1) == r.n); + } + else if (r.family == FormatFamily::MAP_32) + { + CHECK(decode_be(buf, 1) == r.n); + } + } +} + +// ===== FluentdEventTime: time_point boundary ===== +// +// make() takes a system_clock::time_point and rejects: +// - time_points before the epoch (negative since_epoch), +// - time_points beyond UINT32_MAX seconds since epoch. +// The valid-input range and rejection boundaries are listed explicitly. + +namespace +{ + using time_point = std::chrono::system_clock::time_point; + + // Build a time_point from raw (seconds, nanoseconds) since epoch. + // Used to pin specific wire-format byte patterns in the byte-shape + // tests; not the production way to construct a FluentdEventTime. + time_point tp_from_components(int64_t secs_since_epoch, uint32_t ns_remainder) + { + using namespace std::chrono; + return time_point{seconds{secs_since_epoch} + nanoseconds{ns_remainder}}; + } +} + +TEST_CASE("FluentdEventTime::make boundary table") +{ + struct Row + { + int64_t seconds; + uint32_t nanoseconds; + bool valid; + }; + const Row rows[] = { + {-1, 0, false}, + {-1, 999'999'999U, false}, + {0, 0, true}, + {1, 999'999'999U, true}, + {1700000000, 123456789U, true}, + {static_cast(std::numeric_limits::max()), + 999'999'999U, + true}, + {static_cast(std::numeric_limits::max()) + 1, 0, false}, + }; + + for (const auto& row : rows) + { + CAPTURE(row.seconds); + CAPTURE(row.nanoseconds); + bool threw = false; + try + { + const auto et = FluentdEventTime::make( + tp_from_components(row.seconds, row.nanoseconds)); + CHECK(et.seconds() == static_cast(row.seconds)); + CHECK(et.nanoseconds() == row.nanoseconds); + } + catch (const MsgpackEncodeError& e) + { + threw = true; + CHECK(e.error_code() == Error::INVALID_EVENT_TIME); + } + CHECK(threw != row.valid); + } +} + +TEST_CASE("write_fluentd_event_time byte shape") +{ + // Spec (fluentd Forward Protocol v1, EventTime ext type 0, fixext8 + // form): 0xD7 0x00 . + // Concrete value chosen so the bytes contain non-trivial bit patterns + // in every position; any byte-order or layout regression flips at + // least one of these. + const auto et = + FluentdEventTime::make(tp_from_components(0x69F37C9FLL, 0x315B5B4CU)); + std::vector buf{0xAA, 0xBB}; + write_fluentd_event_time(buf, et); + const std::vector expected{ + 0xAA, 0xBB, 0xD7, 0x00, 0x69, 0xF3, 0x7C, 0x9F, 0x31, 0x5B, 0x5B, 0x4C}; + CHECK(buf == expected); +} + +TEST_CASE("known nested values roundtrip through nlohmann") +{ + const std::vector samples = { + json::array({1, "two", 3.0, nullptr, true}), + json::object({{"a", 1}, {"b", "two"}, {"c", json::array({1, 2, 3})}}), + json::object( + {{"nested", + json::array( + {json::object({{"enabled", true}}), json::binary({0, 1, 255})})}}), + }; + for (const auto& value : samples) + { + std::vector buf; + encode_json(buf, value); + CHECK(json::from_msgpack(buf) == value); + } +} + +TEST_CASE("encode_json handles deeply nested values iteratively") +{ + constexpr size_t DEPTH = 4096; + json value = nullptr; + for (size_t i = 0; i < DEPTH; ++i) + { + value = json::array({std::move(value)}); + } + + std::vector buf; + encode_json(buf, value); + + REQUIRE(buf.size() == DEPTH + 1); + CHECK(std::all_of( + buf.begin(), buf.end() - 1, [](uint8_t byte) { return byte == 0x91; })); + CHECK(buf.back() == 0xC0); +} + +TEST_CASE("FluentdEventTime roundtrips through nlohmann") +{ + const auto event_time = + FluentdEventTime::make(tp_from_components(1700000000LL, 123456789U)); + std::vector buf; + write_fluentd_event_time(buf, event_time); + + const auto decoded = json::from_msgpack(buf); + REQUIRE(decoded.is_binary()); + const auto& binary = decoded.get_binary(); + CHECK(binary.has_subtype()); + CHECK(binary.subtype() == 0); + REQUIRE(binary.size() == 8); + + const uint32_t seconds = (uint32_t(binary[0]) << 24) | + (uint32_t(binary[1]) << 16) | (uint32_t(binary[2]) << 8) | + uint32_t(binary[3]); + const uint32_t nanoseconds = (uint32_t(binary[4]) << 24) | + (uint32_t(binary[5]) << 16) | (uint32_t(binary[6]) << 8) | + uint32_t(binary[7]); + CHECK(seconds == event_time.seconds()); + CHECK(nanoseconds == event_time.nanoseconds()); +} + +TEST_CASE("fluentd Message-mode byte-for-byte vector") +{ + const std::vector expected = { + 0x93, 0xAC, 0x6D, 0x79, 0x61, 0x70, 0x70, 0x2E, 0x61, 0x63, 0x63, + 0x65, 0x73, 0x73, 0xD7, 0x00, 0x69, 0xF3, 0x7C, 0x9F, 0x31, 0x5B, + 0x5B, 0x4C, 0x83, 0xA4, 0x70, 0x61, 0x74, 0x68, 0xAB, 0x2F, 0x61, + 0x70, 0x69, 0x2F, 0x76, 0x31, 0x2F, 0x66, 0x6F, 0x6F, 0xA6, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0xCC, 0xC8, 0xA2, 0x6D, 0x73, 0xCB, + 0x40, 0x28, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9A}; + + std::vector buf; + write_array_header(buf, 3); + write_str(buf, "myapp.access"); + write_fluentd_event_time( + buf, FluentdEventTime::make(tp_from_components(0x69F37C9FLL, 0x315B5B4CU))); + write_map_header(buf, 3); + write_str(buf, "path"); + write_str(buf, "/api/v1/foo"); + write_str(buf, "status"); + write_uint(buf, 200); + write_str(buf, "ms"); + write_float(buf, 12.3); + + CHECK(buf == expected); +} + +// ===== write_float: non-finite bit-patterns pass through ===== + +TEST_CASE("write_float passes through non-finite bit-patterns unchanged") +{ + // The encoder doc states NaN / +/-inf / signalling-NaN are emitted + // verbatim with no canonicalisation. Round-trip the bit pattern + // through encode and back-decode; bytes 1..9 must equal the input + // bits exactly. + struct Row + { + uint64_t bits; + const char* label; + }; + const Row rows[] = { + {0x7FF8000000000000ULL, "quiet NaN"}, + {0x7FF0000000000001ULL, "signalling NaN"}, + {0x7FF0000000000000ULL, "+inf"}, + {0xFFF0000000000000ULL, "-inf"}, + {0x8000000000000000ULL, "negative zero"}, + {0x0000000000000000ULL, "positive zero"}, + }; + for (const auto& r : rows) + { + CAPTURE(r.label); + double v; + std::memcpy(&v, &r.bits, sizeof(v)); + std::vector buf; + write_float(buf, v); + REQUIRE(buf.size() == 9); + CHECK(buf[0] == 0xCB); + CHECK(decode_be(buf, 1) == r.bits); + } +} + +// ===== to_string(Error) ===== + +TEST_CASE("to_string(Error) maps every enumerator to a unique stable label") +{ + // Each enum value must produce its own non-empty label; a swap or + // typo in the switch would collapse two distinct codes to the same + // string and would be caught here. + const Error all[] = { + Error::STRING_TOO_LARGE, + Error::BIN_TOO_LARGE, + Error::INVALID_EVENT_TIME, + }; + std::vector seen; + for (const auto e : all) + { + const auto s = to_string(e); + CHECK_FALSE(s.empty()); + for (const auto& prev : seen) + { + CHECK(prev != s); + } + seen.push_back(s); + } + + // Spot-check a couple of specific labels so a future rename of an + // enumerator name in the switch is caught here too. + CHECK(to_string(Error::STRING_TOO_LARGE) == "STRING_TOO_LARGE"); + CHECK(to_string(Error::INVALID_EVENT_TIME) == "INVALID_EVENT_TIME"); +} diff --git a/tests/ci-buckets.txt b/tests/ci-buckets.txt index c21da141b93..84cb69ef81a 100644 --- a/tests/ci-buckets.txt +++ b/tests/ci-buckets.txt @@ -30,6 +30,7 @@ no_bucket: map_bench logger_bench json_bench + msgpack_bench ring_buffer_bench ledger_bench crypto_bench