rpcdaemon: use zlib for streaming (#2871)

Author: lupin012

.github/actions/perf-common-steps/action.yml

@@ -182,6 +182,7 @@ runs:
           run_perf mainnet eth_getTransactionByHash 5 stress_test_eth_getTransactionByHash_13M 1:1,100:30,1000:20,10000:20
           run_perf mainnet eth_getTransactionReceipt 5 stress_test_eth_getTransactionReceipt_14M 1:1,100:30,1000:20,5000:20,10000:20,20000:20
           run_perf mainnet eth_createAccessList 5 stress_test_eth_createAccessList_16M 1:1,100:30,1000:20,10000:20,20000:20
+          run_perf mainnet debug_traceTransaction 5 stress_test_debug_trace_transaction_21M.tar 1:1,100:30,1000:20,2000:20
         fi
         
         if [ $failed_test -eq 0 ]; then

silkworm/rpc/core/evm_debug.cpp

@@ -59,30 +59,32 @@ std::string DebugConfig::to_string() const {
 
     return out.str();
 }
-
 std::string uint256_to_hex(const evmone::uint256& x) {
-    std::stringstream ss;
-    ss << "0x";
+    static constexpr std::string_view kHexDigits{"0123456789abcdef"};
+    std::string out(2 + 64, '\0');
+    char* dest = out.data();
+
+    *dest++ = '0';
+    *dest++ = 'x';
 
     bool leading_zeros = true;
     const uint64_t* px = &x[0];
     for (int i = 3; i >= 0; --i) {
-        if (px[i] == 0 && leading_zeros) {
-            continue;
-        }
-        if (leading_zeros) {
-            ss << std::hex << px[i];
-            leading_zeros = false;
-        } else {
-            ss << std::setfill('0') << std::setw(16) << std::hex << px[i];
+        for (int shift = 60; shift >= 0; shift -= 4) {
+            char hex_digit = kHexDigits[(px[i] >> shift) & 0xF];
+            if (hex_digit != '0' || !leading_zeros) {
+                *dest++ = hex_digit;
+                leading_zeros = false;
+            }
         }
     }
 
     if (leading_zeros) {
-        ss << "0";
+        *dest++ = '0';
     }
 
-    return ss.str();
+    out.resize(static_cast<size_t>(dest - out.data()));
+    return out;
 }
 
 static void output_stack(std::vector<std::string>& vect, const evmone::uint256* stack, int stack_size) {

silkworm/rpc/http/chunker.hpp

@@ -4,8 +4,8 @@
 #pragma once
 
 #include <algorithm>
+#include <deque>
 #include <memory>
-#include <queue>
 
 namespace silkworm::rpc::http {
 
@@ -27,28 +27,27 @@ class Chunker {
         // creates chunk: even if new:buffer is greater kDefaultMaxChunkSize
         while (position < new_buffer.size()) {
             size_t available_space = kDefaultMaxChunkSize - current_chunk_.size();
-            size_t chunk_size = std::min(available_space, new_buffer.size() - position);
+            const size_t chunk_size = std::min(available_space, new_buffer.size() - position);
 
             current_chunk_.append(new_buffer, position, chunk_size);
             position += chunk_size;
 
-            // one chunk is completed copy it in complet_chunk
-            if (current_chunk_.size() == kDefaultMaxChunkSize) {
-                complete_chunk_.push(current_chunk_);
+            // one chunk is completed copy it in complete_chunk queue
+            if (current_chunk_.size() >= kDefaultMaxChunkSize) {
+                complete_chunk_.push_back(std::move(current_chunk_));
                 current_chunk_.clear();
-                current_chunk_.reserve(kDefaultMaxChunkSize);
             }
         }
     }
 
     std::pair<std::string, bool> get_complete_chunk() {
         if (!complete_chunk_.empty()) {
-            // at least one chunk is availble return it , indicating if first chunk or not
+            // at least one chunk is available return it , indicating if first chunk or not
             auto ret_first_chunk = !first_chunk_completed_;
             first_chunk_completed_ = true;
-            std::string chunk = complete_chunk_.front();
-            complete_chunk_.pop();
-            return std::make_pair(chunk, ret_first_chunk);
+            std::string chunk = std::move(complete_chunk_.front());
+            complete_chunk_.pop_front();
+            return std::make_pair(std::move(chunk), ret_first_chunk);
         }
         // queue is empty no chunk are available
         return std::make_pair("", false);
@@ -58,18 +57,18 @@ class Chunker {
         return !complete_chunk_.empty();
     }
 
-    std::pair<std::string, bool> get_remainder() const {
+    std::pair<std::string, bool> get_remainder() {
         if (current_chunk_.empty()) {
             // no bytes are present on current_chunk so return empty string and indication if first chunk or not
             // we are in two possible cases: at least one completed chunk is already produced, or any chunk are produced
             return std::make_pair("", !first_chunk_completed_);
         }
         // returns the chunk
-        return std::make_pair(current_chunk_, !first_chunk_completed_);
+        return std::make_pair(std::move(current_chunk_), !first_chunk_completed_);
     }
 
   private:
-    std::queue<std::string> complete_chunk_;
+    std::deque<std::string> complete_chunk_;
     std::string current_chunk_;
     bool first_chunk_completed_{false};
 };

silkworm/rpc/http/connection.cpp

@@ -268,6 +268,10 @@ Task<void> Connection::open_stream(uint64_t request_id) {
 
     // add chunking supports
     request_data.chunk = std::make_unique<Chunker>();
+    if (request_data.gzip_encoding_requested) {
+        request_data.zlib_compressor = std::make_unique<ZlibCompressor>();
+    }
+    request_data.chunk = std::make_unique<Chunker>();
 
     co_return;
 }
@@ -557,4 +561,10 @@ Task<void> Connection::compress(const std::string& clear_data, std::string& comp
     });
 }
 
+Task<void> Connection::compress_stream(const std::string& clear_data, std::string& compressed_data, const RequestData& req_data, bool last) {
+    co_await async_task(workers_.executor(), [&]() -> void {
+        req_data.zlib_compressor->compress_chunk(clear_data, compressed_data, last);
+    });
+}
+
 }  // namespace silkworm::rpc::http

silkworm/rpc/http/connection.hpp

@@ -20,6 +20,7 @@
 #include <silkworm/rpc/common/interface_log.hpp>
 #include <silkworm/rpc/common/worker_pool.hpp>
 #include <silkworm/rpc/http/chunker.hpp>
+#include <silkworm/rpc/http/zlib_compressor.hpp>
 #include <silkworm/rpc/transport/request_handler.hpp>
 #include <silkworm/rpc/transport/stream_writer.hpp>
 #include <silkworm/rpc/ws/connection.hpp>
@@ -28,7 +29,7 @@ namespace silkworm::rpc::http {
 
 using RequestWithStringBody = boost::beast::http::request<boost::beast::http::string_body>;
 
-inline constexpr size_t kDefaultCapacity = 4 * 1024;
+inline constexpr size_t kDefaultCapacity = 1 * 1024 * 1024;
 
 struct RequestData {
     bool request_keep_alive{false};
@@ -38,6 +39,7 @@ struct RequestData {
     std::string origin;
     boost::beast::http::verb method{boost::beast::http::verb::unknown};
     std::unique_ptr<Chunker> chunk;
+    std::unique_ptr<ZlibCompressor> zlib_compressor;
 };
 
 //! Represents a single connection from a client.
@@ -98,6 +100,7 @@ class Connection : public StreamWriter {
     static std::string get_date_time();
 
     Task<void> compress(const std::string& clear_data, std::string& compressed_data);
+    Task<void> compress_stream(const std::string& clear_data, std::string& compressed_data, const RequestData& req_data, bool last);
     Task<void> create_chunk_header(RequestData& request_data);
     Task<size_t> send_chunk(const std::string& content);
 

silkworm/rpc/http/zlib_compressor.hpp

@@ -0,0 +1,64 @@
+// Copyright 2025 The Silkworm Authors
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include <zlib.h>
+
+#include <memory>
+#include <stdexcept>
+#include <string>
+
+namespace silkworm::rpc::http {
+
+inline constexpr int kZlibCompressionBufferSize = 65536;
+
+class ZlibCompressor {
+  public:
+    ZlibCompressor(const ZlibCompressor&) = delete;
+
+    ZlibCompressor() {
+        memset(&stream_, 0, sizeof(z_stream));
+        stream_.zalloc = Z_NULL;
+        stream_.zfree = Z_NULL;
+        stream_.opaque = Z_NULL;
+
+        if (deflateInit2(&stream_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15 + 16, 8, Z_DEFAULT_STRATEGY) != Z_OK) {
+            throw std::runtime_error("zlib initialization error");
+        }
+    }
+
+    ~ZlibCompressor() {
+        deflateEnd(&stream_);
+    }
+
+    void compress_chunk(const std::string& clear_data, std::string& compressed_data, const bool flush) {
+        stream_.next_in = const_cast<Bytef*>(reinterpret_cast<const Bytef*>(clear_data.data()));
+        stream_.avail_in = static_cast<unsigned int>(clear_data.size());
+        size_t offset = 0;
+        do {
+            compressed_data.resize(kZlibCompressionBufferSize + offset);
+
+            stream_.next_out = reinterpret_cast<Bytef*>(compressed_data.data() + offset);
+            stream_.avail_out = kZlibCompressionBufferSize;
+
+            const int ret = deflate(&stream_, flush ? Z_FINISH : Z_NO_FLUSH);
+            if (ret == Z_STREAM_ERROR) {
+                throw std::runtime_error("zlib compression error");
+            }
+            offset += kZlibCompressionBufferSize - stream_.avail_out;
+
+            if (flush && ret == Z_STREAM_END) {
+                break;
+            }
+
+        } while (stream_.avail_in > 0 || stream_.avail_out == 0);
+
+        compressed_data.resize(offset);
+    }
+
+  private:
+    z_stream stream_;
+};
+
+}  // namespace silkworm::rpc::http