perf: reuse HPACK encode buffer across frames (#929)

`into_encoding` allocated a fresh `BytesMut` for every HEADERS and
PUSH_PROMISE frame, grew it through several capacity doublings while HPACK
encoded into it, then called `.freeze()` — allocating again for the `Bytes`
refcount header. That is at least two allocations per outbound header block,
on every request and every response.

The block does not need to outlive the call: on the common path it is copied
into the connection write buffer and dropped immediately. So `hpack::Encoder`
now keeps one scratch buffer that `into_encoding` takes and `encode` hands
back once the block has been written.

`EncodingHeaderBlock::hpack` becomes `BytesMut`, which removes the
`.freeze()` as well. The CONTINUATION path keeps its remainder instead of
returning the buffer, and splitting that remainder is now `split_to` on a
uniquely-owned buffer rather than a copy.

Same bytes on the wire; no public API change.

`cargo bench --bench main`, 5 interleaved rounds per arm: multi-thread 100k
requests improves 3.8 % (faster in 24 of 25 pairwise comparisons, exact
permutation test p = 0.016), current-thread 1.5 %, and the write-contention
benchmark is unchanged as expected.

Author: dorianverlaine

src/frame/headers.rs

@@ -6,7 +6,7 @@ use crate::hpack::{self, BytesStr};
 use http::header::{self, HeaderName, HeaderValue};
 use http::{uri, HeaderMap, Method, Request, StatusCode, Uri};
 
-use bytes::{Buf, BufMut, Bytes, BytesMut};
+use bytes::{Buf, BufMut, BytesMut};
 
 use std::fmt;
 use std::io::Cursor;
@@ -104,7 +104,7 @@ struct HeaderBlock {
 
 #[derive(Debug)]
 struct EncodingHeaderBlock {
-    hpack: Bytes,
+    hpack: BytesMut,
 }
 
 const END_STREAM: u8 = 0x1;
@@ -287,7 +287,7 @@ impl Headers {
 
         self.header_block
             .into_encoding(encoder)
-            .encode(&head, dst, |_| {})
+            .encode(&head, dst, Some(encoder), |_| {})
     }
 
     fn head(&self) -> Head {
@@ -508,7 +508,7 @@ impl PushPromise {
 
         self.header_block
             .into_encoding(encoder)
-            .encode(&head, dst, |dst| {
+            .encode(&head, dst, Some(encoder), |dst| {
                 dst.put_u32(promised_id.into());
             })
     }
@@ -551,7 +551,7 @@ impl Continuation {
         // Get the CONTINUATION frame head
         let head = self.head();
 
-        self.header_block.encode(&head, dst, |_| {})
+        self.header_block.encode(&head, dst, None, |_| {})
     }
 }
 
@@ -647,7 +647,13 @@ impl Pseudo {
 // ===== impl EncodingHeaderBlock =====
 
 impl EncodingHeaderBlock {
-    fn encode<F>(mut self, head: &Head, dst: &mut EncodeBuf<'_>, f: F) -> Option<Continuation>
+    fn encode<F>(
+        mut self,
+        head: &Head,
+        dst: &mut EncodeBuf<'_>,
+        encoder: Option<&mut hpack::Encoder>,
+        f: F,
+    ) -> Option<Continuation>
     where
         F: FnOnce(&mut EncodeBuf<'_>),
     {
@@ -664,14 +670,20 @@ impl EncodingHeaderBlock {
 
         // Now, encode the header payload
         let continuation = if self.hpack.len() > dst.remaining_mut() {
-            dst.put((&mut self.hpack).take(dst.remaining_mut()));
+            let head_part = self.hpack.split_to(dst.remaining_mut());
+            dst.put_slice(&head_part);
 
             Some(Continuation {
                 stream_id: head.stream_id(),
                 header_block: self,
             })
         } else {
             dst.put_slice(&self.hpack);
+            // The block is fully written, so the buffer can be reused by the
+            // next frame on this connection.
+            if let Some(encoder) = encoder {
+                encoder.return_scratch(self.hpack);
+            }
 
             None
         };
@@ -978,17 +990,16 @@ impl HeaderBlock {
     }
 
     fn into_encoding(self, encoder: &mut hpack::Encoder) -> EncodingHeaderBlock {
-        let mut hpack = BytesMut::new();
+        let mut hpack = encoder.take_scratch();
+        hpack.clear();
         let headers = Iter {
             pseudo: Some(self.pseudo),
             fields: self.fields.into_iter(),
         };
 
         encoder.encode(headers, &mut hpack);
 
-        EncodingHeaderBlock {
-            hpack: hpack.freeze(),
-        }
+        EncodingHeaderBlock { hpack }
     }
 
     /// Calculates the size of the currently decoded header list.

src/hpack/encoder.rs

@@ -8,6 +8,13 @@ use http::header::{HeaderName, HeaderValue};
 pub struct Encoder {
     table: Table,
     size_update: Option<SizeUpdate>,
+    /// Reusable buffer for the encoded header block of a single frame.
+    ///
+    /// A header block only has to live until it is copied into the
+    /// connection write buffer, so the buffer that holds it can be reused
+    /// across frames instead of being allocated and freed per frame. See
+    /// `take_scratch` / `return_scratch`.
+    scratch: BytesMut,
 }
 
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
@@ -21,9 +28,25 @@ impl Encoder {
         Encoder {
             table: Table::new(max_size, capacity),
             size_update: None,
+            scratch: BytesMut::new(),
         }
     }
 
+    /// Takes the reusable buffer for one header-block encode.
+    ///
+    /// The buffer is returned by `return_scratch` once the encoded block has
+    /// been copied into the write buffer. If it is not returned - the
+    /// CONTINUATION path keeps it, since the remainder is still needed - the
+    /// next call simply starts from a fresh buffer.
+    pub(crate) fn take_scratch(&mut self) -> BytesMut {
+        std::mem::take(&mut self.scratch)
+    }
+
+    /// Returns a fully-written header block buffer for reuse.
+    pub(crate) fn return_scratch(&mut self, scratch: BytesMut) {
+        self.scratch = scratch;
+    }
+
     /// Queues a max size update.
     ///
     /// The next call to `encode` will include a dynamic size update frame.