Implement self-describing binary trace format

A compact, self-describing binary trace format for tokio runtime
telemetry. Schemas are embedded in the stream so readers don't need
out-of-band type definitions.

- Encoder/decoder with ~48M events/s encode, ~30M decode throughput
- Derive macro (#[derive(TraceEvent)]) with zero-copy Ref decoding
- u24 delta-encoded timestamps in event headers
- String interning, symbol tables, stack frame support
- Field types: u8/u16/u32 (fixed LE), u64 (LEB128), i64, f64, bool,
  String, Bytes, StackFrames, StringMap, PooledString
- JavaScript decoder (js/decode.js)
- Format comparison example, fuzz targets, insta snapshot tests
- SPEC.md wire format specification

Author: rcoh

.gitignore

@@ -2,4 +2,4 @@
 *.bin
 # except the demo trace
 !dial9-tokio-telemetry/trace_viewer/demo-trace.bin
-/target
+target/

Cargo.lock

@@ -2,9 +2,14 @@
 resolver = "3"
 members = [
     "dial9-tokio-telemetry",
+    "dial9-trace-format",
+    "dial9-trace-format-derive",
     "perf-self-profile",
     "examples/metrics-service",
 ]
+exclude = [
+    "dial9-trace-format/fuzz",
+]
 
 [workspace.package]
 version = "0.1.1"

Cargo.toml

@@ -21,6 +21,7 @@ serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 smallvec = "1"
 dial9-perf-self-profile = { workspace = true, optional = true }
+dial9-trace-format = { path = "../dial9-trace-format" }
 tracing = "0.1.44"
 
 [features]

dial9-tokio-telemetry/Cargo.toml

@@ -5,6 +5,7 @@ pub mod collector;
 pub mod cpu_profile;
 pub mod events;
 pub mod format;
+pub mod new_format;
 pub mod recorder;
 pub mod task_metadata;
 pub mod writer;

dial9-tokio-telemetry/src/telemetry/mod.rs

@@ -0,0 +1,19 @@
+[package]
+name = "dial9-trace-format-derive"
+version.workspace = true
+edition = "2024"
+description = "Derive macro for dial9-trace-format TraceEvent trait"
+license = "Apache-2.0"
+publish = false
+
+[lib]
+proc-macro = true
+
+[dependencies]
+syn = { version = "2", features = ["full"] }
+quote = "1"
+proc-macro2 = "1"
+
+[dev-dependencies]
+insta = "1"
+prettyplease = "0.2"

dial9-trace-format-derive/Cargo.toml

@@ -0,0 +1,196 @@
+use proc_macro::TokenStream;
+use quote::{format_ident, quote};
+use syn::{Data, DeriveInput, Fields, parse_macro_input};
+
+fn derive_trace_event_impl(input: DeriveInput) -> proc_macro2::TokenStream {
+    let name = &input.ident;
+    let vis = &input.vis;
+    let name_str = name.to_string();
+    let ref_name = format_ident!("{}Ref", name);
+
+    let fields = match &input.data {
+        Data::Struct(data) => match &data.fields {
+            Fields::Named(f) => &f.named,
+            _ => panic!("TraceEvent only supports named fields"),
+        },
+        _ => panic!("TraceEvent can only be derived for structs"),
+    };
+
+    // Find the field marked with #[traceevent(timestamp)]
+    let mut timestamp_field_name = None;
+    for field in fields.iter() {
+        for attr in &field.attrs {
+            if attr.path().is_ident("traceevent") {
+                let _ = attr.parse_nested_meta(|meta| {
+                    if meta.path.is_ident("timestamp") {
+                        timestamp_field_name = Some(field.ident.as_ref().unwrap().clone());
+                    }
+                    Ok(())
+                });
+            }
+        }
+    }
+
+    let mut field_def_tokens = Vec::new();
+    let mut encode_tokens = Vec::new();
+    let mut ref_fields = Vec::new();
+    let mut decode_tokens = Vec::new();
+    let mut decode_idx = 0usize;
+
+    for field in fields.iter() {
+        let field_name = field.ident.as_ref().unwrap();
+        let ty = &field.ty;
+
+        // Skip the timestamp field in schema/encode/decode — it's in the event header
+        if timestamp_field_name.as_ref() == Some(field_name) {
+            continue;
+        }
+
+        let field_name_str = field_name.to_string();
+        field_def_tokens.push(quote! {
+            ::dial9_trace_format::schema::FieldDef {
+                name: #field_name_str.to_string(),
+                field_type: <#ty as ::dial9_trace_format::TraceField>::field_type(),
+            }
+        });
+        encode_tokens.push(quote! {
+            <#ty as ::dial9_trace_format::TraceField>::encode(&self.#field_name, enc)?;
+        });
+
+        ref_fields.push(quote! {
+            pub #field_name: <#ty as ::dial9_trace_format::TraceField>::Ref<'a>
+        });
+        let idx = decode_idx;
+        decode_tokens.push(quote! {
+            #field_name: <#ty as ::dial9_trace_format::TraceField>::decode_ref(fields.get(#idx)?)?
+        });
+        decode_idx += 1;
+    }
+
+    let timestamp_impl = if let Some(ref ts_field) = timestamp_field_name {
+        quote! {
+            fn timestamp(&self) -> Option<u64> { Some(self.#ts_field) }
+        }
+    } else {
+        quote! {}
+    };
+
+    let has_timestamp_impl = if timestamp_field_name.is_some() {
+        quote! { fn has_timestamp() -> bool { true } }
+    } else {
+        quote! {}
+    };
+
+    // For the Ref struct, timestamp field is NOT included — it comes from the event header
+    let phantom_field =
+        if fields.is_empty() || (fields.len() == 1 && timestamp_field_name.is_some()) {
+            quote! { _marker: ::std::marker::PhantomData<&'a ()>, }
+        } else {
+            quote! {}
+        };
+    let phantom_init = if fields.is_empty() || (fields.len() == 1 && timestamp_field_name.is_some())
+    {
+        quote! { _marker: ::std::marker::PhantomData, }
+    } else {
+        quote! {}
+    };
+
+    quote! {
+        #[derive(Debug, Clone)]
+        #vis struct #ref_name<'a> {
+            #(#ref_fields,)*
+            #phantom_field
+        }
+
+        impl ::dial9_trace_format::TraceEvent for #name {
+            type Ref<'a> = #ref_name<'a>;
+
+            fn event_name() -> &'static str { #name_str }
+            fn field_defs() -> Vec<::dial9_trace_format::schema::FieldDef> {
+                vec![#(#field_def_tokens),*]
+            }
+            #timestamp_impl
+            #has_timestamp_impl
+            fn encode_fields<W: ::std::io::Write>(&self, enc: &mut ::dial9_trace_format::EventEncoder<'_, W>) -> ::std::io::Result<()> {
+                #(#encode_tokens)*
+                Ok(())
+            }
+            fn decode<'a>(fields: &[::dial9_trace_format::types::FieldValueRef<'a>]) -> Option<Self::Ref<'a>> {
+                Some(#ref_name {
+                    #(#decode_tokens,)*
+                    #phantom_init
+                })
+            }
+        }
+    }
+}
+
+#[proc_macro_derive(TraceEvent, attributes(traceevent))]
+pub fn derive_trace_event(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input as DeriveInput);
+    TokenStream::from(derive_trace_event_impl(input))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use insta::assert_snapshot;
+
+    fn expand_to_string(input: proc_macro2::TokenStream) -> String {
+        let input: DeriveInput = syn::parse2(input).unwrap();
+        let output = derive_trace_event_impl(input);
+        match syn::parse2::<syn::File>(output.clone()) {
+            Ok(file) => prettyplease::unparse(&file),
+            Err(_) => output.to_string(),
+        }
+    }
+
+    #[test]
+    fn simple_event() {
+        assert_snapshot!(expand_to_string(quote! {
+            struct SimpleEvent {
+                timestamp_ns: u64,
+                value: u32,
+            }
+        }));
+    }
+
+    #[test]
+    fn empty_event() {
+        assert_snapshot!(expand_to_string(quote! {
+            struct EmptyEvent {}
+        }));
+    }
+
+    #[test]
+    fn all_field_types() {
+        assert_snapshot!(expand_to_string(quote! {
+            struct AllFieldTypes {
+                a_u8: u8,
+                b_u16: u16,
+                c_u32: u32,
+                d_u64: u64,
+                e_i64: i64,
+                f_f64: f64,
+                g_bool: bool,
+                h_string: String,
+                i_bytes: Vec<u8>,
+                j_interned: InternedString,
+                k_frames: StackFrames,
+                l_map: Vec<(String, String)>,
+            }
+        }));
+    }
+
+    #[test]
+    fn timestamp_attribute() {
+        assert_snapshot!(expand_to_string(quote! {
+            struct PollStart {
+                #[traceevent(timestamp)]
+                timestamp_ns: u64,
+                worker_id: u64,
+                task_id: u64,
+            }
+        }));
+    }
+}