feat!: Add typed list and map FieldTypes (#367)

* feat(trace-format): add DynamicList and DynamicMap field types

* docs(trace-format): document DynamicList and DynamicMap in SPEC.md

* feat!(trace-format): non_exhaustive on FieldType, FieldDef, SchemaEntry with constructors

* refactor!(trace-format): opaque DynamicListRef and DynamicMapRef wrappers

* refactor!(trace-format): make FieldDef and SchemaEntry fields private, add getters

* docs(design): update metrique integration for DynamicList/DynamicMap wire format

* feat(js): add DynamicList and DynamicMap decoding to JS parser

Author: jlizen

dial9-tokio-telemetry/design/metrique-integration.md

@@ -17,7 +17,7 @@ This design depends on the entry descriptor system in metrique (see `docs/entry-
 - **`dial9::Context`**: a `#[doc(hidden)]` dial9-internal field tag carried by `Dial9Context`'s own fields. Users do not interact with it directly; they flatten `Dial9Context` into their entry, and the sink walks the descriptor on first-use to find fields tagged `Context`. The name is not a stable guarantee; a future typed source-extraction mechanism would replace this tag-based discovery.
 - **`Dial9EntryWriter`**: the dial9 adapter that walks `Entry::write` on the flush thread. Uses the cached context- and payload-field index sets to route each callback to either the event header (context) or the payload encoder (Emit), or to skip.
 - **First-use per-descriptor**: the moment a `Dial9Stream` first sees an entry with a given `DescriptorId`. Dial9 walks the descriptor once, caches the index sets and any diagnostics, and uses the cache for every subsequent entry of that type.
-- **Trace format**: dial9's wire format, defined in `dial9-trace-format/SPEC.md`. Carries schema frames (one per entry type), event frames (one per emission), pool frames (deduplicated strings and stack frames), and schema-annotation frames (per-field metadata). This design relies on two format features that ship independently of the integration: `TAG_SCHEMA_ANNOTATIONS` and the `List` / `Map` field types.
+- **Trace format**: dial9's wire format, defined in `dial9-trace-format/SPEC.md`. Carries schema frames (one per entry type), event frames (one per emission), pool frames (deduplicated strings and stack frames), and schema-annotation frames (per-field metadata). This design relies on two format features that ship independently of the integration: `TAG_SCHEMA_ANNOTATIONS` and the `DynamicList` / `DynamicMap` field types.
 
 ## User-facing API
 
@@ -156,8 +156,8 @@ The builder and manual composition paths are unchanged from the original design.
 │       encode according to FieldShape:                          │
 │            Known   : encode scalar                             │
 │            Optional: encode presence byte + inner              │
-│            List    : encode <count> <repeated element>         │
-│            Flex    : encode map<key, value>                    │
+│            List    : encode <count> <tag + value per element>   │
+│            Flex    : encode <count> <key_tag+key+val_tag+val>  │
 │            Opaque  : report + skip (sink-side validation)      │
 │                                                                │
 │     if field is tagged Interned and carries string data:       │
@@ -229,7 +229,7 @@ Per entry:
 3. First-use per `DescriptorId`: walk the descriptor to compute the context-field indices (fields tagged `dial9::Context`) and payload-field indices (tagged `dial9::Emit`). Build the wire schema with annotations for units.
 4. Walk `entry.write(..)` with a `Dial9EntryWriter` that uses the cached index sets to route each callback to either the event header (context) or the payload encoder (Emit), or to skip. `Interned` fields have their string data routed through the dial9 string pool. Relies on the metrique contract that `Entry::write` emits `value` callbacks in descriptor order.
 
-`Dial9EntryWriter` overrides `ValueWriter::values()` (the default implementation comma-joins elements into a string) to preserve the typed list wire encoding for `Vec<T>` fields.
+`Dial9EntryWriter` overrides `ValueWriter::values()` (the default implementation comma-joins elements into a string) to preserve the self-describing list wire encoding for `Vec<T>` fields.
 
 A `catch_unwind(AssertUnwindSafe(..))` guard around the `Entry::write` walk drops offending events (rate-limited log) without poisoning the flush thread's state.
 

dial9-tokio-telemetry/src/background_task/sealed.rs

@@ -86,7 +86,7 @@ fn parse_segment_timestamp(data: &[u8]) -> Result<u64, ParseTimestampError> {
                 let name = dec
                     .registry()
                     .get(type_id)
-                    .map(|s| s.name.as_str())
+                    .map(|s| s.name())
                     .ok_or(ParseTimestampError::UnknownTypeId(type_id.0))?;
                 if name == "ClockSyncEvent" {
                     return match values.first() {

dial9-tokio-telemetry/src/telemetry/format.rs

@@ -322,7 +322,7 @@ pub(crate) fn decode_ref<'a>(
     schema: &SchemaEntry,
 ) -> Option<TelemetryEventRef<'a>> {
     use dial9_trace_format::TraceEvent as _;
-    let field_defs = &schema.fields;
+    let field_defs = schema.fields();
     Some(match name {
         "PollStartEvent" => {
             TelemetryEventRef::PollStart(PollStartEvent::decode(timestamp_ns, fields, field_defs)?)

dial9-tokio-telemetry/src/telemetry/recorder/shared_state.rs

@@ -555,7 +555,7 @@ mod shuttle_tests {
         dec.for_each_event(|ev| {
             if ev.name == "ValidationEvent"
                 && let Some(decoded) =
-                    ValidationEvent::decode(ev.timestamp_ns, ev.fields, &ev.schema.fields)
+                    ValidationEvent::decode(ev.timestamp_ns, ev.fields, &ev.schema.fields())
             {
                 out.push(ValidationEvent {
                     timestamp_ns: decoded.timestamp_ns,

dial9-tokio-telemetry/src/tracing_layer.rs

@@ -94,47 +94,23 @@ fn build_callsite_schemas(meta: &'static tracing::Metadata<'static>) -> Callsite
 
     // Base fields present on all span events
     let mut enter_fields = vec![
-        FieldDef {
-            name: "worker_id".into(),
-            field_type: FieldType::Varint,
-        },
-        FieldDef {
-            name: "span_id".into(),
-            field_type: FieldType::Varint,
-        },
-        FieldDef {
-            name: "parent_span_id".into(),
-            field_type: FieldType::OptionalVarint,
-        },
-        FieldDef {
-            name: "span_name".into(),
-            field_type: FieldType::PooledString,
-        },
+        FieldDef::new("worker_id", FieldType::Varint),
+        FieldDef::new("span_id", FieldType::Varint),
+        FieldDef::new("parent_span_id", FieldType::OptionalVarint),
+        FieldDef::new("span_name", FieldType::PooledString),
     ];
     let mut exit_fields = vec![
-        FieldDef {
-            name: "worker_id".into(),
-            field_type: FieldType::Varint,
-        },
-        FieldDef {
-            name: "span_id".into(),
-            field_type: FieldType::Varint,
-        },
-        FieldDef {
-            name: "span_name".into(),
-            field_type: FieldType::PooledString,
-        },
+        FieldDef::new("worker_id", FieldType::Varint),
+        FieldDef::new("span_id", FieldType::Varint),
+        FieldDef::new("span_name", FieldType::PooledString),
     ];
 
     // Add user-defined fields as optional interned strings
     let mut field_names = Vec::new();
     for field in meta.fields() {
         let name = field.name();
         field_names.push(name);
-        let def = FieldDef {
-            name: name.to_string(),
-            field_type: FieldType::OptionalPooledString,
-        };
+        let def = FieldDef::new(name.to_string(), FieldType::OptionalPooledString);
         enter_fields.push(def.clone());
         exit_fields.push(def);
     }

dial9-tokio-telemetry/tests/tracing_layer.rs

@@ -55,56 +55,56 @@ fn decode_span_events(path: &std::path::Path) -> SpanEvents {
             if ev.name.starts_with("SpanEnter:") {
                 result.enter_count += 1;
                 result.enter_schema_names.insert(ev.name.to_owned());
-                for (field_def, field_val) in ev.schema.fields.iter().zip(ev.fields.iter()) {
-                    if field_def.name == "span_name"
+                for (field_def, field_val) in ev.schema.fields().iter().zip(ev.fields.iter()) {
+                    if field_def.name() == "span_name"
                         && let FieldValueRef::PooledString(id) = field_val
                         && let Some(name) = ev.string_pool.get(*id)
                     {
                         result.enter_names.push(name.to_owned());
                     }
-                    if field_def.name == "span_id"
+                    if field_def.name() == "span_id"
                         && let FieldValueRef::Varint(v) = field_val
                     {
                         result.entered_span_ids.insert(*v);
                     }
-                    if field_def.name == "parent_span_id"
+                    if field_def.name() == "parent_span_id"
                         && let FieldValueRef::Varint(v) = field_val
                         && *v > 0
                     {
                         result.saw_parent_span_id = true;
                     }
-                    if field_def.name == "worker_id"
+                    if field_def.name() == "worker_id"
                         && let FieldValueRef::Varint(v) = field_val
                     {
                         result.worker_ids.insert(*v);
                     }
                     // User-defined fields are optional pooled strings
                     if !["worker_id", "span_id", "parent_span_id", "span_name"]
-                        .contains(&field_def.name.as_str())
+                        .contains(&field_def.name())
                         && let FieldValueRef::PooledString(id) = field_val
                         && let Some(v) = ev.string_pool.get(*id)
                     {
                         result
                             .enter_fields
-                            .push((field_def.name.clone(), v.to_owned()));
+                            .push((field_def.name().to_owned(), v.to_owned()));
                     }
                 }
             } else if ev.name.starts_with("SpanExit:") {
                 result.exit_count += 1;
-                for (field_def, field_val) in ev.schema.fields.iter().zip(ev.fields.iter()) {
-                    if !["worker_id", "span_id", "span_name"].contains(&field_def.name.as_str())
+                for (field_def, field_val) in ev.schema.fields().iter().zip(ev.fields.iter()) {
+                    if !["worker_id", "span_id", "span_name"].contains(&field_def.name())
                         && let FieldValueRef::PooledString(id) = field_val
                         && let Some(v) = ev.string_pool.get(*id)
                     {
                         result
                             .exit_fields
-                            .push((field_def.name.clone(), v.to_owned()));
+                            .push((field_def.name().to_owned(), v.to_owned()));
                     }
                 }
             } else if ev.name == "SpanCloseEvent" {
                 result.close_count += 1;
-                for (field_def, field_val) in ev.schema.fields.iter().zip(ev.fields.iter()) {
-                    if field_def.name == "span_id"
+                for (field_def, field_val) in ev.schema.fields().iter().zip(ev.fields.iter()) {
+                    if field_def.name() == "span_id"
                         && let FieldValueRef::Varint(v) = field_val
                     {
                         result.closed_span_ids.insert(*v);

dial9-trace-format-derive/src/lib.rs

@@ -59,10 +59,10 @@ fn derive_trace_event_impl(input: DeriveInput) -> proc_macro2::TokenStream {
 
         let field_name_str = field_name.to_string();
         field_def_tokens.push(quote! {
-            ::dial9_trace_format::schema::FieldDef {
-                name: #field_name_str.to_string(),
-                field_type: <#ty as ::dial9_trace_format::TraceField>::field_type(),
-            }
+            ::dial9_trace_format::schema::FieldDef::new(
+                #field_name_str,
+                <#ty as ::dial9_trace_format::TraceField>::field_type(),
+            )
         });
         encode_tokens.push(quote! {
             <#ty as ::dial9_trace_format::TraceField>::encode(&self.#field_name, enc)?;
@@ -80,7 +80,7 @@ fn derive_trace_event_impl(input: DeriveInput) -> proc_macro2::TokenStream {
         decode_tokens.push(quote! {
             #field_name: {
                 let val = field_defs.iter().zip(fields.iter())
-                    .find(|(f, _)| f.name == #field_name_str)
+                    .find(|(f, _)| f.name() == #field_name_str)
                     .map(|(_, v)| v);
                 match val {
                     Some(v) => <#ty as ::dial9_trace_format::TraceField>::decode_ref(v)?,