Merge pull request #28 from tinythings/isbm-perfetto-types

Cover all perfetto types

Author: isbm

demo/perfetto/README.md

@@ -4,12 +4,12 @@
 
 ```bash
 # From workspace root — downloads deps and builds all needed tools.
-./scripts/build-perfetto.sh
+./demo/perfetto/build-perfetto.sh
 ```
 
 Or pass a custom source path:
 ```bash
-./scripts/build-perfetto.sh /path/to/perfetto
+./demo/perfetto/build-perfetto.sh /path/to/perfetto
 ```
 
 The script installs missing system packages (git, python3, curl, tar), downloads

scripts/build-perfetto.sh demo/perfetto/build-perfetto.shscripts/build-perfetto.sh renamed to demo/perfetto/build-perfetto.sh

@@ -8,7 +8,7 @@ the trace processor for interactive inspection.
 Build the Perfetto tools (from workspace root):
 
 ```bash
-./scripts/build-perfetto.sh
+./demo/perfetto/build-perfetto.sh
 ```
 
 The script finds tools via `PERFETTO_OUT` (default: `perfetto/out/linux_release`).

demo/perfetto/linux-data-record/README.md

@@ -45,7 +45,7 @@ echo "Recording 5s of ftrace to $PERFETTO_TRACE_OUT..."
 CONFIG_FILE="$SCRIPT_DIR/trace-config.txt"
 cat > "$CONFIG_FILE" <<'ENDCONFIG'
 buffers: {
-    size_kb: 4096
+    size_kb: 8192
     fill_policy: RING_BUFFER
 }
 data_sources: {
@@ -54,6 +54,9 @@ data_sources: {
         ftrace_config {
             ftrace_events: "sched/sched_switch"
             ftrace_events: "sched/sched_waking"
+            ftrace_events: "sched/sched_process_exec"
+            ftrace_events: "sched/sched_process_fork"
+            ftrace_events: "power/cpu_frequency"
         }
     }
 }

demo/perfetto/linux-data-record/run-demo.sh

@@ -8,7 +8,7 @@ plugin into a `.logjet` spool, and view the result in `ljx view`.
 ```bash
 # From workspace root
 make dev
-./scripts/build-perfetto.sh
+./demo/perfetto/build-perfetto.sh
 ```
 
 ## Run
@@ -23,25 +23,30 @@ Requires sudo for ftrace access.
 ## What Happens
 
 1. `traced` + `traced_probes` start in the background.
-2. `tracebox` records 5s of scheduler events (CPU switches) via ftrace.
+2. `tracebox` records 5s of scheduler events (CPU switches, process lifecycle,
+   CPU frequency, interrupts) via ftrace.
 3. `ljd` loads the perfetto-ingest plugin, which spawns `trace_processor`,
-   exports the trace as SQLite, maps `sched_slice` rows to OTel log records
-   with CPU/state/duration, and streams them into a `.logjet` spool.
-4. `ljx view` opens the spool — each CPU scheduling event appears as one line.
+   exports the trace as SQLite, maps every Perfetto table to OTel log records
+   (sched slices, thread states, ftrace events, spurious wakeups, instant
+   events, counters), and streams them into a `.logjet` spool.
+4. `ljx view` opens the spool.
 
 ## What You Should See
 
-- Thousands of log lines, each showing a CPU scheduling event:
+- Thousands of log lines across multiple types:
   ```
-  May  7 10:43:15 I  cpu=7 dur=7.2us state=R utid=19 ts=...
-  May  7 10:43:15 I  cpu=7 dur=2.0us state=R utid=21 ts=...
+  cpu=7 state=R utid=19  dur=7.2us        ← sched_slice
+  state=S dur=12.3us utid=3 cpu=1          ← thread_state
+  sched_switch cpu=5                        ← ftrace_event
+  spurious_wakeup utid=1                    ← spurious_wakeup
   ```
-- Press `Enter` to see full OTel attributes (perfetto.sched.id, cpu, end_state).
-- Press `F` for field filter, `/` to search, `q` to quit.
+- Press `Enter` to see full OTel attributes for each record.
+- Press `F` for field filter (e.g. filter by `perfetto.sched.cpu` to see only one CPU).
+- `/` to search, `q` to quit.
 
 ## Troubleshooting
 
 - **0 records**: The trace needs ftrace events — they require root. The script
   uses `sudo tracebox`. If passwordless sudo isn't configured, run `sudo ./run-demo.sh`.
-- **Fewer records than expected in ljx view**: Delete stale index cache:
-  `rm -rf ~/.cache/ljx && ./run-demo.sh`
+- **ljx view shows fewer records than expected**: The ljx index builder bug was fixed
+  in this PR. If you still see fewer records, delete `~/.cache/ljx/` and re-run.

demo/perfetto/linux-data-record/trace.pftrace

@@ -27,7 +27,7 @@ done
 for bin in "$TRACED" "$TRACED_PROBES" "$TRACEBOX" "$TP"; do
     if [ ! -x "$bin" ]; then
         echo "missing $bin"
-        echo "build perfetto first with: ./scripts/build-perfetto.sh"
+        echo "build perfetto first with: ./demo/perfetto/build-perfetto.sh"
         exit 1
     fi
 done
@@ -57,7 +57,7 @@ echo "Recording 5s of ftrace to $TRACE_FILE..."
 CONFIG_FILE="$SCRIPT_DIR/trace-config.txt"
 cat > "$CONFIG_FILE" <<'ENDCONFIG'
 buffers: {
-    size_kb: 4096
+    size_kb: 8192
     fill_policy: RING_BUFFER
 }
 data_sources: {
@@ -66,7 +66,13 @@ data_sources: {
         ftrace_config {
             ftrace_events: "sched/sched_switch"
             ftrace_events: "sched/sched_waking"
+            ftrace_events: "sched/sched_process_exec"
+            ftrace_events: "sched/sched_process_fork"
+            ftrace_events: "sched/sched_process_exit"
             ftrace_events: "power/cpu_frequency"
+            ftrace_events: "power/cpu_idle"
+            ftrace_events: "irq/irq_handler_entry"
+            ftrace_events: "irq/irq_handler_exit"
         }
     }
 }
@@ -119,27 +125,30 @@ LJD_PID=$!
 cleanup_ljd() {
     kill "$LJD_PID" 2>/dev/null || true
     wait "$LJD_PID" 2>/dev/null || true
+    rm -f "$CONFIG_FILE"
 }
 
 trap cleanup_ljd EXIT INT TERM
 
-# Give the plugin time to finish processing (SQLite export + mapping takes a few seconds).
-sleep 10
+# Poll until records appear (plugin finishes), up to 60s.
+echo "Waiting for import..."
+elapsed=0
+while [ "$elapsed" -lt 60 ]; do
+    if [ -f "$SPOOL_DIR/perfetto.logjet" ]; then
+        COUNT=$("$LJX" count "$SPOOL_DIR/perfetto.logjet" 2>/dev/null || echo "0")
+        if [ "$COUNT" -gt 0 ] 2>/dev/null; then
+            echo "Imported $COUNT records into $SPOOL_DIR/perfetto.logjet"
+            break
+        fi
+    fi
+    sleep 1
+    elapsed=$((elapsed + 1))
+done
+
 kill "$LJD_PID" 2>/dev/null || true
 wait "$LJD_PID" 2>/dev/null || true
 trap - EXIT INT TERM
 
-rm -f "$CONFIG_FILE"
-
-if [ ! -f "$SPOOL_DIR/perfetto.logjet" ]; then
-    echo "No .logjet file produced."
-    exit 1
-fi
-
-RECORDS=$("$LJX" count "$SPOOL_DIR/perfetto.logjet" | tail -1)
-echo "Imported $RECORDS records into $SPOOL_DIR/perfetto.logjet"
-echo ""
-
 # ── View the result ───────────────────────────────────────────────────────────
 
 echo "Opening ljx view..."

demo/perfetto/perfetto-to-logjet/README.md

@@ -1,88 +1,111 @@
 # Perfetto Ingest Plugin (`lj-perfetto-ingest`)
 
 Imports Perfetto trace files (`.pftrace` / `.perfetto-trace`) into the logjet
-ecosystem as OTel traces, metrics, logs, and events.
+ecosystem as OTel logs, traces, and metrics.
 
 ## Architecture
 
 ```
  .pftrace ──→ trace_processor (spawned as subprocess)
-                 ├── export sqlite  ──→ sqlite_reader ──→ trace_mapper  ──→ OTel spans
+                 ├── export sqlite  ──→ sqlite_reader ──→ trace_mapper ──→ OTel spans
                  └── --run-metrics  ──→ metrics_reader ──→ metric_mapper ──→ OTel metrics
-                                                              log_mapper   ──→ OTel logs
+                                                              log_mapper ──→ OTel logs
+                                                                                  │
+                                                                                  ▼
+                                                                         buffer & sort by ts
                                                                                   │
                                                                                   ▼
                                                                            ljd spool (.logjet)
 ```
 
 The plugin is an **active source** (`mode: 1`). ljd calls `lj_ingest_fetch()` once,
-which runs the full pipeline and streams OTel payloads through the generic record
-callback.
+which runs the full pipeline. All records from traces, logs, and metrics are
+collected, sorted by timestamp, then streamed through the generic record callback
+to guarantee monotonic timestamps in the logjet block format.
 
 ## Requirements
 
 - Perfetto trace processor binary (`trace_processor` or `trace_processor_shell`).
   Build it from the bundled Perfetto source:
   ```bash
-  ./scripts/build-perfetto.sh
+  ./demo/perfetto/build-perfetto.sh
   ```
 - A `.pftrace` trace file to import.
 
 ## Usage
 
 ```bash
 # Build the plugin and ljd:
-make build
+make dev
+
+# Create a config file (ljd uses YAML config, not CLI flags):
+cat > /tmp/perfetto.conf <<EOF
+output: file
+file.path: ./spool
+file.size: 10mb
+file.name: perfetto.logjet
+ingest.protocol: plugin
+ingest.plugin-path: ./target/debug/liblj_perfetto_ingest.so
+EOF
 
 # Run the import:
 LJD_PERFETTO_TRACE_FILE=/path/to/trace.pftrace \
 LJD_PERFETTO_TRACE_PROCESSOR=/path/to/trace_processor_shell \
-  ljd serve \
-    --ingest-protocol plugin \
-    --ingest-plugin perfetto \
-    --storage ./otel-spool
+  ljd serve --config /tmp/perfetto.conf
 ```
 
+See `demo/perfetto/perfetto-to-logjet/run-demo.sh` for a complete end-to-end
+example that records, imports, and opens the result in `ljx view`.
+
 ## Environment Variables
 
 | Variable | Required | Default | Description |
 |----------|----------|---------|-------------|
 | `LJD_PERFETTO_TRACE_FILE` | **Yes** | — | Path to the `.pftrace` input file. |
 | `LJD_PERFETTO_TRACE_PROCESSOR` | No | PATH search | Path to `trace_processor_shell` binary. |
 | `LJD_PERFETTO_TIMESTAMP_POLICY` | No | `best-effort` | `best-effort` or `require-realtime`. |
-| `LJD_PERFETTO_METRICS` | No | (none) | Comma-separated metric names to run, e.g. `trace_stats,android_startup`. |
+| `LJD_PERFETTO_METRICS` | No | (none) | Comma-separated metric names to run, e.g. `trace_stats`. |
 
-## Output Signals
+## Covered Perfetto Types
 
-| Perfetto Source | OTel Signal | Record Type |
-|-----------------|-------------|-------------|
-| `slice` table | Traces (Spans) | `Traces` |
-| Metrics JSON | Metrics (Gauges) | `Metrics` |
-| Analysis summary | Logs | `Logs` |
-| (reserved) | Events | `Events` |
+Every table in the exported SQLite DB has a typed model and DB reader. Types
+with data are mapped to OTel log records with structured attributes:
 
-Spans are batched in groups of 200 per OTLP export request.
+| Perfetto Table | OTel Signal | Attributes |
+|---------------|-------------|------------|
+| `sched_slice` | Logs | cpu, end_state, dur_ns |
+| `thread_state` | Logs | state, dur_ns, cpu, io_wait, blocked_function |
+| `ftrace_event` | Logs | name, cpu, utid |
+| `spurious_sched_wakeup` | Logs | utid, waker_utid |
+| `instant` | Logs | name, track_id |
+| `slice` | Traces + Logs | name, dur_ns, depth, parent_id |
+| `counter` | Metrics (planned) | value |
+| `process`, `thread`, `cpu`, `machine`, `metadata`, `args`, `clock_snapshot` | Metadata | used internally |
+| `flow`, `heap_*`, `stack_*`, `memory_*`, `protolog`, `android_logs`, `filedescriptor` | Models ready | not yet mapped |
+
+Each log record carries integer attributes (`perfetto.sched.dur_ns`, etc.) for
+structured downstream consumption alongside a human-readable body.
 
 ## Timestamp Policy
 
 Perfetto timestamps are trace-clock values (typically `CLOCK_MONOTONIC`). The plugin
 converts them to Unix epoch nanoseconds using `clock_snapshot` REALTIME entries.
 
-- **best-effort** (default): Spans without realtime data are skipped. Spans before
-  the first snapshot are extrapolated backwards.
+- **best-effort** (default): Spans without realtime data use extrapolation.
+  Spans before the first snapshot are extrapolated backwards.
 - **require-realtime**: The pipeline fails if any span cannot be converted.
 
+Records from all mappers are collected into a buffer, sorted by timestamp, then
+emitted sequentially. This guarantees monotonicity within logjet blocks even when
+different mapper types produce interleaved time ranges.
+
 ## Limitations
 
-- **No flow-to-link mapping**: `flow` table entries are read but not yet mapped
-  to OTel span links.
-- **No args-to-attributes mapping**: Per-slice key-value arguments are read but
-  not attached to spans.
-- **Thread/process context**: Thread and process names are loaded but not fully
-  joined to spans via track relationships.
-- **Replay/bridge**: Traces and metrics stored in `.logjet` can be exported via
-  `ljx export` but are not yet forwarded by `ljd bridge/replay` (which currently
-  only forwards logs).
-- **Metrics**: Only scalar metric values are supported (no histograms).
-- **Event signal**: The `Events` record type is reserved but not yet generated
-  by this plugin.
+- **Replay/bridge**: Only logs are forwarded by `ljd bridge/replay`. Traces and
+  metrics stored in `.logjet` can be exported via `ljx export` (parquet).
+- **ljx view**: Only decodes `ExportLogsServiceRequest` — trace/metric records
+  appear as binary data in the detail pane. Trace mapping is disabled by default.
+- **Trace span emission**: Currently disabled (ljx can't render it). Enable by
+  uncommenting `trace_mapper::map_traces` in `lib.rs`.
+- **Histograms**: Metrics only support scalar gauge values.
+- **Event signal**: The `Events` record type is reserved but not yet generated.

demo/perfetto/perfetto-to-logjet/run-demo.sh

@@ -680,7 +680,8 @@ impl ViewApp {
         };
 
         let body_height = filtered_sev.len().max(filtered_svc.len()).max(1) as u16;
-        let popup_h = (body_height + 4).clamp(20, screen.height * 60 / 100);
+        let max_h = (screen.height * 60 / 100).max(20);
+        let popup_h = (body_height + 4).min(max_h);
         let popup_w = (screen.width * 60 / 100).max(40);
         let area = Rect::new(screen.width.saturating_sub(popup_w) / 2, screen.height * 20 / 100, popup_w, popup_h);
         frame.render_widget(Clear, area);

doc/perfetto-ingest.md

@@ -672,7 +672,9 @@ unsafe extern "C" fn on_generic_record(user: *mut c_void, record: *const LjInges
         SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as u64
     };
 
-    let wire = WireRecord { record_type, seq: ctx.next_seq.fetch_add(1, Ordering::Relaxed), ts_unix_ns: ts, payload };
+    let seq = ctx.next_seq.fetch_add(1, Ordering::Relaxed);
+
+    let wire = WireRecord { record_type, seq, ts_unix_ns: ts, payload };
 
     if let Err(err) = super::daemon::append_to_spool(&ctx.spool, wire) {
         eprintln!("ljd plugin callback spool error: {err}");