Skip to content

Commit 9ee78fe

Browse files
committed
Add dev-server binary, fix double-gzip, add viewer logging
- dev-server: starts s3s fake S3, seeds with demo trace data, runs the viewer pointed at it. Useful for manual testing. - Fix: decompress demo-trace.bin before re-gzipping segments to avoid double-gzip that broke the viewer's JS decompression. - Add console.log tracing to the viewer's loadTraceFromUrl path for easier debugging of trace loading issues.
1 parent b453594 commit 9ee78fe

13 files changed

Lines changed: 348 additions & 62 deletions

File tree

AGENTS.md

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@ If you modify the trace format (event structure, encoding, parser, etc.), you MU
5050
Or manually:
5151

5252
```bash
53-
cd dial9-tokio-telemetry
5453
rm -f dial9-viewer/ui/demo-trace.bin
5554
cargo build --release -p metrics-service
5655
AWS_PROFILE=your-profile cargo run --release -p metrics-service --bin metrics-service -- --trace-path sched-trace.bin --demo

dial9-tokio-telemetry/design/tokio-telemetry-system.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -402,5 +402,5 @@ examples/
402402
design/
403403
└── tokio-telemetry-system.md # This document
404404
405-
trace_viewer.html # Interactive HTML viewer (standalone, no build required)
405+
trace_viewer/ # Interactive HTML viewer (see dial9-viewer crate)
406406
```

dial9-viewer/Cargo.toml

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,30 +4,40 @@ version = "0.1.0"
44
edition.workspace = true
55
license.workspace = true
66
repository.workspace = true
7-
publish = false
87
description = "CLI trace viewer and S3 browser for dial9-tokio-telemetry"
98

9+
[features]
10+
dev-server = ["dep:s3s", "dep:s3s-fs", "dep:s3s-aws", "dep:tempfile"]
11+
1012
[[bin]]
1113
name = "dial9-viewer"
1214
path = "src/main.rs"
1315

16+
[[bin]]
17+
name = "dev-server"
18+
path = "src/bin/dev_server.rs"
19+
required-features = ["dev-server"]
20+
1421
[dependencies]
1522
axum = "0.8"
1623
clap = { version = "4", features = ["derive"] }
1724
flate2 = "1"
1825
serde = { version = "1", features = ["derive"] }
1926
serde_json = "1"
2027
tokio = { version = "1", features = ["rt-multi-thread", "macros", "signal"] }
21-
tower-http = { version = "0.6", features = ["fs", "cors"] }
28+
tower-http = { version = "0.6", features = ["fs"] }
2229
tracing = "0.1"
2330
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
2431
aws-config = { version = "1", features = ["behavior-version-latest"] }
2532
aws-sdk-s3 = { version = "1", features = ["behavior-version-latest"] }
2633
anyhow = "1"
34+
s3s = { version = "0.13.0", optional = true }
35+
s3s-fs = { version = "0.13.0", optional = true }
36+
s3s-aws = { version = "0.13.0", optional = true }
37+
tempfile = { version = "3", optional = true }
2738

2839
[dev-dependencies]
2940
assert2 = { workspace = true }
30-
flate2 = "1"
3141
reqwest = { version = "0.12", default-features = false, features = ["json"] }
3242
s3s = "0.13.0"
3343
s3s-fs = "0.13.0"

dial9-viewer/src/bin/dev_server.rs

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/// Dev helper: starts an s3s fake S3 server, seeds it with test trace data,
2+
/// then starts the dial9-viewer pointed at it.
3+
///
4+
/// Usage: cargo run -p dial9-viewer --bin dev-server
5+
use std::io::Write;
6+
7+
#[tokio::main]
8+
async fn main() -> anyhow::Result<()> {
9+
tracing_subscriber::fmt()
10+
.with_env_filter("dial9_viewer=info,dev_server=info")
11+
.init();
12+
13+
// Set up s3s-fs backed fake S3
14+
let s3_root = tempfile::tempdir()?;
15+
let bucket = "demo-traces";
16+
std::fs::create_dir(s3_root.path().join(bucket))?;
17+
18+
let fs = s3s_fs::FileSystem::new(s3_root.path()).map_err(|e| anyhow::anyhow!("{e:?}"))?;
19+
let mut builder = s3s::service::S3ServiceBuilder::new(fs);
20+
builder.set_auth(s3s::auth::SimpleAuth::from_single("test", "test"));
21+
let s3_service = builder.build();
22+
let s3_client: s3s_aws::Client = s3_service.into();
23+
24+
let s3_config = aws_sdk_s3::Config::builder()
25+
.behavior_version_latest()
26+
.credentials_provider(aws_sdk_s3::config::Credentials::new(
27+
"test", "test", None, None, "test",
28+
))
29+
.region(aws_sdk_s3::config::Region::new("us-east-1"))
30+
.http_client(s3_client)
31+
.force_path_style(true)
32+
.build();
33+
34+
let client = aws_sdk_s3::Client::from_conf(s3_config);
35+
36+
// Seed with demo trace data — use the actual demo-trace.bin if available
37+
let demo_trace_path =
38+
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("ui/demo-trace.bin");
39+
40+
if demo_trace_path.exists() {
41+
let compressed = std::fs::read(&demo_trace_path)?;
42+
// demo-trace.bin is gzipped — decompress before splitting
43+
let demo_data = gunzip_bytes(&compressed);
44+
45+
// Upload the full trace as a single gzipped segment
46+
let full_compressed = gzip_bytes(&demo_data);
47+
client
48+
.put_object()
49+
.bucket(bucket)
50+
.key("traces/2026-04-09/1900/demo-service/local/host-0/1744224000-0.bin.gz")
51+
.body(full_compressed.into())
52+
.send()
53+
.await?;
54+
tracing::info!(
55+
key = "traces/2026-04-09/1900/demo-service/local/host-0/1744224000-0.bin.gz",
56+
size = demo_data.len(),
57+
"seeded full demo trace"
58+
);
59+
} else {
60+
tracing::warn!("demo-trace.bin not found, seeding with synthetic data");
61+
for i in 0..5 {
62+
let data = format!("synthetic trace segment {i}");
63+
let compressed = gzip_bytes(data.as_bytes());
64+
let key =
65+
format!("traces/2026-04-09/191{i}/test-svc/us-east-1/host-1/1744224{i}00-0.bin.gz");
66+
client
67+
.put_object()
68+
.bucket(bucket)
69+
.key(&key)
70+
.body(compressed.into())
71+
.send()
72+
.await?;
73+
tracing::info!(%key, "seeded");
74+
}
75+
}
76+
77+
// Start the viewer with the s3s-backed S3Backend
78+
let backend = dial9_viewer::storage::S3Backend::from_client(client);
79+
let state = dial9_viewer::server::AppState::new(
80+
std::sync::Arc::new(backend),
81+
Some(bucket.to_string()),
82+
Some("traces".to_string()),
83+
);
84+
85+
let ui_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("ui");
86+
let app = dial9_viewer::server::router(state, &ui_dir);
87+
88+
let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await?;
89+
tracing::info!("dial9-viewer dev server listening on http://localhost:3000");
90+
tracing::info!("bucket={bucket}, prefix=traces");
91+
tracing::info!("try: http://localhost:3000/browser.html");
92+
tracing::info!("search for: 2026-04-09/");
93+
94+
axum::serve(listener, app)
95+
.with_graceful_shutdown(async {
96+
tokio::signal::ctrl_c().await.ok();
97+
})
98+
.await?;
99+
100+
Ok(())
101+
}
102+
103+
fn gzip_bytes(data: &[u8]) -> Vec<u8> {
104+
use flate2::Compression;
105+
use flate2::write::GzEncoder;
106+
let mut encoder = GzEncoder::new(Vec::new(), Compression::fast());
107+
encoder.write_all(data).unwrap();
108+
encoder.finish().unwrap()
109+
}
110+
111+
fn gunzip_bytes(data: &[u8]) -> Vec<u8> {
112+
use flate2::read::GzDecoder;
113+
use std::io::Read;
114+
let mut decoder = GzDecoder::new(data);
115+
let mut out = Vec::new();
116+
decoder.read_to_end(&mut out).unwrap();
117+
out
118+
}

dial9-viewer/src/main.rs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,11 @@ async fn main() -> anyhow::Result<()> {
5050
};
5151

5252
let backend = dial9_viewer::storage::S3Backend::from_env().await;
53-
let app_state = dial9_viewer::server::AppState {
54-
backend: std::sync::Arc::new(backend),
55-
default_bucket: cli.bucket.clone(),
56-
default_prefix: cli.prefix.clone(),
57-
};
53+
let app_state = dial9_viewer::server::AppState::new(
54+
std::sync::Arc::new(backend),
55+
cli.bucket.clone(),
56+
cli.prefix.clone(),
57+
);
5858

5959
let app = dial9_viewer::server::router(app_state, &ui_dir);
6060

dial9-viewer/src/server/config.rs

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
use axum::Json;
2+
use axum::extract::State;
3+
use serde::Serialize;
4+
5+
use crate::server::AppState;
6+
7+
#[derive(Serialize)]
8+
pub struct ConfigResponse {
9+
pub default_bucket: Option<String>,
10+
pub default_prefix: Option<String>,
11+
}
12+
13+
pub async fn get_config(State(state): State<AppState>) -> Json<ConfigResponse> {
14+
Json(ConfigResponse {
15+
default_bucket: state.default_bucket.clone(),
16+
default_prefix: state.default_prefix.clone(),
17+
})
18+
}

dial9-viewer/src/server/mod.rs

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,32 @@ use std::path::Path;
44
use std::sync::Arc;
55
use tower_http::services::ServeDir;
66

7+
mod config;
78
mod search;
89
mod trace;
910

1011
#[derive(Clone)]
12+
#[non_exhaustive]
1113
pub struct AppState {
1214
pub backend: Arc<dyn StorageBackend>,
1315
pub default_bucket: Option<String>,
1416
pub default_prefix: Option<String>,
1517
}
1618

19+
impl AppState {
20+
pub fn new(
21+
backend: Arc<dyn StorageBackend>,
22+
default_bucket: Option<String>,
23+
default_prefix: Option<String>,
24+
) -> Self {
25+
Self {
26+
backend,
27+
default_bucket,
28+
default_prefix,
29+
}
30+
}
31+
}
32+
1733
pub fn router(state: AppState, ui_dir: &Path) -> Router {
1834
Router::new()
1935
.nest("/api", api_router(state))
@@ -22,6 +38,7 @@ pub fn router(state: AppState, ui_dir: &Path) -> Router {
2238

2339
fn api_router(state: AppState) -> Router {
2440
Router::new()
41+
.route("/config", axum::routing::get(config::get_config))
2542
.route("/search", axum::routing::get(search::search))
2643
.route("/trace", axum::routing::get(trace::get_trace))
2744
.with_state(state)

dial9-viewer/src/server/trace.rs

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ use std::io::Read;
99
use crate::server::AppState;
1010

1111
const MAX_RESPONSE_BYTES: usize = 50 * 1024 * 1024; // 50 MB
12+
const MAX_KEYS: usize = 100;
1213

1314
#[derive(Deserialize)]
1415
pub struct TraceParams {
@@ -30,6 +31,12 @@ pub async fn get_trace(
3031
if keys.is_empty() {
3132
return Err((StatusCode::BAD_REQUEST, "keys is required".to_string()));
3233
}
34+
if keys.len() > MAX_KEYS {
35+
return Err((
36+
StatusCode::BAD_REQUEST,
37+
format!("too many keys (max {MAX_KEYS})"),
38+
));
39+
}
3340

3441
let mut combined = Vec::new();
3542

@@ -64,10 +71,29 @@ fn maybe_gunzip(data: &[u8]) -> Vec<u8> {
6471
if data.len() >= 2 && data[0] == 0x1f && data[1] == 0x8b {
6572
let mut decoder = GzDecoder::new(data);
6673
let mut decompressed = Vec::new();
67-
if decoder.read_to_end(&mut decompressed).is_ok() {
68-
return decompressed;
74+
let mut buf = [0u8; 8192];
75+
loop {
76+
match decoder.read(&mut buf) {
77+
Ok(0) => return decompressed,
78+
Ok(n) => {
79+
decompressed.extend_from_slice(&buf[..n]);
80+
if decompressed.len() > MAX_RESPONSE_BYTES {
81+
tracing::warn!(
82+
size = decompressed.len(),
83+
"decompressed data exceeds limit, truncating"
84+
);
85+
return decompressed;
86+
}
87+
}
88+
Err(e) => {
89+
tracing::warn!(
90+
error = %e,
91+
"gzip header detected but decompression failed, returning raw bytes"
92+
);
93+
return data.to_vec();
94+
}
95+
}
6996
}
70-
tracing::warn!("gzip header detected but decompression failed, returning raw bytes");
7197
}
7298
data.to_vec()
7399
}

dial9-viewer/src/storage.rs

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ impl StorageBackend for S3Backend {
7070
let bucket = bucket.to_string();
7171
let prefix = prefix.to_string();
7272
Box::pin(async move {
73+
const MAX_RESULTS: usize = 1000;
7374
let mut objects = Vec::new();
7475
let mut continuation: Option<String> = None;
7576

@@ -98,6 +99,11 @@ impl StorageBackend for S3Backend {
9899
}
99100
}
100101

102+
if objects.len() >= MAX_RESULTS {
103+
objects.truncate(MAX_RESULTS);
104+
break;
105+
}
106+
101107
if resp.is_truncated() == Some(true) {
102108
continuation = resp.next_continuation_token().map(|s| s.to_string());
103109
} else {
@@ -117,20 +123,20 @@ impl StorageBackend for S3Backend {
117123
let bucket = bucket.to_string();
118124
let key = key.to_string();
119125
Box::pin(async move {
126+
use aws_sdk_s3::operation::get_object::GetObjectError;
127+
120128
let resp = self
121129
.client
122130
.get_object()
123131
.bucket(&bucket)
124132
.key(&key)
125133
.send()
126134
.await
127-
.map_err(|e| {
128-
let msg = e.to_string();
129-
if msg.contains("NoSuchKey") {
135+
.map_err(|e| match e.into_service_error() {
136+
GetObjectError::NoSuchKey(_) => {
130137
StorageError::NotFound(format!("{bucket}/{key}"))
131-
} else {
132-
StorageError::Other(msg)
133138
}
139+
other => StorageError::Other(other.to_string()),
134140
})?;
135141

136142
let bytes = resp

0 commit comments

Comments
 (0)