Skip to content

Commit 6518593

Browse files
eldryotheldryoth
committed
Expose route cache runtime stats
1 parent 1d60a68 commit 6518593

3 files changed

Lines changed: 163 additions & 44 deletions

File tree

‎docs/cache-backends.md‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -77,11 +77,11 @@ internal cache implementation.
7777
tiered Pingora storage adapter: memory is L1, disk is L2, misses are written
7878
to both tiers, disk hits are promoted back into memory when they fit, and
7979
purge invalidates both tiers.
80-
- The protected admin endpoint `GET /_fluxheim/cache/status` reports per-vhost
81-
and aggregate cache enablement, tiering, memory counters, disk counters, and
82-
cache activity counters for hits, misses, stores, refused stores, and purges.
83-
`POST /_fluxheim/cache/activity/reset` resets activity counters without
84-
clearing cached objects.
80+
- The protected admin endpoint `GET /_fluxheim/cache/status` reports aggregate,
81+
per-vhost, and per-route cache enablement, tiering, memory counters, disk
82+
counters, and cache activity counters for hits, misses, stores, refused
83+
stores, and purges. `POST /_fluxheim/cache/activity/reset` resets vhost and
84+
route activity counters without clearing cached objects.
8585
`POST /_fluxheim/cache/purge` invalidates one cache identity from the
8686
selected vhost. If the object has negotiated `Vary` variants, memory and disk
8787
purge remove every stored variant for that primary identity. `POST

‎src/admin.rs‎

Lines changed: 61 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -359,8 +359,8 @@ impl AdminApp {
359359
fn cache_activity_reset_response(&self) -> AdminResponse {
360360
let result = self.proxy.reset_cache_activity();
361361
let body = format!(
362-
r#"{{"status":"ok","memory_tiers":{},"disk_tiers":{},"tiered_vhosts":{}}}"#,
363-
result.memory_tiers, result.disk_tiers, result.tiered_vhosts
362+
r#"{{"status":"ok","memory_tiers":{},"disk_tiers":{},"tiered_vhosts":{},"tiered_routes":{}}}"#,
363+
result.memory_tiers, result.disk_tiers, result.tiered_vhosts, result.tiered_routes
364364
);
365365
json_response(StatusCode::OK, body.as_bytes())
366366
}
@@ -1304,10 +1304,12 @@ fn cache_purge_results_json(results: &[crate::proxy::CachePurgeResult]) -> Strin
13041304
#[cfg(feature = "cache")]
13051305
fn cache_totals_json(totals: &crate::proxy::CacheRuntimeTotals) -> String {
13061306
format!(
1307-
r#"{{"vhosts":{},"enabled_vhosts":{},"tiered_vhosts":{},"memory_entries":{},"memory_weighted_size_bytes":{},"memory_max_size_bytes":{},"disk_entries":{},"disk_size_bytes":{},"disk_max_size_bytes":{},"activity":{}}}"#,
1307+
r#"{{"vhosts":{},"enabled_vhosts":{},"tiered_vhosts":{},"enabled_routes":{},"tiered_routes":{},"memory_entries":{},"memory_weighted_size_bytes":{},"memory_max_size_bytes":{},"disk_entries":{},"disk_size_bytes":{},"disk_max_size_bytes":{},"activity":{}}}"#,
13081308
totals.vhosts,
13091309
totals.enabled_vhosts,
13101310
totals.tiered_vhosts,
1311+
totals.enabled_routes,
1312+
totals.tiered_routes,
13111313
totals.memory_entries,
13121314
totals.memory_weighted_size_bytes,
13131315
totals.memory_max_size_bytes,
@@ -1332,12 +1334,32 @@ fn cache_vhost_stats_json(vhosts: &[crate::proxy::CacheVhostStats]) -> String {
13321334
body.push(',');
13331335
}
13341336
body.push_str(&format!(
1335-
r#"{{"name":"{}","enabled":{},"tiered":{},"memory":{},"disk":{}}}"#,
1337+
r#"{{"name":"{}","enabled":{},"tiered":{},"memory":{},"disk":{},"routes":[{}]}}"#,
13361338
json_escape(&vhost.name),
13371339
vhost.enabled,
13381340
vhost.tiered,
13391341
memory_cache_stats_json(vhost.memory.as_ref()),
1340-
disk_cache_stats_json(vhost.disk.as_ref())
1342+
disk_cache_stats_json(vhost.disk.as_ref()),
1343+
cache_route_stats_json(&vhost.routes)
1344+
));
1345+
}
1346+
body
1347+
}
1348+
1349+
#[cfg(feature = "cache")]
1350+
fn cache_route_stats_json(routes: &[crate::proxy::CacheRouteStats]) -> String {
1351+
let mut body = String::new();
1352+
for (index, route) in routes.iter().enumerate() {
1353+
if index > 0 {
1354+
body.push(',');
1355+
}
1356+
body.push_str(&format!(
1357+
r#"{{"name":"{}","enabled":{},"tiered":{},"memory":{},"disk":{}}}"#,
1358+
json_escape(&route.name),
1359+
route.enabled,
1360+
route.tiered,
1361+
memory_cache_stats_json(route.memory.as_ref()),
1362+
disk_cache_stats_json(route.disk.as_ref())
13411363
));
13421364
}
13431365
body
@@ -1580,7 +1602,7 @@ mod tests {
15801602
WebConfig,
15811603
};
15821604
#[cfg(feature = "cache")]
1583-
use crate::config::{ByteSize, CacheConfig};
1605+
use crate::config::{ByteSize, CacheConfig, RouteConfig};
15841606
use crate::proxy::{FluxProxy, ProxyHealthReporter, ProxyHealthSignal};
15851607
use crate::snapshot::SnapshotStore;
15861608
use crate::test_support::unique_temp_path;
@@ -1817,7 +1839,7 @@ mod tests {
18171839
},
18181840
headers: crate::config::VhostHeaderPolicyConfig::default(),
18191841
web: WebConfig::default(),
1820-
routes: Vec::new(),
1842+
routes: vec![cached_assets_route()],
18211843
}],
18221844
..Config::default()
18231845
};
@@ -1834,6 +1856,8 @@ mod tests {
18341856
assert!(body.contains(r#""totals":{"vhosts":1"#));
18351857
assert!(body.contains(r#""enabled_vhosts":1"#));
18361858
assert!(body.contains(r#""tiered_vhosts":1"#));
1859+
assert!(body.contains(r#""enabled_routes":1"#));
1860+
assert!(body.contains(r#""tiered_routes":0"#));
18371861
assert!(body.contains(r#""memory_entries":0"#));
18381862
assert!(body.contains(r#""disk_entries":0"#));
18391863
assert!(body.contains(r#""activity":{"hits":0,"misses":0,"stores":0"#));
@@ -1842,6 +1866,7 @@ mod tests {
18421866
assert!(body.contains(r#""tiered":true"#));
18431867
assert!(body.contains(r#""memory":{"entries":0"#));
18441868
assert!(body.contains(r#""disk":{"entries":0"#));
1869+
assert!(body.contains(r#""routes":[{"name":"assets""#));
18451870

18461871
std::fs::remove_dir_all(cache_path).unwrap();
18471872
}
@@ -1875,7 +1900,7 @@ mod tests {
18751900
},
18761901
headers: crate::config::VhostHeaderPolicyConfig::default(),
18771902
web: WebConfig::default(),
1878-
routes: Vec::new(),
1903+
routes: vec![cached_assets_route()],
18791904
}],
18801905
..Config::default()
18811906
};
@@ -1899,9 +1924,10 @@ mod tests {
18991924
assert_eq!(response.status, StatusCode::OK);
19001925
let body = String::from_utf8(response.body).unwrap();
19011926
assert!(body.contains(r#""status":"ok""#));
1902-
assert!(body.contains(r#""memory_tiers":1"#));
1927+
assert!(body.contains(r#""memory_tiers":2"#));
19031928
assert!(body.contains(r#""disk_tiers":1"#));
19041929
assert!(body.contains(r#""tiered_vhosts":1"#));
1930+
assert!(body.contains(r#""tiered_routes":0"#));
19051931

19061932
std::fs::remove_dir_all(cache_path).unwrap();
19071933
}
@@ -2850,6 +2876,32 @@ mod tests {
28502876
headers
28512877
}
28522878

2879+
#[cfg(feature = "cache")]
2880+
fn cached_assets_route() -> RouteConfig {
2881+
RouteConfig {
2882+
name: "assets".to_owned(),
2883+
path_exact: None,
2884+
path_prefix: Some("/assets/".to_owned()),
2885+
fallback: false,
2886+
https_redirect_exempt: false,
2887+
strip_prefix: None,
2888+
max_request_body_bytes: None,
2889+
redirect: None,
2890+
proxy: Some(ProxyConfig::default()),
2891+
web: None,
2892+
cache: Some(CacheConfig {
2893+
enabled: true,
2894+
memory: crate::config::CacheMemoryConfig {
2895+
enabled: true,
2896+
max_size_bytes: ByteSize::from_bytes(1024),
2897+
},
2898+
max_object_bytes: ByteSize::from_bytes(512),
2899+
..CacheConfig::default()
2900+
}),
2901+
headers: crate::config::VhostHeaderPolicyConfig::default(),
2902+
}
2903+
}
2904+
28532905
struct TestDir {
28542906
path: std::path::PathBuf,
28552907
}

‎src/proxy.rs‎

Lines changed: 97 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,8 @@ pub struct CacheRuntimeTotals {
357357
pub vhosts: u64,
358358
pub enabled_vhosts: u64,
359359
pub tiered_vhosts: u64,
360+
pub enabled_routes: u64,
361+
pub tiered_routes: u64,
360362
pub memory_entries: u64,
361363
pub memory_weighted_size_bytes: u64,
362364
pub memory_max_size_bytes: u64,
@@ -378,6 +380,17 @@ pub struct CacheVhostStats {
378380
pub tiered: bool,
379381
pub memory: Option<crate::cache::MemoryCacheStats>,
380382
pub disk: Option<crate::cache::DiskCacheStats>,
383+
pub routes: Vec<CacheRouteStats>,
384+
}
385+
386+
#[cfg(feature = "cache")]
387+
#[derive(Debug, Clone, Eq, PartialEq)]
388+
pub struct CacheRouteStats {
389+
pub name: String,
390+
pub enabled: bool,
391+
pub tiered: bool,
392+
pub memory: Option<crate::cache::MemoryCacheStats>,
393+
pub disk: Option<crate::cache::DiskCacheStats>,
381394
}
382395

383396
#[cfg(feature = "cache")]
@@ -386,6 +399,7 @@ pub struct CacheActivityResetResult {
386399
pub memory_tiers: u64,
387400
pub disk_tiers: u64,
388401
pub tiered_vhosts: u64,
402+
pub tiered_routes: u64,
389403
}
390404

391405
impl ProxySnapshot {
@@ -446,40 +460,36 @@ impl ProxySnapshot {
446460
}
447461

448462
let memory = vhost.pingora_memory_storage.map(|storage| storage.stats());
449-
if let Some(memory) = memory {
450-
totals.memory_entries = totals.memory_entries.saturating_add(memory.entries);
451-
totals.memory_weighted_size_bytes = totals
452-
.memory_weighted_size_bytes
453-
.saturating_add(memory.weighted_size_bytes);
454-
totals.memory_max_size_bytes = totals
455-
.memory_max_size_bytes
456-
.saturating_add(memory.max_size_bytes.as_u64());
457-
totals.hits = totals.hits.saturating_add(memory.activity.hits);
458-
totals.misses = totals.misses.saturating_add(memory.activity.misses);
459-
totals.stores = totals.stores.saturating_add(memory.activity.stores);
460-
totals.store_refusals = totals
461-
.store_refusals
462-
.saturating_add(memory.activity.store_refusals);
463-
totals.purges = totals.purges.saturating_add(memory.activity.purges);
464-
}
465-
466463
let disk = vhost
467464
.pingora_disk_storage
468465
.map(|storage| storage.stats())
469466
.transpose()?;
470-
if let Some(disk) = disk {
471-
totals.disk_entries = totals.disk_entries.saturating_add(disk.entries);
472-
totals.disk_size_bytes = totals.disk_size_bytes.saturating_add(disk.size_bytes);
473-
totals.disk_max_size_bytes = totals
474-
.disk_max_size_bytes
475-
.saturating_add(disk.max_size_bytes.as_u64());
476-
totals.hits = totals.hits.saturating_add(disk.activity.hits);
477-
totals.misses = totals.misses.saturating_add(disk.activity.misses);
478-
totals.stores = totals.stores.saturating_add(disk.activity.stores);
479-
totals.store_refusals = totals
480-
.store_refusals
481-
.saturating_add(disk.activity.store_refusals);
482-
totals.purges = totals.purges.saturating_add(disk.activity.purges);
467+
accumulate_cache_stats(&mut totals, memory.as_ref(), disk.as_ref());
468+
469+
let mut routes = Vec::new();
470+
for route in &vhost.routes {
471+
let Some(cache) = &route.cache else {
472+
continue;
473+
};
474+
if cache.config.enabled {
475+
totals.enabled_routes = totals.enabled_routes.saturating_add(1);
476+
}
477+
if cache.pingora_tiered_storage.is_some() {
478+
totals.tiered_routes = totals.tiered_routes.saturating_add(1);
479+
}
480+
let route_memory = cache.pingora_memory_storage.map(|storage| storage.stats());
481+
let route_disk = cache
482+
.pingora_disk_storage
483+
.map(|storage| storage.stats())
484+
.transpose()?;
485+
accumulate_cache_stats(&mut totals, route_memory.as_ref(), route_disk.as_ref());
486+
routes.push(CacheRouteStats {
487+
name: cache.name.clone(),
488+
enabled: cache.config.enabled,
489+
tiered: cache.pingora_tiered_storage.is_some(),
490+
memory: route_memory,
491+
disk: route_disk,
492+
});
483493
}
484494

485495
vhosts.push(CacheVhostStats {
@@ -488,6 +498,7 @@ impl ProxySnapshot {
488498
tiered: vhost.pingora_tiered_storage.is_some(),
489499
memory,
490500
disk,
501+
routes,
491502
});
492503
}
493504
Ok(CacheRuntimeStats { totals, vhosts })
@@ -499,6 +510,7 @@ impl ProxySnapshot {
499510
memory_tiers: 0,
500511
disk_tiers: 0,
501512
tiered_vhosts: 0,
513+
tiered_routes: 0,
502514
};
503515
for vhost in &self.state.vhosts {
504516
if let Some(storage) = vhost.pingora_memory_storage {
@@ -512,6 +524,22 @@ impl ProxySnapshot {
512524
if vhost.pingora_tiered_storage.is_some() {
513525
result.tiered_vhosts = result.tiered_vhosts.saturating_add(1);
514526
}
527+
for route in &vhost.routes {
528+
let Some(cache) = &route.cache else {
529+
continue;
530+
};
531+
if let Some(storage) = cache.pingora_memory_storage {
532+
storage.reset_activity();
533+
result.memory_tiers = result.memory_tiers.saturating_add(1);
534+
}
535+
if let Some(storage) = cache.pingora_disk_storage {
536+
storage.reset_activity();
537+
result.disk_tiers = result.disk_tiers.saturating_add(1);
538+
}
539+
if cache.pingora_tiered_storage.is_some() {
540+
result.tiered_routes = result.tiered_routes.saturating_add(1);
541+
}
542+
}
515543
}
516544
result
517545
}
@@ -601,6 +629,45 @@ impl ProxySnapshot {
601629
}
602630
}
603631

632+
#[cfg(feature = "cache")]
633+
fn accumulate_cache_stats(
634+
totals: &mut CacheRuntimeTotals,
635+
memory: Option<&crate::cache::MemoryCacheStats>,
636+
disk: Option<&crate::cache::DiskCacheStats>,
637+
) {
638+
if let Some(memory) = memory {
639+
totals.memory_entries = totals.memory_entries.saturating_add(memory.entries);
640+
totals.memory_weighted_size_bytes = totals
641+
.memory_weighted_size_bytes
642+
.saturating_add(memory.weighted_size_bytes);
643+
totals.memory_max_size_bytes = totals
644+
.memory_max_size_bytes
645+
.saturating_add(memory.max_size_bytes.as_u64());
646+
totals.hits = totals.hits.saturating_add(memory.activity.hits);
647+
totals.misses = totals.misses.saturating_add(memory.activity.misses);
648+
totals.stores = totals.stores.saturating_add(memory.activity.stores);
649+
totals.store_refusals = totals
650+
.store_refusals
651+
.saturating_add(memory.activity.store_refusals);
652+
totals.purges = totals.purges.saturating_add(memory.activity.purges);
653+
}
654+
655+
if let Some(disk) = disk {
656+
totals.disk_entries = totals.disk_entries.saturating_add(disk.entries);
657+
totals.disk_size_bytes = totals.disk_size_bytes.saturating_add(disk.size_bytes);
658+
totals.disk_max_size_bytes = totals
659+
.disk_max_size_bytes
660+
.saturating_add(disk.max_size_bytes.as_u64());
661+
totals.hits = totals.hits.saturating_add(disk.activity.hits);
662+
totals.misses = totals.misses.saturating_add(disk.activity.misses);
663+
totals.stores = totals.stores.saturating_add(disk.activity.stores);
664+
totals.store_refusals = totals
665+
.store_refusals
666+
.saturating_add(disk.activity.store_refusals);
667+
totals.purges = totals.purges.saturating_add(disk.activity.purges);
668+
}
669+
}
670+
604671
impl ProxyRuntimeState {
605672
fn from_config(config: &Config) -> io::Result<Self> {
606673
#[cfg(feature = "load-balancer")]

0 commit comments

Comments
 (0)