Skip to content

Commit 37b54ce

Browse files
evanflodenevanfloden
committed
Refine scheduler performance gains reporting
1 parent 8d15712 commit 37b54ce

2 files changed

Lines changed: 243 additions & 25 deletions

File tree

‎modules/local/benchmark_report/overrides/functions/machine_metrics.R‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,25 @@ safe_weighted_mean <- function(values, weights) {
66
sum(values[valid] * weights[valid]) / sum(weights[valid])
77
}
88

9+
positive_gap <- function(upper, lower) {
10+
dplyr::if_else(
11+
is.na(upper) | is.na(lower),
12+
NA_real_,
13+
pmax(upper - lower, 0)
14+
)
15+
}
16+
17+
compute_scheduler_booked <- function(capacity, efficiency_pct, fallback = NA_real_) {
18+
booked_from_capacity <- dplyr::if_else(
19+
!is.na(capacity) & !is.na(efficiency_pct),
20+
pmin(capacity, pmax(0, capacity * efficiency_pct / 100)),
21+
NA_real_
22+
)
23+
24+
fallback <- dplyr::if_else(is.na(fallback), NA_real_, pmax(fallback, 0))
25+
dplyr::coalesce(booked_from_capacity, fallback)
26+
}
27+
928
parse_machine_percent <- function(x) {
1029
if (is.numeric(x)) {
1130
return(as.numeric(x))
@@ -111,6 +130,16 @@ summarise_machine_metrics <- function(machine_data, run_lookup, task_run_metrics
111130
by = c("run_id", "pipeline")
112131
) %>%
113132
dplyr::mutate(
133+
requestedVmCpuEfficiency = dplyr::if_else(vmCpuH > 0, requestedCpuH / vmCpuH * 100, NA_real_),
134+
requestedVmMemEfficiency = dplyr::if_else(vmMemGibH > 0, requestedMemGibH / vmMemGibH * 100, NA_real_),
135+
schedulerBookedCpuH = compute_scheduler_booked(vmCpuH, schedAllocCpuEfficiency, requestedCpuH),
136+
schedulerBookedMemGibH = compute_scheduler_booked(vmMemGibH, schedAllocMemEfficiency, requestedMemGibH),
137+
schedulerRightsizedCpuH = positive_gap(requestedCpuH, schedulerBookedCpuH),
138+
schedulerRightsizedMemGibH = positive_gap(requestedMemGibH, schedulerBookedMemGibH),
139+
schedulerOverbookCpuH = positive_gap(schedulerBookedCpuH, realCpuH),
140+
schedulerOverbookMemGibH = positive_gap(schedulerBookedMemGibH, realMemGibH),
141+
vmPackingSlackCpuH = positive_gap(vmCpuH, schedulerBookedCpuH),
142+
vmPackingSlackMemGibH = positive_gap(vmMemGibH, schedulerBookedMemGibH),
114143
realVmCpuEfficiency = dplyr::if_else(vmCpuH > 0, realCpuH / vmCpuH * 100, NA_real_),
115144
realVmMemEfficiency = dplyr::if_else(vmMemGibH > 0, realMemGibH / vmMemGibH * 100, NA_real_)
116145
)

‎modules/local/benchmark_report/overrides/quarto_content/run_overview.qmd‎

Lines changed: 214 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ run_overview <- reactable(
125125

126126
## Run metrics
127127

128-
This section provides a visual overview of the pipeline run metrics. Task-derived efficiencies are recomputed from the task logs, and optional scheduler/VM metrics are added when machine CSVs are available.
128+
This section provides a visual overview of the pipeline run metrics. When machine CSVs are available, the report also decomposes each run into four layers: user-requested resources, scheduler-booked resources, real task usage, and total VM capacity.
129129

130130
```{r run-metrics}
131131
library(RColorBrewer)
@@ -177,10 +177,23 @@ if (params$aws_cost != "NA") {
177177
}
178178
179179
## Select which columns to keep
180+
vm_layer_metric_cols <- c(
181+
"nMachines", "vmCpuH", "vmMemGibH",
182+
"requestedCpuH", "requestedMemGibH",
183+
"requestedVmCpuEfficiency", "requestedVmMemEfficiency",
184+
"schedAllocCpuEfficiency", "schedAllocMemEfficiency",
185+
"schedulerBookedCpuH", "schedulerBookedMemGibH",
186+
"schedulerRightsizedCpuH", "schedulerRightsizedMemGibH",
187+
"schedulerOverbookCpuH", "schedulerOverbookMemGibH",
188+
"vmPackingSlackCpuH", "vmPackingSlackMemGibH",
189+
"realVmCpuEfficiency", "realVmMemEfficiency",
190+
"realCpuH", "realMemGibH"
191+
)
192+
180193
if(grepl("cost",profile) & params$aws_cost != "NA") {
181-
merged_logs <- merged_logs %>% select(any_of(c("pipeline","group","run_id","Wall_time","duration","cpuTime","task_runtime_ms","cost","readBytes","writeBytes","cpuEfficiency","memoryEfficiency","pipeline_workdirSize_GB","pipeline_workdirSize_price","used_cost","unused_cost","nMachines","vmCpuH","vmMemGibH","schedAllocCpuEfficiency","schedAllocMemEfficiency","realVmCpuEfficiency","realVmMemEfficiency","realCpuH","realMemGibH","requestedCpuH","requestedMemGibH")))
194+
merged_logs <- merged_logs %>% select(any_of(c("pipeline","group","run_id","Wall_time","duration","cpuTime","task_runtime_ms","cost","readBytes","writeBytes","cpuEfficiency","memoryEfficiency","pipeline_workdirSize_GB","pipeline_workdirSize_price","used_cost","unused_cost", vm_layer_metric_cols)))
182195
}else {
183-
merged_logs <- merged_logs %>% select(any_of(c("pipeline","group","run_id","Wall_time","duration","cpuTime","task_runtime_ms","cost","readBytes","writeBytes","cpuEfficiency","memoryEfficiency","pipeline_workdirSize_GB","pipeline_workdirSize_price","nMachines","vmCpuH","vmMemGibH","schedAllocCpuEfficiency","schedAllocMemEfficiency","realVmCpuEfficiency","realVmMemEfficiency","realCpuH","realMemGibH","requestedCpuH","requestedMemGibH")))
196+
merged_logs <- merged_logs %>% select(any_of(c("pipeline","group","run_id","Wall_time","duration","cpuTime","task_runtime_ms","cost","readBytes","writeBytes","cpuEfficiency","memoryEfficiency","pipeline_workdirSize_GB","pipeline_workdirSize_price", vm_layer_metric_cols)))
184197
}
185198
186199
metrics_reactable <- merged_logs %>%
@@ -466,29 +479,34 @@ if(n_runs_max > 1){
466479
}
467480
```
468481

482+
## Scheduler VM metrics
483+
469484
```{r vm-metrics-table}
470485
#| echo: false
471486
#| results: asis
472487
473-
vm_metrics_cols <- c("nMachines", "vmCpuH", "vmMemGibH", "schedAllocCpuEfficiency", "schedAllocMemEfficiency", "realVmCpuEfficiency", "realVmMemEfficiency", "realCpuH", "realMemGibH", "requestedCpuH", "requestedMemGibH")
488+
vm_metrics_cols <- c(
489+
"nMachines", "vmCpuH", "vmMemGibH",
490+
"requestedCpuH", "requestedMemGibH",
491+
"requestedVmCpuEfficiency", "requestedVmMemEfficiency",
492+
"schedulerBookedCpuH", "schedulerBookedMemGibH",
493+
"schedAllocCpuEfficiency", "schedAllocMemEfficiency",
494+
"realCpuH", "realMemGibH",
495+
"realVmCpuEfficiency", "realVmMemEfficiency",
496+
"schedulerRightsizedCpuH", "schedulerRightsizedMemGibH",
497+
"schedulerOverbookCpuH", "schedulerOverbookMemGibH",
498+
"vmPackingSlackCpuH", "vmPackingSlackMemGibH"
499+
)
500+
501+
has_vm_metrics <- all(vm_metrics_cols %in% colnames(metrics_reactable)) && any(!is.na(metrics_reactable$vmCpuH))
474502
475-
if (all(vm_metrics_cols %in% colnames(metrics_reactable)) && any(!is.na(metrics_reactable$vmCpuH))) {
476-
cat("## Scheduler VM metrics\n\n")
477-
cat("These values are computed from optional machine CSVs when they are provided alongside each run dump. Task efficiencies are derived from task logs, while VM efficiencies relate those task metrics back to total machine lifetime capacity.\n\n")
503+
if (has_vm_metrics) {
504+
cat("These values separate the three scheduler layers the benchmark cares about: user-requested resources, scheduler-booked resources, and VM capacity. `*-Predv1` groups can show positive scheduler right-sizing, while `Batch-*` groups act as the AWS Batch baseline without Seqera scheduler right-sizing.\n\n")
478505
479506
vm_metrics_table <- metrics_reactable %>%
480507
select(pipeline, group, run_id, all_of(vm_metrics_cols)) %>%
481508
mutate(
482-
vmCpuH = round(vmCpuH, 2),
483-
vmMemGibH = round(vmMemGibH, 2),
484-
schedAllocCpuEfficiency = round(schedAllocCpuEfficiency, 2),
485-
schedAllocMemEfficiency = round(schedAllocMemEfficiency, 2),
486-
realVmCpuEfficiency = round(realVmCpuEfficiency, 2),
487-
realVmMemEfficiency = round(realVmMemEfficiency, 2),
488-
realCpuH = round(realCpuH, 2),
489-
realMemGibH = round(realMemGibH, 2),
490-
requestedCpuH = round(requestedCpuH, 2),
491-
requestedMemGibH = round(requestedMemGibH, 2)
509+
across(-c(pipeline, group, run_id), ~ round(.x, 2))
492510
)
493511
494512
vm_metrics_reactable <- reactable(
@@ -502,14 +520,24 @@ if (all(vm_metrics_cols %in% colnames(metrics_reactable)) && any(!is.na(metrics_
502520
nMachines = colDef(name = "Machines"),
503521
vmCpuH = colDef(name = "VM CPU<br>hours", html = TRUE),
504522
vmMemGibH = colDef(name = "VM memory<br>GiB hours", html = TRUE),
505-
schedAllocCpuEfficiency = colDef(name = "Scheduler CPU<br>efficiency (%)", html = TRUE),
506-
schedAllocMemEfficiency = colDef(name = "Scheduler memory<br>efficiency (%)", html = TRUE),
507-
realVmCpuEfficiency = colDef(name = "Real VM CPU<br>efficiency (%)", html = TRUE),
508-
realVmMemEfficiency = colDef(name = "Real VM memory<br>efficiency (%)", html = TRUE),
509-
realCpuH = colDef(name = "Real CPU<br>hours", html = TRUE),
510-
realMemGibH = colDef(name = "Real memory<br>GiB hours", html = TRUE),
511-
requestedCpuH = colDef(name = "Requested CPU<br>hours", html = TRUE),
512-
requestedMemGibH = colDef(name = "Requested memory<br>GiB hours", html = TRUE)
523+
requestedCpuH = colDef(name = "User requested<br>CPU hours", html = TRUE),
524+
requestedMemGibH = colDef(name = "User requested<br>memory GiB hours", html = TRUE),
525+
requestedVmCpuEfficiency = colDef(name = "User requested / VM<br>CPU (%)", html = TRUE),
526+
requestedVmMemEfficiency = colDef(name = "User requested / VM<br>memory (%)", html = TRUE),
527+
schedulerBookedCpuH = colDef(name = "Scheduler booked<br>CPU hours", html = TRUE),
528+
schedulerBookedMemGibH = colDef(name = "Scheduler booked<br>memory GiB hours", html = TRUE),
529+
schedAllocCpuEfficiency = colDef(name = "Scheduler booked / VM<br>CPU (%)", html = TRUE),
530+
schedAllocMemEfficiency = colDef(name = "Scheduler booked / VM<br>memory (%)", html = TRUE),
531+
realCpuH = colDef(name = "Real used<br>CPU hours", html = TRUE),
532+
realMemGibH = colDef(name = "Real used<br>memory GiB hours", html = TRUE),
533+
realVmCpuEfficiency = colDef(name = "Real used / VM<br>CPU (%)", html = TRUE),
534+
realVmMemEfficiency = colDef(name = "Real used / VM<br>memory (%)", html = TRUE),
535+
schedulerRightsizedCpuH = colDef(name = "Scheduler right-sized<br>CPU hours", html = TRUE),
536+
schedulerRightsizedMemGibH = colDef(name = "Scheduler right-sized<br>memory GiB hours", html = TRUE),
537+
schedulerOverbookCpuH = colDef(name = "Scheduler over-booked<br>CPU hours", html = TRUE),
538+
schedulerOverbookMemGibH = colDef(name = "Scheduler over-booked<br>memory GiB hours", html = TRUE),
539+
vmPackingSlackCpuH = colDef(name = "VM packing slack<br>CPU hours", html = TRUE),
540+
vmPackingSlackMemGibH = colDef(name = "VM packing slack<br>memory GiB hours", html = TRUE)
513541
),
514542
wrap = FALSE,
515543
resizable = TRUE,
@@ -523,3 +551,164 @@ if (all(vm_metrics_cols %in% colnames(metrics_reactable)) && any(!is.na(metrics_
523551
print(vm_metrics_reactable)
524552
}
525553
```
554+
555+
## Performance gains
556+
557+
```{r performance-gains}
558+
#| echo: false
559+
#| results: asis
560+
#| fig-width: 12
561+
#| fig-height: 6
562+
563+
if (has_vm_metrics) {
564+
performance_layers <- metrics_reactable %>%
565+
filter(!is.na(vmCpuH), vmCpuH > 0) %>%
566+
mutate(
567+
display_group = as.character(group),
568+
display_label = if (dplyr::n_distinct(group) == n()) display_group else paste0(display_group, " / ", run_id)
569+
) %>%
570+
arrange(vmCpuH)
571+
572+
cat("This section attributes paid VM capacity to the three scheduler responsibilities you called out: provisioning, packing, and task right-sizing. `User requested` comes from the Nextflow task directives, `scheduler booked` comes from machine-level scheduler allocation, `real used` comes from observed task usage, and `VM capacity` comes from total machine lifetime.\n\n")
573+
cat("For `*-Predv1` runs, scheduler right-sizing is estimated from aggregate `/machines` utilization. When that aggregate is noisier than the right-sizing signal, the report clamps negative right-sizing to zero. That is an under-count, not evidence that right-sizing was absent.\n\n")
574+
}
575+
```
576+
577+
```{r performance-gains-cpu-mix}
578+
#| echo: false
579+
#| warning: false
580+
#| message: false
581+
#| fig-width: 12
582+
#| fig-height: 5
583+
584+
if (has_vm_metrics) {
585+
cpu_mix_data <- performance_layers %>%
586+
transmute(
587+
display_label,
588+
`Used (real work)` = realCpuH,
589+
`Scheduler over-booked` = schedulerOverbookCpuH,
590+
`VM slack (packing)` = vmPackingSlackCpuH
591+
) %>%
592+
pivot_longer(-display_label, names_to = "layer", values_to = "hours")
593+
594+
cpu_mix_plot <- ggplot(cpu_mix_data, aes(x = display_label, y = hours, fill = layer)) +
595+
geom_col(color = "#160F26") +
596+
coord_flip() +
597+
scale_fill_manual(values = c(
598+
"Used (real work)" = "#065647",
599+
"Scheduler over-booked" = "#2b8cbe",
600+
"VM slack (packing)" = "#cfd0d1"
601+
)) +
602+
labs(
603+
title = "Performance gains: CPU capacity mix",
604+
subtitle = "Total bar = VM CPU-hours paid for",
605+
x = "",
606+
y = "CPU-hours",
607+
fill = ""
608+
) +
609+
seqera_light_theme() +
610+
theme(legend.position = "top")
611+
612+
print(cpu_mix_plot)
613+
}
614+
```
615+
616+
```{r performance-gains-mem-mix}
617+
#| echo: false
618+
#| warning: false
619+
#| message: false
620+
#| fig-width: 12
621+
#| fig-height: 5
622+
623+
if (has_vm_metrics) {
624+
mem_mix_data <- performance_layers %>%
625+
transmute(
626+
display_label,
627+
`Used (real work)` = realMemGibH,
628+
`Scheduler over-booked` = schedulerOverbookMemGibH,
629+
`VM slack (packing)` = vmPackingSlackMemGibH
630+
) %>%
631+
pivot_longer(-display_label, names_to = "layer", values_to = "hours")
632+
633+
mem_mix_plot <- ggplot(mem_mix_data, aes(x = display_label, y = hours, fill = layer)) +
634+
geom_col(color = "#160F26") +
635+
coord_flip() +
636+
scale_fill_manual(values = c(
637+
"Used (real work)" = "#065647",
638+
"Scheduler over-booked" = "#2b8cbe",
639+
"VM slack (packing)" = "#cfd0d1"
640+
)) +
641+
labs(
642+
title = "Performance gains: Memory capacity mix",
643+
subtitle = "Total bar = VM memory GiB-hours paid for",
644+
x = "",
645+
y = "GiB-hours",
646+
fill = ""
647+
) +
648+
seqera_light_theme() +
649+
theme(legend.position = "top")
650+
651+
print(mem_mix_plot)
652+
}
653+
```
654+
655+
```{r performance-gains-savings}
656+
#| echo: false
657+
#| warning: false
658+
#| message: false
659+
#| fig-width: 12
660+
#| fig-height: 8
661+
662+
if (has_vm_metrics) {
663+
savings_cpu_data <- performance_layers %>%
664+
transmute(
665+
display_label,
666+
`Scheduler right-sized (already saved)` = schedulerRightsizedCpuH,
667+
`Scheduler over-booked (remaining)` = schedulerOverbookCpuH,
668+
`VM packing slack` = vmPackingSlackCpuH
669+
) %>%
670+
pivot_longer(-display_label, names_to = "layer", values_to = "hours")
671+
672+
savings_mem_data <- performance_layers %>%
673+
transmute(
674+
display_label,
675+
`Scheduler right-sized (already saved)` = schedulerRightsizedMemGibH,
676+
`Scheduler over-booked (remaining)` = schedulerOverbookMemGibH,
677+
`VM packing slack` = vmPackingSlackMemGibH
678+
) %>%
679+
pivot_longer(-display_label, names_to = "layer", values_to = "hours")
680+
681+
savings_palette <- c(
682+
"Scheduler right-sized (already saved)" = "#43b48d",
683+
"Scheduler over-booked (remaining)" = "#2b8cbe",
684+
"VM packing slack" = "#cfd0d1"
685+
)
686+
687+
cpu_savings_plot <- ggplot(savings_cpu_data, aes(x = display_label, y = hours, fill = layer)) +
688+
geom_col(color = "#160F26") +
689+
scale_fill_manual(values = savings_palette) +
690+
labs(
691+
title = "Potential savings by layer (CPU)",
692+
x = "",
693+
y = "CPU-hours",
694+
fill = ""
695+
) +
696+
seqera_light_theme() +
697+
theme(legend.position = "top")
698+
699+
mem_savings_plot <- ggplot(savings_mem_data, aes(x = display_label, y = hours, fill = layer)) +
700+
geom_col(color = "#160F26") +
701+
scale_fill_manual(values = savings_palette) +
702+
labs(
703+
title = "Potential savings by layer (Memory)",
704+
x = "",
705+
y = "GiB-hours",
706+
fill = ""
707+
) +
708+
seqera_light_theme() +
709+
theme(legend.position = "top")
710+
711+
print(cpu_savings_plot)
712+
print(mem_savings_plot)
713+
}
714+
```

0 commit comments

Comments
 (0)