Enhance benchmark reporting by adding total duration and pass count metrics

Author: 0hmX

scripts/highdensity-benchmark/index.ts

@@ -4,13 +4,16 @@ import { parseArgs } from "node:util"
 import { availableParallelism } from "node:os"
 import { calculateMse } from "./metrics/calculateMse.ts"
 import { runBenchmarkWithWorkers } from "./runBenchmarkWithWorkers/index.ts"
+import { formatSeconds } from "./runBenchmarkWithWorkers/shared.ts"
 import { NodeWithPortPoints } from "lib/types/high-density-types.ts"
 
 type CliOptions = {
   concurrency: number
   timeoutSeconds: number
 }
 
+const formatPercent = (value: number) => `${(value * 100).toFixed(1)}%`
+
 const usage = () =>
   [
     "Usage: bun scripts/highdensity-benchmark/index.ts [options]",
@@ -82,10 +85,18 @@ const main = async ({ concurrency, timeoutSeconds }: CliOptions) => {
     concurrency,
     timeoutMs: timeoutSeconds * 1000,
   })
+  const completedCount = completedScores.results.length
+  const passRate =
+    completedCount === 0 ? 0 : completedScores.passCount / completedCount
 
   // Avoid a divide-by-zero MSE when every single problem times out.
-  if (completedScores.results.length === 0) {
+  if (completedCount === 0) {
+    console.log(
+      "Total duration:",
+      `${formatSeconds(completedScores.totalDurationMs)} seconds`,
+    )
     console.log("Completed problems: 0")
+    console.log("Pass rate: 0.0% (0/0)")
     console.log(
       "Timed out problems:",
       completedScores.timedOutProblemIds.length,
@@ -95,7 +106,15 @@ const main = async ({ concurrency, timeoutSeconds }: CliOptions) => {
   }
 
   const mse = calculateMse(completedScores.results)
-  console.log("Completed problems:", completedScores.results.length)
+  console.log(
+    "Total duration:",
+    `${formatSeconds(completedScores.totalDurationMs)} seconds`,
+  )
+  console.log("Completed problems:", completedCount)
+  console.log(
+    "Pass rate:",
+    `${formatPercent(passRate)} (${completedScores.passCount}/${completedCount})`,
+  )
   console.log("Timed out problems:", completedScores.timedOutProblemIds.length)
   console.log("Mean Squared Error:", mse)
 }

scripts/highdensity-benchmark/runBenchmarkWithWorkers/BenchmarkWorkerPool.ts

@@ -3,7 +3,6 @@ import {
   BenchmarkTask,
   RunBenchmarkOptions,
   RunBenchmarkResult,
-  formatSeconds,
   getProblemId,
 } from "./shared.ts"
 
@@ -13,6 +12,7 @@ export class BenchmarkWorkerPool {
   private readonly timeoutMs: number
   private readonly results: number[] = []
   private readonly timedOutProblemIds: string[] = []
+  private passCount = 0
   private completedProblems = 0
   private nextTaskIndex = 0
   private nextTaskId = 1
@@ -45,13 +45,17 @@ export class BenchmarkWorkerPool {
       return {
         results: [],
         timedOutProblemIds: [],
+        totalDurationMs: 0,
+        passCount: 0,
       }
     }
 
     console.log(
       `Starting high-density benchmark with ${this.workers.length} workers across ${this.tasks.length} cases`,
     )
 
+    const startedAt = Date.now()
+
     try {
       await this.runWorkers()
     } finally {
@@ -61,6 +65,8 @@ export class BenchmarkWorkerPool {
     return {
       results: this.results,
       timedOutProblemIds: this.timedOutProblemIds,
+      totalDurationMs: Date.now() - startedAt,
+      passCount: this.passCount,
     }
   }
 
@@ -76,14 +82,16 @@ export class BenchmarkWorkerPool {
       )
       if (result === null) {
         this.timedOutProblemIds.push(task.problemId)
+        this.logProgress()
         continue
       }
 
       this.completedProblems += 1
-      console.log(
-        `${task.problemId} ${result.solved ? "pass" : "fail"} solve in ${formatSeconds(result.solveDurationMs)} seconds (${this.completedProblems}/${this.tasks.length})`,
-      )
+      if (result.solved) {
+        this.passCount += 1
+      }
       this.results.push(result.value)
+      this.logProgress()
     }
   }
 
@@ -110,4 +118,17 @@ export class BenchmarkWorkerPool {
     this.nextTaskIndex += 1
     return task
   }
+
+  private logProgress() {
+    const processedProblems =
+      this.completedProblems + this.timedOutProblemIds.length
+    const completedRatio =
+      this.completedProblems === 0 ? 0 : this.passCount / this.completedProblems
+
+    console.log(
+      `Progress ${processedProblems}/${this.tasks.length} | pass ${(
+        completedRatio * 100
+      ).toFixed(1)}%`,
+    )
+  }
 }

scripts/highdensity-benchmark/runBenchmarkWithWorkers/shared.ts

@@ -40,6 +40,8 @@ export type RunBenchmarkOptions = {
 export type RunBenchmarkResult = {
   results: number[]
   timedOutProblemIds: string[]
+  totalDurationMs: number
+  passCount: number
 }
 
 export const getProblemId = (problem: NodeWithPortPoints, index: number) => {