[BENCHMARK TEST] Tune max iterations/hyper parameters to get more consistent runtimes (-2% on benchmark, 10% -> 3% timeout) (#639)

* add support for single sample profiling

* wip

* Tune max iterations to avoid timing out

* format

Author: seveibar

lib/autorouter-pipelines/AutoroutingPipeline2_PortPointPathing/AutoroutingPipelineSolver2_PortPointPathing.ts

@@ -338,6 +338,7 @@ export class AutoroutingPipelineSolver2_PortPointPathing extends BaseSolver {
         connMap: cms.connMap,
         viaDiameter: cms.viaDiameter,
         traceWidth: cms.minTraceWidth,
+        effort: cms.effort,
       },
     ]),
     definePipelineStep(

lib/solvers/HighDensitySolver/HighDensitySolver.ts

@@ -26,6 +26,7 @@ export class HighDensitySolver extends BaseSolver {
   readonly defaultTraceThickness = 0.15
   viaDiameter: number
   traceWidth: number
+  effort: number
 
   failedSolvers: (IntraNodeRouteSolver | HyperSingleIntraNodeSolver)[]
   activeSubSolver: IntraNodeRouteSolver | HyperSingleIntraNodeSolver | null =
@@ -38,20 +39,23 @@ export class HighDensitySolver extends BaseSolver {
     connMap,
     viaDiameter,
     traceWidth,
+    effort,
   }: {
     nodePortPoints: NodeWithPortPoints[]
     colorMap?: Record<string, string>
     connMap?: ConnectivityMap
     viaDiameter?: number
     traceWidth?: number
+    effort?: number
   }) {
     super()
     this.unsolvedNodePortPoints = nodePortPoints
     this.colorMap = colorMap ?? {}
     this.connMap = connMap
     this.routes = []
     this.failedSolvers = []
-    this.MAX_ITERATIONS = 50e6
+    this.effort = effort ?? 1
+    this.MAX_ITERATIONS = 10e6 * this.effort
     this.viaDiameter = viaDiameter ?? this.defaultViaDiameter
     this.traceWidth = traceWidth ?? this.defaultTraceThickness
   }
@@ -96,6 +100,7 @@ export class HighDensitySolver extends BaseSolver {
       connMap: this.connMap,
       viaDiameter: this.viaDiameter,
       traceWidth: this.traceWidth,
+      effort: this.effort,
     })
     this.updateCacheStats()
   }

lib/solvers/HyperHighDensitySolver/HyperSingleIntraNodeSolver.ts

@@ -32,15 +32,19 @@ export class HyperSingleIntraNodeSolver extends HyperParameterSupervisorSolver<
   solvedRoutes: HighDensityIntraNodeRoute[] = []
   nodeWithPortPoints: NodeWithPortPoints
   connMap?: ConnectivityMap
+  effort: number
 
   constructor(
-    opts: ConstructorParameters<typeof CachedIntraNodeRouteSolver>[0],
+    opts: ConstructorParameters<typeof CachedIntraNodeRouteSolver>[0] & {
+      effort?: number
+    },
   ) {
     super()
     this.nodeWithPortPoints = opts.nodeWithPortPoints
     this.connMap = opts.connMap
     this.constructorParams = opts
-    this.MAX_ITERATIONS = 30_000_000
+    this.effort = opts.effort ?? 1
+    this.MAX_ITERATIONS = 10_000_000 * this.effort
     this.GREEDY_MULTIPLIER = 5
     this.MIN_SUBSTEPS = 100
   }
@@ -138,7 +142,7 @@ export class HyperSingleIntraNodeSolver extends HyperParameterSupervisorSolver<
       },
       {
         name: "orderings50",
-        possibleValues: Array.from({ length: 50 }, (_, i) => ({
+        possibleValues: Array.from({ length: 20 }, (_, i) => ({
           SHUFFLE_SEED: 100 + i,
         })),
       },
@@ -267,6 +271,7 @@ export class HyperSingleIntraNodeSolver extends HyperParameterSupervisorSolver<
         colorMap: this.constructorParams.colorMap,
         viaDiameter: this.constructorParams.viaDiameter,
         traceWidth: this.constructorParams.traceWidth,
+        effort: this.effort,
       }) as any
     }
     return new CachedIntraNodeRouteSolver({

lib/solvers/MultiSectionPortPointOptimizer/MultiSectionPortPointOptimizer.ts

@@ -216,7 +216,7 @@ export class MultiSectionPortPointOptimizer extends BaseSolver {
   MAX_ATTEMPTS_PER_NODE = 100
 
   /** Maximum total number of section optimization attempts */
-  MAX_SECTION_ATTEMPTS = 50
+  MAX_SECTION_ATTEMPTS = 15
 
   /** Acceptable probability of failure threshold */
   ACCEPTABLE_PF = 0.05
@@ -275,8 +275,6 @@ export class MultiSectionPortPointOptimizer extends BaseSolver {
       params.JUMPER_PF_FN_ENABLED ?? this.JUMPER_PF_FN_ENABLED
     this.SHUFFLE_SEEDS_PER_SECTION = params.SHUFFLE_SEEDS_PER_SECTION
 
-    this.MAX_SECTION_ATTEMPTS *= this.effort
-
     this.nodeMap = new Map(
       params.inputNodes.map((n) => [n.capacityMeshNodeId, n]),
     )

scripts/profile-solvers.ts

@@ -16,6 +16,7 @@ type SolverRecord = {
 }
 
 type ProfileOptions = {
+  scenarioName?: string
   scenarioLimit?: number
 }
 
@@ -66,15 +67,27 @@ const parseArgs = (): ProfileOptions => {
 
   for (let i = 0; i < args.length; i++) {
     const arg = args[i]
-    if (arg === "--scenario-limit") {
-      options.scenarioLimit = Number.parseInt(args[i + 1], 10)
+    if (arg === "--scenario") {
+      const scenarioName = args[i + 1]
+      if (!scenarioName || scenarioName.startsWith("-")) {
+        throw new Error("--scenario requires a scenario name")
+      }
+      options.scenarioName = scenarioName
+      i += 1
+    } else if (arg === "--scenario-limit") {
+      const rawScenarioLimit = args[i + 1]
+      if (!rawScenarioLimit || rawScenarioLimit.startsWith("-")) {
+        throw new Error("--scenario-limit requires a value")
+      }
+      options.scenarioLimit = Number.parseInt(rawScenarioLimit, 10)
       i += 1
     } else if (arg === "-h" || arg === "--help") {
       console.log(
         [
-          "Usage: bun scripts/profile-solvers.ts [--scenario-limit N]",
+          "Usage: bun scripts/profile-solvers.ts [--scenario NAME] [--scenario-limit N]",
           "",
           "Options:",
+          "  --scenario NAME      Run only the named scenario",
           "  --scenario-limit N   Run only first N scenarios",
           "  -h, --help           Show this help",
         ].join("\n"),
@@ -95,12 +108,18 @@ const parseArgs = (): ProfileOptions => {
   return options
 }
 
-const loadScenarios = (scenarioLimit?: number) => {
+const loadScenarios = (scenarioName?: string, scenarioLimit?: number) => {
   const allScenarios = Object.entries(dataset)
     .filter(([, value]) => Boolean(value) && typeof value === "object")
     .sort(([a], [b]) => a.localeCompare(b)) as Array<[string, SimpleRouteJson]>
 
-  return scenarioLimit ? allScenarios.slice(0, scenarioLimit) : allScenarios
+  const filteredScenarios = scenarioName
+    ? allScenarios.filter(([name]) => name === scenarioName)
+    : allScenarios
+
+  return scenarioLimit
+    ? filteredScenarios.slice(0, scenarioLimit)
+    : filteredScenarios
 }
 
 const getPercentile = (values: number[], p: number): number | null => {
@@ -143,9 +162,12 @@ const formatTable = (headers: string[], body: string[][]): string => {
 // --- Main ---
 const main = () => {
   const opts = parseArgs()
-  const scenarios = loadScenarios(opts.scenarioLimit)
+  const scenarios = loadScenarios(opts.scenarioName, opts.scenarioLimit)
 
   if (scenarios.length === 0) {
+    if (opts.scenarioName) {
+      throw new Error(`Scenario not found: ${opts.scenarioName}`)
+    }
     throw new Error("No scenarios found")
   }