Enable nightly tests comparing classic facets with the new during-col…

README.md

@@ -201,7 +201,7 @@ in a separate phase, and a new implementation that computes facets during collec
 To compare performance for the two implementations run
 
 ```
-python src/python/localrunFacets.py -source facetsWikimediumAll
+python src/python/runFacets.py -source facetsWikimediumAll
 ```
 
 Note that only comparison of taxonomy based facets is supported at the moment. We need to add SSDV facets support

src/python/benchUtil.py

@@ -44,6 +44,8 @@
 else:
   print(f"perf executable is {PERF_EXE}; will collect aggregate CPU profiling data")
 
+TASKSET_EXE = which("taskset")
+
 PYTHON_MAJOR_VER = sys.version_info.major
 
 VMSTAT_PATH = shutil.which("vmstat")
@@ -1193,6 +1195,16 @@ def runSimpleSearchBench(self, iter, id, c, coldRun, seed, staticSeed, filter=No
       doSort = ""
 
     command = []
+
+    if c.cpus is not None:
+      if TASKSET_EXE is None:
+        raise RuntimeError("Number of CPUs is requested, but taskset command is not found")
+      # Limit to first N CPUs only
+      cpu_list = ",".join(str(i) for i in range(c.searchConcurrency))
+      command.append(f"{TASKSET_EXE}")
+      command.append("-c")
+      command.append(f"{cpu_list}")
+
     if PERF_EXE is not None:
       command += [PERF_EXE, "stat", "-dd"]
     command += c.javaCommand.split()
@@ -1612,6 +1624,9 @@ def simpleReport(self, baseLogFiles, cmpLogFiles, jira=False, html=False, baseDe
         # When we add a whole new task (e.g. VectorSearch), just skip the comparison for the first nightly run
         # since baseline will not have this task yet:
         continue
+      if currentCat not in cmpLatencyMetrics:
+        # TODO: This could happen during the first run after separating facet tasks from search tasks - do we want to revert this after first run?
+        continue
       currentBaseMetrics = baseLatencyMetrics[currentCat]
       currentCmpMetrics = cmpLatencyMetrics[currentCat]
       pctP50 = 100 * (currentCmpMetrics["p50"] - currentBaseMetrics["p50"]) / currentBaseMetrics["p50"]

src/python/competition.py

@@ -300,6 +300,7 @@ def __init__(
     topN=100,
     testContext="",
     pollute=True,
+    cpus=None,  # int, number of CPUs to use for the test
   ):
     self.name = name
     self.checkout = checkout
@@ -330,6 +331,7 @@ def __init__(
     self.vectorDimension = vectorDimension
     self.vectorScale = vectorScale
     self.javacCommand = javacCommand
+    self.cpus = cpus
 
     # nocommit
     if False and searchConcurrency != 0 and numConcurrentQueries > 1: