Tweak milabench realtime event tracking

benchmarks/_templates/stdout/README.md

@@ -2,3 +2,30 @@
 # Template
 
 Rewrite this README to explain what the benchmark is!
+
+
+
+
+
+# CUDA
+
+uv pip install vllm --torch-backend=auto
+
+
+# AMD
+
+# Install PyTorch
+pip uninstall torch -y
+pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/rocm6.4
+
+
+
+# XPU
+<!-- git clone https://github.com/vllm-project/vllm.git
+cd vllm
+pip install --upgrade pips
+pip install -v -r requirements/xpu.txt
+
+VLLM_TARGET_DEVICE=xpu python setup.py install -->
+
+

benchmarks/dinov2/prepare.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 import os
 from benchmate.datagen import generate_fakeimagenet, device_count
-from tqdm import tqdm
+from benchmate.progress import tqdm
 
 
 def loop_on(iterable:list):

benchmarks/geo_gnn/pcqm4m_subset.py

@@ -9,7 +9,7 @@
 from torch_geometric.data.data import BaseData
 from torch_geometric.datasets import PCQM4Mv2
 from torch_geometric.utils import from_smiles as _from_smiles
-from tqdm import tqdm
+from benchmate.progress import tqdm
 
 
 class PCQM4Mv2Subset(PCQM4Mv2):

benchmarks/inference/Makefile

@@ -0,0 +1,31 @@
+# Use global base if possible
+ifndef MILABENCH_BASE
+	MILABENCH_BASE="base"
+endif
+
+export MILABENCH_BASE
+
+BENCH_NAME=template
+MILABENCH_CONFIG=dev.yaml
+MILABENCH_ARGS=--config $(MILABENCH_CONFIG) --base $(MILABENCH_BASE)
+
+all:
+	install prepare single gpus nodes
+
+install:
+	milabench install $(MILABENCH_ARGS) --force
+
+prepare:
+	milabench prepare $(MILABENCH_ARGS)
+
+tests: install prepare
+	milabench run $(MILABENCH_ARGS)
+
+single:
+	milabench run $(MILABENCH_ARGS) --select $(BENCH_NAME)-single
+
+gpus:
+	milabench run $(MILABENCH_ARGS) --select $(BENCH_NAME)-gpus
+
+nodes:
+	milabench run $(MILABENCH_ARGS) --select $(BENCH_NAME)-nodes

benchmarks/inference/README.md

@@ -0,0 +1,4 @@
+
+# Template
+
+Rewrite this README to explain what the benchmark is!

benchmarks/inference/benchfile.py

@@ -0,0 +1,78 @@
+from milabench.pack import Package
+import milabench.commands as cmd 
+from milabench.utils import assemble_options
+
+
+class VLLM(Package):
+    # Requirements file installed by install(). It can be empty or absent.
+    base_requirements = "requirements.in"
+
+    # The preparation script called by prepare(). It must be executable,
+    # but it can be any type of script. It can be empty or absent.
+    prepare_script = "prepare.py"
+
+    # The main script called by run(). It must be a Python file. It has to
+    # be present.
+    main_script = f"main.py"
+
+    # You can remove the functions below if you don't need to modify them.
+
+    def make_env(self):
+        # Return a dict of environment variables for prepare_script and
+        # main_script.
+        return super().make_env()
+
+    async def install(self):
+        await super().install()
+
+    async def prepare(self):
+        await super().prepare()  # super() call executes prepare_script
+
+    @property
+    def client_argv(self):
+        return assemble_options(self.config.get("client", {}).get("argv", []))
+
+    @property
+    def server_argv(self):
+        return assemble_options(self.config.get("server", {}).get("argv", []))
+
+    @property
+    def argv(self):
+        return self.server_argv + ['--'] + self.client_argv
+
+    def build_run_plan(self):
+        main = self.dirs.code / self.main_script
+        pack = cmd.PackCommand(self, *self.argv, lazy=True)
+        return cmd.VoirCommand(pack, cwd=main.parent).use_stdout()
+        # return super().build_run_plan().use_stdout()
+
+
+        # we can send early stop events when we want to stop one
+        # but what about the other ?
+        # What will end the server in particular ?
+        # The client might be able to send a stop server
+        # but that is unlikely
+
+        # client_pack = cmd.ClientServer.new_client_pack(self)
+
+        # server_pack = cmd.ClientServer.new_server_pack(self)
+
+        # client_cmd = cmd.PackCommand(
+        #     client_pack, self.client_main, *self.client_argv
+        # )
+
+        # server_cmd = cmd.PackCommand(
+        #     server_pack, self.server_main, *self.server_argv
+        # )
+
+        # return cmd.ClientServer(
+        #     self,
+        #     client_cmd,
+        #     server_cmd,
+        # )
+
+#
+# 
+#
+
+__pack__ = VLLM

benchmarks/inference/dev.yaml

@@ -0,0 +1,34 @@
+_inference:
+  inherits: _defaults
+  definition: .
+  install-variant: unpinned
+  install_group: torch
+
+
+whisper-transcribe-single:
+  inherits: _inference
+  num_machines: 1
+  tags:
+    - multigpu
+  plan:
+    method: per_gpu
+
+  argv:
+    --model: 
+    --dataset:
+    --batch-size:
+
+
+txt-to-image-gpus:
+  inherits: _inference
+  num_machines: 1
+  tags:
+    - multigpu
+  plan:
+    method: njobs
+    n: 1
+
+  argv:
+    --model: 
+    --dataset:
+    --batch-size:

benchmarks/inference/main.py

@@ -0,0 +1,53 @@
+from dataclasses import dataclass
+
+import torch
+import torchcompat.core as accelerator
+
+
+def prepare_voir():
+    from benchmate.observer import BenchObserver
+    from benchmate.monitor import bench_monitor
+
+    observer = BenchObserver(
+        accelerator.Event, 
+        earlystop=65,
+        batch_size_fn=lambda x: len(x[0]),
+        raise_stop_program=False,
+        stdout=True,
+    )
+
+    return observer, bench_monitor
+
+
+@dataclass
+class Arguments:
+    pass
+
+def load_model():
+    pass
+
+
+def load_dataset():
+    pass
+
+
+def main(argv):
+    global _log 
+
+    observer, monitor = prepare_voir()
+
+    with monitor():
+        with torch.no_grad():
+            model = load_model()
+
+            dataset = load_dataset()
+
+            for batch in dataset:
+
+                output = model(batch)
+
+
+if __name__ == "__main__":
+    import sys
+
+    main(sys.argv)

benchmarks/inference/prepare.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+
+import os
+
+if __name__ == "__main__":
+    # If you need the whole configuration:
+    # config = json.loads(os.environ["MILABENCH_CONFIG"])
+
+    data_directory = os.environ["MILABENCH_DIR_DATA"]
+
+    # Download (or generate) the needed dataset(s). You are responsible
+    # to check if it has already been properly downloaded or not, and to
+    # do nothing if it has been.
+    print("Hello I am doing some data stuff!")
+
+    # If there is nothing to download or generate, just delete this file.

benchmarks/inference/requirements.in

@@ -0,0 +1,3 @@
+voir>=0.2.19,<0.3
+torch
+vllm

benchmarks/llama/voirfile.py

@@ -1,9 +1,12 @@
 from voir.phase import StopProgram
 from voir import configurable
+from voir.overseer import Overseer
 
 
 @configurable
-def instrument_main(ov):
+def instrument_main(ov: Overseer):
+    yield ov.phases.load_script
+
     try:
         yield ov.phases.run_script
     except StopProgram:

benchmarks/llm/prepare.py

@@ -107,6 +107,7 @@ def load_dataset(recipe, cfg):
         setattr(recipe, "dp_size", 1)
         setattr(recipe, "dp_rank", 0)
 
+        # This is slow AF though
         recipe._tokenizer = config.instantiate(cfg.tokenizer)
         recipe._loss_fn = config.instantiate(cfg.loss)
 

benchmarks/recursiongfn/prepare.py

@@ -20,6 +20,12 @@ def parser():
         default=os.getenv("MILABENCH_DIR_DATA", None),
         help="Dataset path",
     )
+    parser.add_argument(
+        "--cache",
+        type=str,
+        default=os.getenv("XDG_CACHE_HOME", None),
+        help="Dataset path",
+    )
     return parser
 
 
@@ -35,11 +41,9 @@ def parser():
 
     args, _ = parser().parse_known_args()
 
-    xdg_cache = os.getenv("XDG_CACHE_HOME", args.data)
-
-    print("+ Loading proxy model weights to MILABENCH_DIR_DATA={}".format(xdg_cache))
+    print("+ Loading proxy model weights to MILABENCH_DIR_DATA={}".format(args.data))
     _ = load_original_model(
         cache=True,
-        location=Path(os.path.join(xdg_cache, "bengio2021flow_proxy.pkl.gz")),
-        )
+        location=Path(os.path.join(args.data, "bengio2021flow_proxy.pkl.gz")),
+    )
 

benchmarks/timm/benchfile.py

@@ -18,7 +18,7 @@ def argv(self):
         return [
             *super().argv,
             "--output",
-            self.dirs.extra / self.logdir.name / self.tag,
+            self.dirs.data,
             "--checkpoint-hist",
             1,
         ]

benchmarks/vjepa/prepare.py

@@ -4,7 +4,10 @@
 import cv2
 import numpy as np
 
-def generate_random_video(output_file, width=640, height=480, num_frames=300, fps=30):
+from benchmate.progress import tqdm
+
+
+def generate_random_video(offset, output_file, width=640, height=480, num_frames=300, fps=30):
     """
     Generates a .mp4 video file with random content.
 
@@ -17,6 +20,9 @@ def generate_random_video(output_file, width=640, height=480, num_frames=300, fp
     fourcc = cv2.VideoWriter_fourcc(*'mp4v')  # Use MP4 encoding
     video_writer = cv2.VideoWriter(output_file, fourcc, fps, (width, height))
 
+    seed = int(0 + offset)
+    np.random.seed(seed)
+
     for _ in range(num_frames):
         frame = np.random.randint(0, 256, (height, width, 3), dtype=np.uint8)
         video_writer.write(frame)
@@ -28,7 +34,6 @@ def generate_random_video(output_file, width=640, height=480, num_frames=300, fp
     import sys
     import csv
     import os
-    import tqdm
     import multiprocessing
 
     sys.path.append(os.path.dirname(__file__) + "/jepa/")
@@ -50,18 +55,18 @@ def generate_random_video(output_file, width=640, height=480, num_frames=300, fp
     def gen_video(i):
         output_file = os.path.join(dest, f"{i + 1}.mp4")
         if not os.path.exists(output_file):
-            generate_random_video(output_file=output_file, width=640, height=480, num_frames=num_frames, fps=30)
+            generate_random_video(offset=i, output_file=output_file, width=640, height=480, num_frames=num_frames, fps=30)
 
     n_worker = min(multiprocessing.cpu_count(), 16)
 
     with multiprocessing.Pool(n_worker) as pool:
-        for _ in tqdm.tqdm(pool.imap_unordered(gen_video, range(num_videos)), total=num_videos):
+        for _ in tqdm(pool.imap_unordered(gen_video, range(num_videos)), total=num_videos):
             pass
 
     with open(csv_file, mode='w', newline='') as file:
         # CSV separated by space genius
         writer = csv.writer(file, delimiter=" ")
-        for file in tqdm.tqdm(os.listdir(dest)):
+        for file in tqdm(os.listdir(dest)):
             if file.endswith(".mp4"):
                 writer.writerow([os.path.join(dest, file), 0])