Merge pull request #4 from DragonRuby/bytecode-bench

Run VM benchmarks via bytecode

Author: AlexDenisov

.pre-commit-config.yaml

@@ -20,3 +20,7 @@ repos:
     rev: 'v19.1.4'
     hooks:
       - id: clang-format
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 24.10.0
+    hooks:
+      - id: black

tests/benchmarks/CMakeLists.txt

@@ -1,12 +1,34 @@
+function(add_bench_executable ruby)
+  set(full_ruby_path ${CMAKE_CURRENT_LIST_DIR}/${ruby})
+  set(host_c ${CMAKE_CURRENT_BINARY_DIR}/bytecode_${ruby}.c)
+  set(target_name bench_${ruby}.exe)
+  add_custom_command(
+    OUTPUT ${host_c}
+    COMMAND $<TARGET_FILE:mrbc_binary> -Blightstorm_bench -o ${host_c}
+            ${full_ruby_path}
+    DEPENDS ${full_ruby_path})
+  add_executable(${target_name} benchmarks_bytecode_main.c ${host_c})
+  target_include_directories(
+    ${target_name}
+    PRIVATE ${CMAKE_SOURCE_DIR}/third_party/mruby/include
+            ${CMAKE_SOURCE_DIR}/third_party/mruby/build/host/include)
+  target_compile_options(${target_name} PRIVATE -g)
+  target_link_libraries(${target_name} PRIVATE mruby_static)
+  add_dependencies(${target_name} mruby_static)
+endfunction()
+
 file(
   GLOB files
   RELATIVE ${CMAKE_CURRENT_LIST_DIR}
   "${CMAKE_CURRENT_LIST_DIR}/*.rb")
+
 foreach(file ${files})
   add_lightstorm_executable(${file})
+  add_bench_executable(${file})
   set(bench_targets ${bench_targets} $<TARGET_FILE:${file}.exe>
-                    ${CMAKE_CURRENT_LIST_DIR}/${file})
+                    $<TARGET_FILE:bench_${file}.exe>)
 endforeach()
+
 add_custom_target(
   run-benchmarks COMMAND python3 ${CMAKE_CURRENT_LIST_DIR}/bench.py
-                         $<TARGET_FILE:mruby_binary> ${bench_targets})
+                         ${bench_targets})

tests/benchmarks/bench.py

@@ -3,19 +3,27 @@
 
 mruby_binary = sys.argv[1]
 
-for idx in range(2, len(sys.argv), 2):
+for idx in range(1, len(sys.argv), 2):
     bin = sys.argv[idx]
-    rb_full = sys.argv[idx + 1]
-    rb_name = rb_full.split('/')[-1]
+    bin_bytecode = sys.argv[idx + 1]
+    rb_name = bin_bytecode.split("/")[-1].split(".")[0]
     print("Benchmarking " + rb_name)
-    r = subprocess.run([
-        'hyperfine',
-        '--warmup', '1',
-        # Calling compiled binary
-        '-n', 'lightstorm ' + rb_name, bin,
-        # Calling mruby against the original file
-        '-n', 'mruby ' + rb_name, mruby_binary + ' ' + rb_full
-    ], capture_output=True)
-    summary = r.stdout.decode('utf8').split("Summary")[-1].splitlines()
+    r = subprocess.run(
+        [
+            "hyperfine",
+            "--warmup",
+            "1",
+            # Calling compiled binary
+            "-n",
+            "lightstorm " + rb_name,
+            bin,
+            # Calling compiled binary (mruby bytecode)
+            "-n",
+            "mruby " + rb_name,
+            bin_bytecode,
+        ],
+        capture_output=True,
+    )
+    summary = r.stdout.decode("utf8").split("Summary")[-1].splitlines()
     summary = [s.strip() for s in summary]
     print(" ".join(summary).strip())

tests/benchmarks/benchmarks_bytecode_main.c

@@ -0,0 +1,12 @@
+#include <mruby.h>
+#include <mruby/irep.h>
+#include <mruby/proc.h>
+
+extern const uint8_t lightstorm_bench[];
+
+int main() {
+  mrb_state *mrb = mrb_open();
+  mrb_load_irep(mrb, lightstorm_bench);
+  mrb_close(mrb);
+  return 0;
+}