[ESI][Runtime][Cosim] Switch Verilator backend to call verilator_bin directly (#9822)

Replace the call to the `verilator` Perl wrapper with a direct call to
`verilator_bin`. The C++ build is now handled by generating a CMakeLists.txt
and building with CMake (Ninja generator) instead of relying on verilator's
`--exe --build` flags. If cmake+ninja don't exist, use 'make' as a fallback.

Key changes:
- Default binary changed from `verilator` to `verilator_bin`
- Added `_find_verilator_root()` to locate VERILATOR_ROOT
- Removed `--exe`, `--build`, `-CFLAGS`, `-LDFLAGS`, and driver.cpp from
  the verilator command (these are now handled in the CMake build)
- Added `_write_cmake()` to generate a CMakeLists.txt for building
- Overrode `compile()` to run verilator_bin then cmake+ninja
- Updated `run_command()` for the new executable location
- Added unit tests for the new functionality

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: teqdruid <1498080+teqdruid@users.noreply.github.com>
Co-authored-by: John Demme <jodemme@microsoft.com>
AI-assisted-by: Claude Opus 4.6 (local run)

Author: Copilot

lib/Dialect/ESI/runtime/python/esiaccel/cosim/verilator.py

@@ -3,14 +3,19 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 import os
+import shutil
 from pathlib import Path
 from typing import List, Optional, Callable, Dict
 
 from .simulator import CosimCollateralDir, Simulator, SourceFiles
 
 
 class Verilator(Simulator):
-  """Run and compile funcs for Verilator."""
+  """Run and compile funcs for Verilator.
+
+  Calls ``verilator_bin`` directly (bypassing the Perl wrapper) to generate
+  C++ from RTL, then builds the simulation executable with CMake + Ninja.
+  Falls back to ``make`` when cmake/ninja are not available."""
 
   DefaultDriver = CosimCollateralDir / "driver.cpp"
 
@@ -39,13 +44,64 @@ def __init__(
         make_default_logs=make_default_logs,
         macro_definitions=macro_definitions,
     )
-    self.verilator = "verilator"
+    self.verilator_bin = "verilator_bin"
     if "VERILATOR_PATH" in os.environ:
-      self.verilator = os.environ["VERILATOR_PATH"]
+      vpath = os.environ["VERILATOR_PATH"]
+      # Backwards compatibility: if the env var points to the Perl wrapper,
+      # redirect to verilator_bin.
+      basename = Path(vpath).stem
+      if basename == "verilator":
+        self.verilator_bin = str(Path(vpath).parent / "verilator_bin")
+      else:
+        self.verilator_bin = vpath
+
+  def _find_verilator_root(self) -> Path:
+    """Locate VERILATOR_ROOT for runtime includes and sources.
+
+    Checks the ``VERILATOR_ROOT`` environment variable first, then attempts
+    to derive the root from the location of ``verilator_bin``.  Supports both
+    source-tree layouts (``$ROOT/include/verilated.h``) and system package
+    layouts (``$PREFIX/share/verilator/include/verilated.h``)."""
+    if "VERILATOR_ROOT" in os.environ:
+      root = Path(os.environ["VERILATOR_ROOT"])
+      if root.is_dir():
+        return root
+    # verilator_bin is typically in $PREFIX/bin/
+    verilator_bin_path = shutil.which(self.verilator_bin)
+    if verilator_bin_path:
+      prefix = Path(verilator_bin_path).resolve().parent.parent
+      # Source-tree layout: $VERILATOR_ROOT/bin/verilator_bin
+      if (prefix / "include" / "verilated.h").exists():
+        return prefix
+      # System package layout: $PREFIX/share/verilator/include/verilated.h
+      pkg_root = prefix / "share" / "verilator"
+      if (pkg_root / "include" / "verilated.h").exists():
+        return pkg_root
+    raise RuntimeError(
+        "Cannot find VERILATOR_ROOT. Set the VERILATOR_ROOT environment "
+        "variable or ensure verilator_bin is in PATH.")
+
+  @property
+  def _use_cmake(self) -> bool:
+    """True when both cmake and ninja are available on PATH."""
+    return shutil.which("cmake") is not None and \
+        shutil.which("ninja") is not None
 
   def compile_commands(self) -> List[List[str]]:
+    """Return the commands for the full compile flow.
+
+    When cmake and ninja are available the returned list contains three
+    commands run sequentially:
+      1. ``verilator_bin`` – generates C++ from RTL.
+      2. ``cmake`` – configures the C++ build (Ninja generator).
+      3. ``ninja`` – builds the simulation executable.
+
+    Otherwise falls back to two commands:
+      1. ``verilator_bin --exe`` – generates C++ and a Makefile.
+      2. ``make`` – builds via the generated Makefile.
+    """
     cmd: List[str] = [
-        self.verilator,
+        self.verilator_bin,
         "--cc",
     ]
 
@@ -62,30 +118,115 @@ def compile_commands(self) -> List[List[str]]:
         "-Wno-TIMESCALEMOD",
         "-Wno-fatal",
         "-sv",
-        "--exe",
-        "--build",
         "-j",
         "0",
         "--output-split",
         "--autoflush",
         "--assert",
-        str(Verilator.DefaultDriver),
-    ]
-    cflags = [
-        "-DTOP_MODULE=" + self.sources.top,
     ]
     if self.debug:
       cmd += [
           "--trace-fst", "--trace-params", "--trace-structs",
           "--trace-underscore"
       ]
+
+    if self._use_cmake:
+      cmd += [str(p) for p in self.sources.rtl_sources]
+      build_dir = str(Path.cwd() / "obj_dir" / "cmake_build")
+      cmake_cmd = ["cmake", "-G", "Ninja", "-S", build_dir, "-B", build_dir]
+      ninja_cmd = ["ninja", "-C", build_dir]
+      return [cmd, cmake_cmd, ninja_cmd]
+
+    # -- make fallback --
+    # Let verilator generate a Makefile with --exe so it includes the
+    # driver, CFLAGS, and LDFLAGS directly.
+    cmd += ["--exe", str(Verilator.DefaultDriver)]
+    cflags = ["-DTOP_MODULE=" + self.sources.top]
+    if self.debug:
       cflags.append("-DTRACE")
-    if len(cflags) > 0:
-      cmd += ["-CFLAGS", " ".join(cflags)]
-    if len(self.sources.dpi_so) > 0:
+    cmd += ["-CFLAGS", " ".join(cflags)]
+    if self.sources.dpi_so:
       cmd += ["-LDFLAGS", " ".join(["-l" + so for so in self.sources.dpi_so])]
     cmd += [str(p) for p in self.sources.rtl_sources]
-    return [cmd]
+    top = self.sources.top
+    make_cmd = ["make", "-C", "obj_dir", "-f", f"V{top}.mk", "-j"]
+    return [cmd, make_cmd]
+
+  def _write_cmake(self, obj_dir: Path) -> Path:
+    """Write a CMakeLists.txt for building the verilated simulation.
+
+    Returns the path to the CMake build directory."""
+    verilator_root = self._find_verilator_root()
+    include_dir = verilator_root / "include"
+    exe_name = "V" + self.sources.top
+
+    runtime_sources = [
+        include_dir / "verilated.cpp",
+        include_dir / "verilated_threads.cpp",
+    ]
+    if self.sources.dpi_so:
+      runtime_sources.append(include_dir / "verilated_dpi.cpp")
+    if self.debug:
+      runtime_sources.append(include_dir / "verilated_fst_c.cpp")
+
+    rt_src = "\n  ".join(str(s) for s in runtime_sources)
+    driver = str(Verilator.DefaultDriver)
+    inc = str(include_dir)
+    vltstd = str(include_dir / "vltstd")
+
+    defs = [f"TOP_MODULE={self.sources.top}"]
+    if self.debug:
+      defs.append("TRACE")
+    defs_str = "\n  ".join(defs)
+
+    # Link DPI shared objects by full path.
+    dpi_link = ""
+    if self.sources.dpi_so:
+      dpi_paths = self.sources.dpi_so_paths()
+      dpi_link = "\n  ".join(str(p) for p in dpi_paths)
+
+    content = f"""\
+cmake_minimum_required(VERSION 3.20)
+project({exe_name} CXX)
+
+file(GLOB GENERATED_SOURCES "${{CMAKE_CURRENT_SOURCE_DIR}}/../*.cpp")
+
+add_executable({exe_name}
+  ${{GENERATED_SOURCES}}
+  {rt_src}
+  {driver}
+)
+
+target_include_directories({exe_name} PRIVATE
+  {inc}
+  {vltstd}
+  ${{CMAKE_CURRENT_SOURCE_DIR}}/..
+)
+
+target_compile_definitions({exe_name} PRIVATE
+  {defs_str}
+)
+
+find_package(Threads REQUIRED)
+target_link_libraries({exe_name} PRIVATE
+  Threads::Threads
+  {dpi_link}
+)
+"""
+    build_dir = obj_dir / "cmake_build"
+    build_dir.mkdir(parents=True, exist_ok=True)
+    (build_dir / "CMakeLists.txt").write_text(content)
+    return build_dir
+
+  def compile(self) -> int:
+    """Set VERILATOR_ROOT, write the CMakeLists.txt (if using cmake), then
+    delegate to the base class which runs all commands from
+    :meth:`compile_commands`."""
+    verilator_root = self._find_verilator_root()
+    os.environ["VERILATOR_ROOT"] = str(verilator_root)
+    if self._use_cmake:
+      self._write_cmake(Path.cwd() / "obj_dir")
+    return super().compile()
 
   @property
   def waveform_extension(self) -> str:
@@ -95,5 +236,9 @@ def waveform_extension(self) -> str:
   def run_command(self, gui: bool):
     if gui:
       raise RuntimeError("Verilator does not support GUI mode.")
-    exe = Path.cwd() / "obj_dir" / ("V" + self.sources.top)
+    exe_name = "V" + self.sources.top
+    if self._use_cmake:
+      exe = Path.cwd() / "obj_dir" / "cmake_build" / exe_name
+    else:
+      exe = Path.cwd() / "obj_dir" / exe_name
     return [str(exe)]

lib/Dialect/ESI/runtime/tests/integration/sw/test_loopback.py

@@ -116,6 +116,15 @@ def test_cosim_loopback(conn: AcceleratorConnection) -> None:
   run(conn)
 
 
+def test_cosim_loopback_make_fallback(monkeypatch) -> None:
+  """Run the loopback test using the ``make`` fallback (no cmake/ninja)."""
+  from esiaccel.cosim.verilator import Verilator
+  monkeypatch.setattr(Verilator, "_use_cmake", property(lambda self: False))
+  # Delegate to the already-decorated cosim test.  The monkeypatch is
+  # inherited by the forked child process.
+  test_cosim_loopback()
+
+
 if __name__ == "__main__":
   platform = sys.argv[1]
   connstr = sys.argv[2]