feat(eval): implement LeRobot evaluation plugin with CI gating

- Add PolicyAdapter protocol and LeRobotPolicyAdapter for checkpoint loading
- Extract shared LeRobot env utilities into lerobot_env.py
- Add evaluate_lerobot_policy() entry point with action shape validation
- Update examples/lerobot_eval_harness.py with --checkpoint-path and --repo-id CLI
- Add tests for policy adapter, env creation, eval plugin, and harness example
- Update README with LeRobot CI evaluation section
- Add torch/lerobot to mypy ignore_missing_imports

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: MiaoDX

README.md

@@ -121,6 +121,29 @@ Uses LeRobot's official `make_env("lerobot/unitree-g1-mujoco")` factory for stan
 
 </details>
 
+<details>
+<summary><b>LeRobot Evaluation in CI</b></summary>
+
+```bash
+pip install roboharness[lerobot]
+
+# Evaluate a real LeRobot checkpoint with visual checkpoints + JSON report
+python examples/lerobot_eval_harness.py \
+  --checkpoint-path /path/to/lerobot/checkpoint \
+  --repo-id lerobot/unitree-g1-mujoco \
+  --n-episodes 5 \
+  --checkpoint-steps 10 50 100 \
+  --assert-threshold \
+  --min-success-rate 0.8
+```
+
+Produces:
+- `episode_000/step_0010/default_rgb.png` — checkpoint screenshots
+- `lerobot_eval_report.json` — structured per-episode stats
+- CI exit code 1 when thresholds are not met
+
+</details>
+
 <a id="sonic-planner"></a>
 <details>
 <summary><b>SONIC Planner</b></summary>

docs/roadmap-2026-q2.md

@@ -56,6 +56,8 @@ Directions are split into "do now" and "do later." "Do now" means the next avail
 
 **Exit criteria:** A LeRobot user can run one command to get visual regression testing in CI.
 
+**Status:** Complete. `examples/lerobot_eval_harness.py --checkpoint-path <path> --repo-id <repo>` loads real LeRobot policies, captures checkpoint screenshots, produces `lerobot_eval_report.json`, and supports `--assert-threshold` for CI pass/fail gates.
+
 **Related issues:** New issue needed. Extends #83 (native LeRobot integration).
 
 ### B. Constraint Evaluator

examples/lerobot_eval_harness.py

@@ -14,6 +14,12 @@
 Run (standalone — CartPole demo):
     python examples/lerobot_eval_harness.py --env CartPole-v1 --n-episodes 5
 
+Run (with real LeRobot checkpoint):
+    python examples/lerobot_eval_harness.py \
+        --checkpoint-path /path/to/lerobot/checkpoint \
+        --repo-id lerobot/unitree-g1-mujoco \
+        --n-episodes 5
+
 Run (with success threshold — CI gate):
     python examples/lerobot_eval_harness.py --env CartPole-v1 --n-episodes 10 \
         --min-success-rate 0.0 --assert-threshold
@@ -38,6 +44,7 @@
 from roboharness.evaluate.lerobot_plugin import (
     LeRobotEvalConfig,
     check_eval_threshold,
+    evaluate_lerobot_policy,
     evaluate_policy,
 )
 
@@ -83,27 +90,24 @@ def main() -> None:
         action="store_true",
         help="Exit non-zero if thresholds are not met (CI mode)",
     )
+    parser.add_argument(
+        "--checkpoint-path",
+        type=str,
+        default=None,
+        help="Path to a LeRobot policy checkpoint directory",
+    )
+    parser.add_argument(
+        "--repo-id",
+        type=str,
+        default=None,
+        help="HuggingFace repo ID for the LeRobot environment (inferred if omitted)",
+    )
     args = parser.parse_args()
 
     print("=" * 60)
     print("  Roboharness: LeRobot Evaluation Harness")
     print("=" * 60)
 
-    # 1. Create environment
-    print(f"\n[1/3] Creating environment: {args.env}")
-    try:
-        import gymnasium as gym
-
-        env = gym.make(args.env, render_mode="rgb_array")
-    except ImportError:
-        print("ERROR: gymnasium is required. Install with: pip install roboharness[demo]")
-        sys.exit(1)
-
-    print(f"      Obs space: {env.observation_space}")
-    print(f"      Act space: {env.action_space}")
-
-    # 2. Run evaluation
-    print(f"[2/3] Evaluating ({args.n_episodes} episodes, max {args.max_steps} steps each) ...")
     output_dir = Path(args.output_dir) / "lerobot_eval"
 
     config = LeRobotEvalConfig(
@@ -113,14 +117,43 @@ def main() -> None:
         output_dir=str(output_dir),
     )
 
-    # Use random policy as fallback
-    action_space = env.action_space
+    # 1. Create environment / load policy
+    if args.checkpoint_path:
+        print(f"\n[1/3] Loading LeRobot policy from: {args.checkpoint_path}")
+        if not Path(args.checkpoint_path).exists():
+            print(f"ERROR: Checkpoint path does not exist: {args.checkpoint_path}")
+            sys.exit(1)
+
+        # 2. Run evaluation with real LeRobot policy
+        print(f"[2/3] Evaluating ({args.n_episodes} episodes, max {args.max_steps} steps each) ...")
+        report = evaluate_lerobot_policy(
+            checkpoint_path=args.checkpoint_path,
+            repo_id=args.repo_id,
+            config=config,
+        )
+    else:
+        print(f"\n[1/3] Creating environment: {args.env}")
+        try:
+            import gymnasium as gym
+
+            env = gym.make(args.env, render_mode="rgb_array")
+        except ImportError:
+            print("ERROR: gymnasium is required. Install with: pip install roboharness[demo]")
+            sys.exit(1)
+
+        print(f"      Obs space: {env.observation_space}")
+        print(f"      Act space: {env.action_space}")
 
-    def policy_fn(obs: np.ndarray) -> np.ndarray:
-        return _random_policy(obs, action_space)
+        # Use random policy as fallback
+        action_space = env.action_space
 
-    report = evaluate_policy(env, policy_fn, config)
-    env.close()
+        def policy_fn(obs: np.ndarray) -> np.ndarray:
+            return _random_policy(obs, action_space)
+
+        # 2. Run evaluation
+        print(f"[2/3] Evaluating ({args.n_episodes} episodes, max {args.max_steps} steps each) ...")
+        report = evaluate_policy(env, policy_fn, config)
+        env.close()
 
     # 3. Report results
     print("[3/3] Results:")
@@ -144,6 +177,14 @@ def policy_fn(obs: np.ndarray) -> np.ndarray:
             f"  length={ep.episode_length:4d}"
         )
 
+    if not args.checkpoint_path:
+        print(
+            "\n  Tip: pass --checkpoint-path to evaluate a real LeRobot policy:\n"
+            "    python examples/lerobot_eval_harness.py \\\n"
+            "      --checkpoint-path /path/to/lerobot/checkpoint \\\n"
+            "      --repo-id lerobot/unitree-g1-mujoco"
+        )
+
     # 4. CI gate
     if args.assert_threshold:
         passed = check_eval_threshold(

examples/lerobot_g1_native.py

@@ -38,11 +38,11 @@
 from pathlib import Path
 from typing import Any
 
-import gymnasium as gym  # noqa: TC002 — used at runtime inside create_native_env
 import numpy as np
 
 from roboharness.core.protocol import TaskPhase, TaskProtocol
-from roboharness.wrappers import RobotHarnessWrapper, VectorEnvAdapter
+from roboharness.evaluate.lerobot_env import create_native_env
+from roboharness.wrappers import RobotHarnessWrapper
 
 # ---------------------------------------------------------------------------
 # Constants
@@ -65,201 +65,6 @@
 }
 
 
-# ---------------------------------------------------------------------------
-# Environment creation via make_env()
-# ---------------------------------------------------------------------------
-
-
-def _patch_config_for_headless(env_id: str) -> None:
-    """Patch the HuggingFace-cached config.yaml for headless (CI) rendering.
-
-    The lerobot/unitree-g1-mujoco env.py loads config.yaml at import time.
-    The default config has ``ENABLE_ONSCREEN: true`` which requires GLFW/display.
-    For headless environments (MUJOCO_GL=osmesa, no DISPLAY), we disable onscreen
-    rendering so the simulator uses offscreen-only mode.
-    """
-    import os
-
-    has_display = bool(os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"))
-    if has_display:
-        return  # Display available, no patching needed
-
-    try:
-        from huggingface_hub import snapshot_download
-
-        repo_dir = Path(snapshot_download(env_id, repo_type="model"))
-    except Exception:
-        return  # Can't patch, let make_env handle errors
-
-    config_path = repo_dir / "config.yaml"
-    if not config_path.exists():
-        return
-
-    import yaml
-
-    config = yaml.safe_load(config_path.read_text())
-    if config.get("ENABLE_ONSCREEN") is True:
-        config["ENABLE_ONSCREEN"] = False
-        config["ENABLE_OFFSCREEN"] = True
-        config_path.write_text(yaml.dump(config, default_flow_style=False))
-        print("      Patched config.yaml: ENABLE_ONSCREEN=false (headless mode)")
-
-
-def create_native_env(
-    env_id: str = LEROBOT_ENV_ID,
-    *,
-    n_envs: int = 1,
-) -> gym.Env:
-    """Create a LeRobot environment, preferring the official ``make_env()`` factory.
-
-    Strategy (in order):
-      1. Try LeRobot's ``make_env()`` — wraps the hub env in ``SyncVectorEnv``.
-         We unwrap the batch dimension via ``VectorEnvAdapter`` so downstream
-         wrappers see a standard single-env interface.
-      2. Fall back to importing the hub's ``env.py`` directly (works without
-         the full LeRobot install; avoids the ``SyncVectorEnv`` obs-space
-         mismatch that the upstream env has).
-    """
-    try:
-        from huggingface_hub import snapshot_download  # noqa: F401 — used below
-    except ImportError:
-        print(
-            "ERROR: huggingface_hub is required for native integration.\n"
-            "Install with: pip install roboharness[demo,unitree] lerobot"
-        )
-        sys.exit(1)
-
-    # Patch config for headless CI environments before importing env module
-    _patch_config_for_headless(env_id)
-
-    env = _try_lerobot_make_env(env_id, n_envs=n_envs)
-    if env is None:
-        env = _fallback_hub_make_env(env_id, n_envs=n_envs)
-
-    # Add MuJoCo rendering capability — the hub env has a MuJoCo model but
-    # doesn't expose render_camera(), so the wrapper can't capture screenshots.
-    _add_mujoco_rendering(env)
-
-    print(f"      Env type: {type(env).__name__}")
-    print(f"      Obs space (declared): {env.observation_space}")
-    print(f"      Act space: {env.action_space}")
-
-    return env
-
-
-def _try_lerobot_make_env(env_id: str, *, n_envs: int = 1) -> gym.Env | None:
-    """Try creating the env via LeRobot's official ``make_env()`` factory.
-
-    Returns a ``VectorEnvAdapter``-wrapped env on success, or ``None`` if
-    LeRobot is not installed or ``make_env()`` fails.
-    """
-    try:
-        from lerobot.common.envs.factory import make_env  # type: ignore[import-untyped]
-    except ImportError:
-        print("      LeRobot not installed — falling back to hub env import")
-        return None
-
-    try:
-        vec_env = make_env(env_id, n_envs=n_envs)
-    except Exception as exc:
-        print(f"      LeRobot make_env() failed ({exc}) — falling back to hub env import")
-        return None
-
-    # make_env() wraps in SyncVectorEnv; adapt to standard gym.Env.
-    env = VectorEnvAdapter(vec_env)
-    print("      Created via LeRobot make_env() + VectorEnvAdapter")
-    return env
-
-
-def _fallback_hub_make_env(env_id: str, *, n_envs: int = 1) -> gym.Env:
-    """Import the hub's ``env.py`` directly (no LeRobot dependency)."""
-    from huggingface_hub import snapshot_download
-
-    repo_dir = Path(snapshot_download(env_id, repo_type="model"))
-    sys.path.insert(0, str(repo_dir))
-    try:
-        from env import make_env as hub_make_env  # type: ignore[import-not-found]
-    except ImportError as e:
-        print(f"ERROR: Failed to import hub env module: {e}")
-        sys.exit(1)
-
-    env = hub_make_env(n_envs=n_envs)
-
-    # Obs-space shape mismatch (upstream declares (97,) but returns (100,) due to
-    # floating_base_acc being 6-D not 3-D) is handled automatically by
-    # RobotHarnessWrapper(auto_fix_obs_space=True). See issue #110.
-
-    print("      Created via direct hub env import (fallback)")
-    return env
-
-
-def _add_mujoco_rendering(
-    env: gym.Env,
-    width: int = 640,
-    height: int = 480,
-) -> None:
-    """Patch the env to support render_camera() using MuJoCo's renderer.
-
-    The hub env has a MuJoCo model/data underneath but doesn't expose camera
-    rendering. We find the model/data, create a mujoco.Renderer, and add
-    render_camera() + cameras property so RobotHarnessWrapper can capture
-    multi-view screenshots.
-    """
-    import mujoco
-
-    unwrapped = getattr(env, "unwrapped", env)
-
-    # Find the MuJoCo model and data on the env (attribute names vary by env)
-    # Search the unwrapped env and one level deeper (e.g. env.sim_env.mj_model
-    # for the lerobot/unitree-g1-mujoco hub env).
-    model = None
-    data = None
-    search_targets = [unwrapped]
-    for nested in ("sim_env", "simulator", "sim"):
-        obj = getattr(unwrapped, nested, None)
-        if obj is not None:
-            search_targets.append(obj)
-
-    for target in search_targets:
-        for attr in ("model", "_model", "mj_model"):
-            candidate = getattr(target, attr, None)
-            if candidate is not None and hasattr(candidate, "ncam"):
-                model = candidate
-                break
-        if model is not None:
-            break
-
-    for target in search_targets:
-        for attr in ("data", "_data", "mj_data"):
-            candidate = getattr(target, attr, None)
-            if candidate is not None and hasattr(candidate, "qpos"):
-                data = candidate
-                break
-        if data is not None:
-            break
-
-    if model is None or data is None:
-        print("      Warning: could not find MuJoCo model/data — no screenshots")
-        return
-
-    renderer = mujoco.Renderer(model, height, width)
-    camera_names = [model.camera(i).name for i in range(model.ncam)]
-
-    def render_camera(camera_name: str) -> np.ndarray:
-        if camera_name not in camera_names:
-            raise ValueError(f"Unknown camera: {camera_name}. Available: {camera_names}")
-        renderer.update_scene(data, camera=camera_name)
-        return renderer.render()
-
-    # Patch the unwrapped env so the wrapper detects render_camera capability
-    unwrapped.render_camera = render_camera  # type: ignore[attr-defined]
-    unwrapped.cameras = camera_names  # type: ignore[attr-defined]
-    # Store model/data for controller state access
-    unwrapped.mj_model = model  # type: ignore[attr-defined]
-    unwrapped.mj_data = data  # type: ignore[attr-defined]
-    print(f"      Added MuJoCo rendering: {len(camera_names)} cameras {camera_names}")
-
-
 # ---------------------------------------------------------------------------
 # Validation
 # ---------------------------------------------------------------------------

pyproject.toml

@@ -52,6 +52,7 @@ lerobot = [
     "gymnasium>=0.29",
     "mujoco>=3.0",
     "Pillow",
+    "torch",
 ]
 dev = [
     "pytest>=7.0",
@@ -137,5 +138,7 @@ module = [
     "onnxruntime.*",
     "huggingface_hub.*",
     "mcp.*",
+    "torch.*",
+    "lerobot.*",
 ]
 ignore_missing_imports = true