|
37 | 37 | - continue_on_error: true/false - whether to continue if this test fails (default: false) |
38 | 38 | - depends_on: Comma-separated list of test IDs that must pass before this test runs |
39 | 39 | - hidden: true/false - if true, hides the code block from the website (default: false) |
| 40 | + - setup: Shell commands to run before the test script (e.g. venv activation). |
| 41 | + For Python tests, wraps execution in a shell that runs the setup first: |
| 42 | + setup="source llm-env/bin/activate" → bash -c "source llm-env/bin/activate && python <script>" |
| 43 | + For shell tests, the setup commands are prepended to the script body. |
| 44 | +
|
| 45 | +Setup attribute: |
| 46 | + The `setup` attribute lets you specify shell commands (e.g. venv activation) |
| 47 | + that run before the test script. This is especially useful for Python code |
| 48 | + blocks which are otherwise executed directly with `python <script>`: |
| 49 | +
|
| 50 | + <!-- @test:id=verify-imports platform=all setup="source llm-env/bin/activate" --> |
| 51 | + ```python |
| 52 | + import torch |
| 53 | + print(f"PyTorch version: {torch.__version__}") |
| 54 | + ``` |
| 55 | + <!-- @test:end --> |
| 56 | +
|
| 57 | + The runner expands this to: `bash -c "source llm-env/bin/activate && python test_verify-imports.py"` |
| 58 | + On Windows, it uses PowerShell instead of bash. |
| 59 | +
|
| 60 | + For shell-based tests, the setup commands are prepended to the script body. |
40 | 61 |
|
41 | 62 | Inline #hide marker: |
42 | 63 | Lines ending with `#hide` inside a code block are executed by the test runner |
@@ -81,6 +102,7 @@ class TestBlock: |
81 | 102 | continue_on_error: bool = False |
82 | 103 | depends_on: list[str] = field(default_factory=list) |
83 | 104 | hidden: bool = False |
| 105 | + setup: Optional[str] = None |
84 | 106 | language: str = "bash" |
85 | 107 | code: str = "" |
86 | 108 | line_number: int = 0 |
@@ -185,6 +207,7 @@ def extract_tests(readme_path: Path, target_platform: str) -> list[TestBlock]: |
185 | 207 | continue_on_error=attrs.get("continue_on_error", False), |
186 | 208 | depends_on=attrs.get("depends_on", []), |
187 | 209 | hidden=attrs.get("hidden", False), |
| 210 | + setup=attrs.get("setup"), |
188 | 211 | language=language, |
189 | 212 | code=code, |
190 | 213 | line_number=line_number, |
@@ -256,6 +279,8 @@ def run_test( |
256 | 279 | print(f"Timeout: {test.timeout}s") |
257 | 280 | if test.depends_on: |
258 | 281 | print(f"Dependencies: {', '.join(test.depends_on)}") |
| 282 | + if test.setup: |
| 283 | + print(f"Setup: {test.setup}") |
259 | 284 | print(f"{'='*60}") |
260 | 285 |
|
261 | 286 | # Check dependencies |
@@ -317,33 +342,53 @@ def run_test( |
317 | 342 | # Determine shell and script extension based on language and platform |
318 | 343 | is_windows = sys.platform == "win32" |
319 | 344 |
|
| 345 | + # If setup is provided, prepend it to shell-based tests or wrap Python tests |
| 346 | + setup_prefix = test.setup if test.setup else None |
| 347 | + |
320 | 348 | if test.language in ["bash", "sh", "shell"]: |
321 | 349 | if is_windows: |
322 | | - # Use PowerShell on Windows for bash-like commands |
323 | 350 | shell_cmd = ["powershell", "-Command"] |
324 | 351 | script_content = effective_code |
325 | 352 | else: |
326 | 353 | shell_cmd = ["bash", "-c"] |
327 | 354 | script_content = effective_code |
| 355 | + # Prepend setup commands to the shell script body |
| 356 | + if setup_prefix: |
| 357 | + script_content = f"{setup_prefix}\n{script_content}" |
328 | 358 | elif test.language in ["cmd", "batch"]: |
329 | 359 | shell_cmd = ["cmd", "/c"] |
330 | 360 | script_content = effective_code |
| 361 | + if setup_prefix: |
| 362 | + script_content = f"{setup_prefix}\n{script_content}" |
331 | 363 | elif test.language in ["powershell", "pwsh", "ps1"]: |
332 | 364 | shell_cmd = ["powershell", "-Command"] |
333 | 365 | script_content = effective_code |
| 366 | + if setup_prefix: |
| 367 | + script_content = f"{setup_prefix}\n{script_content}" |
334 | 368 | elif test.language == "python": |
335 | 369 | # For Python code blocks, write to temp file and execute |
336 | 370 | script_file = results_dir / f"test_{test.id}.py" |
337 | 371 | script_file.write_text(effective_code, encoding="utf-8") |
338 | | - shell_cmd = ["python", str(script_file)] |
339 | | - script_content = None |
| 372 | + if setup_prefix: |
| 373 | + # Wrap in a shell so setup commands (e.g. venv activation) run first |
| 374 | + if is_windows: |
| 375 | + shell_cmd = ["powershell", "-Command"] |
| 376 | + script_content = f'{setup_prefix}; python "{script_file}"' |
| 377 | + else: |
| 378 | + shell_cmd = ["bash", "-c"] |
| 379 | + script_content = f'{setup_prefix} && python "{script_file}"' |
| 380 | + else: |
| 381 | + shell_cmd = ["python", str(script_file)] |
| 382 | + script_content = None |
340 | 383 | else: |
341 | 384 | # Default to shell execution |
342 | 385 | if is_windows: |
343 | 386 | shell_cmd = ["powershell", "-Command"] |
344 | 387 | else: |
345 | 388 | shell_cmd = ["bash", "-c"] |
346 | 389 | script_content = effective_code |
| 390 | + if setup_prefix: |
| 391 | + script_content = f"{setup_prefix}\n{script_content}" |
347 | 392 |
|
348 | 393 | # Build the command |
349 | 394 | if script_content is not None: |
|
0 commit comments