ping feature

Author: pietz

CHANGELOG.md

@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.0] - 2025-01-27
+
+### Added
+- New `ping` command to check model availability without running benchmarks
+- Tests for the ping command functionality
+
+### Fixed
+- CLI callback logic to properly handle subcommands
+
 ## [0.3.0] - 2025-01-26
 
 ### Changed

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tacho"
-version = "0.3.0"
+version = "0.4.0"
 description = "CLI tool for measuring and comparing LLM inference speeds"
 readme = "README.md"
 authors = [

tacho/cli.py

@@ -21,6 +21,11 @@ def cli_main(
     lim: int = typer.Option(500, "--lim", "-l"),
 ):
     """Default command when models are provided directly"""
+    # Check if the first argument is a known command
+    if models and models[0] in ["bench", "ping"]:
+        # This is a subcommand, let it handle the arguments
+        return
+    
     if ctx.invoked_subcommand is None and models:
         bench(models, runs, prompt, lim)
 
@@ -81,6 +86,33 @@ def bench(
     console.print(table)
 
 
+@app.command()
+def ping(
+    models: list[str] = typer.Argument(
+        ...,
+        help="List of models to check availability (e.g., gpt-4o gemini-2.5-flash)",
+    ),
+):
+    """Check which LLM models are accessible without running benchmarks"""
+    res = asyncio.run(ping_models(models))
+    
+    # Count successful models
+    successful = sum(res)
+    
+    # Print summary
+    console.print()
+    if successful == len(models):
+        console.print(f"[bold green]All {len(models)} models are accessible![/bold green]")
+    elif successful > 0:
+        console.print(f"[bold yellow]{successful}/{len(models)} models are accessible[/bold yellow]")
+    else:
+        console.print("[bold red]No models are accessible[/bold red]")
+    
+    # Exit with appropriate code
+    if successful == 0:
+        raise typer.Exit(1)
+
+
 def main():
     """Main entry point that suppresses warnings on exit."""
     os.environ["PYTHONWARNINGS"] = "ignore"

tests/test_integration.py

@@ -63,3 +63,29 @@ def test_error_handling_no_api_key(self):
             if original_key:
                 os.environ["OPENAI_API_KEY"] = original_key
 
+    def test_ping_single_model(self):
+        if not os.getenv("OPENAI_API_KEY"):
+            pytest.skip("OPENAI_API_KEY not set")
+            
+        result = runner.invoke(app, ["ping", "gpt-4.1-mini"])
+        assert result.exit_code == 0
+        assert "✓ gpt-4.1-mini" in result.stdout
+        assert "models are accessible" in result.stdout
+        
+    def test_ping_multiple_models(self):
+        if not os.getenv("OPENAI_API_KEY"):
+            pytest.skip("OPENAI_API_KEY not set")
+            
+        result = runner.invoke(app, ["ping", "gpt-4.1-mini", "invalid-model-xyz"])
+        assert result.exit_code == 0
+        assert "✓ gpt-4.1-mini" in result.stdout
+        assert "✗ invalid-model-xyz" in result.stdout
+        assert "1/2 models are accessible" in result.stdout
+        
+    def test_ping_all_invalid_models(self):
+        result = runner.invoke(app, ["ping", "invalid-model-1", "invalid-model-2"])
+        assert result.exit_code == 1
+        assert "✗ invalid-model-1" in result.stdout
+        assert "✗ invalid-model-2" in result.stdout
+        assert "No models are accessible" in result.stdout
+