Merge pull request #1 from kaushikb11/add/tests

tests: add unit tests

Author: kaushikb11

.gitignore

@@ -169,16 +169,3 @@ test_vmf.db
 *.db
 vmf_metrics.db
 verifiers_monitor.db
-
-# Development/test files
-test.py
-test_*.py
-demo_*.py
-check_*.html
-*.log
-*_GUIDE.md
-*_DESIGN.md
-*_ROADMAP.md
-*_IMPACT.md
-*_ASPECTS_TO_TRACK.md
-TRAINING_*.md

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -10,7 +10,7 @@ repos:
       - id: check-merge-conflict
 
   - repo: https://github.com/psf/black
-    rev: 23.12.1
+    rev: 24.10.0
     hooks:
       - id: black
         language_version: python3
@@ -22,7 +22,7 @@ repos:
         args: ["--profile", "black"]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.5.1
+    rev: v1.18.2
     hooks:
       - id: mypy
         additional_dependencies: [

pyproject.toml

@@ -4,14 +4,14 @@ build-backend = "hatchling.build"
 
 [project]
 name = "verifiers-monitor"
-version = "0.1.0"
+version = "0.1.1"
 description = "Observability framework for Verifiers RL training and evaluation"
 authors = [
     {name = "Kaushik Bokka", email = "kaushikbokka@gmail.com"},
 ]
 license = {text = "Apache-2.0"}
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 keywords = ["machine-learning", "reinforcement-learning", "monitoring", "observability", "verifiers"]
 classifiers = [
     "Development Status :: 4 - Beta",
@@ -20,7 +20,6 @@ classifiers = [
     "License :: OSI Approved :: Apache Software License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
@@ -31,35 +30,29 @@ classifiers = [
 ]
 
 dependencies = [
-    "fastapi>=0.68.0",
-    "uvicorn>=0.15.0",
-    "websockets>=10.0",
-    "psutil>=5.8.0",
-    "aiohttp>=3.8.0",
-    "pyngrok>=7.0.0",
-    "sqlmodel>=0.0.14",
-    "rich>=10.0.0",
+    "fastapi>=0.118.0",
+    "uvicorn>=0.37.0",
+    "websockets>=15.0.0",
+    "psutil>=7.1.0",
+    "aiohttp>=3.12.0",
+    "pyngrok>=7.4.0",
+    "sqlmodel>=0.0.25",
+    "rich>=14.1.0",
     "verifiers",
 ]
 
 [project.optional-dependencies]
 dev = [
-    "pytest>=6.0",
-    "pytest-asyncio>=0.18.0",
-    "black>=21.0.0",
-    "isort>=5.0.0",
-    "mypy>=0.910",
-    "pre-commit>=2.15.0",
+    "pytest>=8.4.0",
+    "pytest-asyncio>=0.25.0",
+    "black>=25.9.0",
+    "isort>=6.1.0",
+    "mypy>=1.18.0",
+    "pre-commit>=4.0.0",
 ]
 
 analysis = [
-    "pandas>=1.3.0",
-]
-
-export = [
-    "wandb>=0.12.0",
-    "prometheus-client>=0.14.0",
-    "pandas>=1.3.0",
+    "pandas>=2.3.0",
 ]
 
 gpu = [

tests/test_collectors.py

@@ -0,0 +1,93 @@
+"""
+Test metrics collection functionality
+"""
+
+import time
+
+import pytest
+
+from verifiers_monitor.core.collectors import MetricsCollector
+
+
+def test_metrics_collector_creation(sqlite_storage):
+    """Test metrics collector creation"""
+    collector = MetricsCollector(sqlite_storage)
+
+    assert collector.storage == sqlite_storage
+    assert collector._evaluation_count == 0
+
+
+def test_training_step_collection(sqlite_storage):
+    """Test training step metrics collection"""
+    collector = MetricsCollector(sqlite_storage)
+
+    # Collect training metrics
+    metrics = {"step": 1, "loss": 2.5, "compute_time": 0.1, "timestamp": time.time()}
+
+    collector.collect_training_step(metrics)
+
+    # Verify storage
+    stored_data = sqlite_storage.get_recent("training_step")
+    assert len(stored_data) == 1
+    assert stored_data[0]["step"] == 1
+    assert stored_data[0]["loss"] == 2.5
+
+
+def test_evaluation_session(sqlite_storage):
+    """Test evaluation session management"""
+    collector = MetricsCollector(sqlite_storage)
+
+    # Start evaluation session
+    collector.start_evaluation_session("test_model", 10)
+
+    # Verify session started
+    session_data = sqlite_storage.get_recent("evaluation_session")
+    assert len(session_data) == 1
+    assert session_data[0]["model"] == "test_model"
+    assert session_data[0]["num_examples"] == 10
+
+
+def test_rollout_collection(sqlite_storage):
+    """Test rollout metrics collection"""
+    collector = MetricsCollector(sqlite_storage)
+
+    # Start session first
+    collector.start_evaluation_session("test_model", 3)
+
+    # Collect rollout data
+    rollout_data = {
+        "prompt": "test prompt",
+        "completion": "test completion",
+        "reward": 0.8,
+        "rollout_time": 0.05,
+    }
+
+    collector.collect_rollout(rollout_data)
+
+    # Verify storage
+    stored_data = sqlite_storage.get_recent("rollout")
+    assert len(stored_data) == 1
+    assert stored_data[0]["reward"] == 0.8
+    assert stored_data[0]["example_number"] == 1
+
+
+def test_advantages_collection(sqlite_storage):
+    """Test advantages metrics collection"""
+    collector = MetricsCollector(sqlite_storage)
+
+    # Mock advantages with statistics
+    advantages = {"mean": 0.1, "std": 0.2, "min": -0.1, "max": 0.3}
+
+    metrics = {
+        "step": 1,
+        "loss": 2.0,
+        "advantages": advantages,
+        "timestamp": time.time(),
+    }
+
+    collector.collect_training_step(metrics)
+
+    # Verify advantages are stored
+    stored_data = sqlite_storage.get_recent("training_step")
+    assert stored_data[0]["advantages_mean"] == 0.1
+    assert stored_data[0]["advantages_std"] == 0.2

tests/test_core.py

@@ -0,0 +1,50 @@
+"""
+Test core Verifiers Monitor functionality
+"""
+
+import time
+
+import pytest
+
+from verifiers_monitor import monitor
+
+
+def test_import():
+    """Test that Verifiers Monitor can be imported successfully"""
+    from verifiers_monitor import monitor
+
+    assert monitor is not None
+
+
+def test_trainer_monitoring(mock_trainer):
+    """Test basic trainer monitoring functionality"""
+    monitored_trainer = monitor(mock_trainer, dashboard=False)
+
+    # Test that the wrapper preserves the original interface
+    assert hasattr(monitored_trainer, "train")
+    assert hasattr(monitored_trainer, "state")
+
+    # Test training execution
+    result = monitored_trainer.train()
+    assert result == "success"
+    assert monitored_trainer.state.global_step == 3
+
+
+def test_environment_monitoring(mock_environment):
+    """Test basic environment monitoring functionality"""
+    monitored_env = monitor(mock_environment, dashboard=False)
+
+    # Test that the wrapper preserves the original interface
+    assert hasattr(monitored_env, "evaluate")
+    assert hasattr(monitored_env, "rollout")
+
+    # Test evaluation execution
+    results = monitored_env.evaluate("client", "model")
+    assert len(results.reward) == 3
+    assert all(isinstance(r, float) for r in results.reward)
+
+
+def test_monitor_invalid_object():
+    """Test that monitor raises error for invalid objects"""
+    with pytest.raises(ValueError, match="Cannot monitor object"):
+        monitor("invalid_object", dashboard=False)

tests/test_dashboard.py

@@ -0,0 +1,41 @@
+"""
+Test dashboard functionality
+"""
+
+import pytest
+
+from verifiers_monitor.core.sqlmodel_storage import SQLModelStorage
+from verifiers_monitor.dashboard.server import DashboardServer
+
+
+def test_dashboard_creation(sqlite_storage):
+    """Test dashboard server creation"""
+    dashboard = DashboardServer(sqlite_storage, port=8888)
+
+    assert dashboard.storage == sqlite_storage
+    assert dashboard.port == 8888
+    assert dashboard.app is not None
+
+
+def test_dashboard_html_generation(sqlite_storage):
+    """Test dashboard HTML generation"""
+    dashboard = DashboardServer(sqlite_storage, port=8888)
+
+    html = dashboard._get_dashboard_html_static()
+
+    # Check for key components
+    assert "Verifiers Monitor Dashboard" in html or "Verifiers Pro" in html
+    assert "<!DOCTYPE html>" in html
+    assert "</html>" in html
+
+
+def test_dashboard_routes_setup(sqlite_storage):
+    """Test that dashboard routes are properly configured"""
+    dashboard = DashboardServer(sqlite_storage, port=8888)
+
+    # Check that FastAPI app has routes configured
+    routes = [route.path for route in dashboard.app.routes]
+
+    expected_routes = ["/", "/api/metrics", "/api/status"]
+    for expected in expected_routes:
+        assert any(expected in route for route in routes)

tests/test_storage.py

@@ -0,0 +1,69 @@
+"""
+Test storage backend functionality
+"""
+
+import time
+
+import pytest
+
+from verifiers_monitor.core.sqlmodel_storage import SQLModelStorage
+
+
+def test_sqlite_storage_basic(sqlite_storage):
+    """Test basic SQLite storage operations with rollout data"""
+    rollout_data = {
+        "session_id": "test_session",
+        "example_number": 1,
+        "rollout_number": 1,
+        "prompt": [{"role": "user", "content": "test"}],
+        "completion": [{"role": "assistant", "content": "response"}],
+        "reward": 0.8,
+        "rollout_time": 0.5,
+        "timestamp": time.time(),
+    }
+
+    # Test store and retrieve
+    sqlite_storage.store("rollout", rollout_data)
+    data = sqlite_storage.get_recent("rollout")
+
+    assert len(data) == 1
+    assert data[0]["reward"] == 0.8
+
+
+def test_sqlite_storage_session(sqlite_storage):
+    """Test SQLite storage with evaluation sessions"""
+    session_data = {
+        "session_id": "test_session_123",
+        "model_name": "gpt-4o-mini",
+        "env_id": "math-python",
+        "num_examples": 10,
+        "timestamp": time.time(),
+    }
+
+    # Store session
+    sqlite_storage.store("evaluation_session", session_data)
+
+    # Verify session was stored
+    sessions = sqlite_storage.get_sessions()
+    assert len(sessions) > 0
+    assert any(s.session_id == "test_session_123" for s in sessions)
+
+
+def test_storage_get_stats(sqlite_storage):
+    """Test get_stats functionality"""
+    # Store some rollout data first
+    rollout_data = {
+        "session_id": "test_session",
+        "example_number": 1,
+        "rollout_number": 1,
+        "prompt": [{"role": "user", "content": "test"}],
+        "completion": [{"role": "assistant", "content": "response"}],
+        "reward": 0.9,
+        "rollout_time": 0.5,
+        "timestamp": time.time(),
+    }
+    sqlite_storage.store("rollout", rollout_data)
+
+    stats = sqlite_storage.get_stats()
+    assert "total_rollouts" in stats
+    assert stats["total_rollouts"] >= 1

verifiers_monitor/core/collectors.py

@@ -126,9 +126,9 @@ def start_evaluation_session(
 
         # Verifiers uses round-robin pattern, not sequential
         self._prompt_hash_map: Dict[str, int] = {}  # prompt_hash -> example_number
-        self._example_rollout_counts: Dict[
-            int, int
-        ] = {}  # example_number -> rollout_count
+        self._example_rollout_counts: Dict[int, int] = (
+            {}
+        )  # example_number -> rollout_count
         self._next_example_number = 1
 
         session_info = {