diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 836461c..93b4e21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,10 +110,24 @@ jobs: uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659 # v0.8.1 with: environments: ${{ matrix.environment }} + - name: Get cache week key + id: cache-week + run: echo "week=$(date +%Y-W%V)" >> $GITHUB_OUTPUT + shell: bash + - name: Cache test index + uses: actions/cache@v4 + with: + path: ~/.holoviz-mcp-test + key: holoviz-mcp-test-index-${{ runner.os }}-${{ hashFiles('src/holoviz_mcp/config/config.yaml', 'src/holoviz_mcp/holoviz_mcp/data.py') }}-${{ steps.cache-week.outputs.week }} + restore-keys: | + holoviz-mcp-test-index-${{ runner.os }}-${{ hashFiles('src/holoviz_mcp/config/config.yaml', 'src/holoviz_mcp/holoviz_mcp/data.py') }}- + holoviz-mcp-test-index-${{ runner.os }}- - name: Install repository run: pixi run -e ${{ matrix.environment }} postinstall - - name: Run pytest - run: pixi run -e ${{ matrix.environment }} test-coverage --color=yes + - name: Run unit tests (fast) + run: pixi run -e ${{ matrix.environment }} pytest tests/ --ignore=tests/docs_mcp --color=yes + - name: Run docs integration tests (slow, uses cached index) + run: pixi run -e ${{ matrix.environment }} pytest tests/docs_mcp --color=yes pytest_ui: name: ui:${{ matrix.environment }}:${{ matrix.os }} diff --git a/AGENTS.md b/AGENTS.md index cc8891f..b2cbc6b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,7 +11,7 @@ A README for AI coding agents to effectively contribute to the HoloViz MCP proje **Language**: Python 3.11+ **Build System**: Pixi (conda-based) + Hatchling (PEP 517) **Framework**: FastMCP (Python MCP SDK) -**Key Dependencies**: ChromaDB, sentence-transformers, Panel, PyTorch +**Key Dependencies**: ChromaDB, sentence-transformers, Panel ### Project Statistics - **Source Files**: ~20 Python modules in `src/holoviz_mcp/` @@ -564,7 +564,7 @@ First-time indexing takes 5-10 minutes: ### Memory Usage -Heavy dependencies (PyTorch, sentence-transformers): +Heavy dependencies (sentence-transformers, ChromaDB): - **Development**: ~2-4 GB RAM - **Running server**: ~1-2 GB RAM - **Indexing docs**: ~3-5 GB RAM (temporary spike) diff --git a/pyproject.toml b/pyproject.toml index d369fc2..a3cc51d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -176,6 +176,9 @@ filterwarnings = [ "ignore::UserWarning", ] testpaths = ["tests"] +markers = [ + "integration: marks tests that require the documentation index (slow)", +] [tool.codespell] ignore-words-list = "acn" diff --git a/tests/docs_mcp/conftest.py b/tests/docs_mcp/conftest.py new file mode 100644 index 0000000..14d357a --- /dev/null +++ b/tests/docs_mcp/conftest.py @@ -0,0 +1,108 @@ +"""Test fixtures for documentation MCP tests. + +Configures a minimal 3-repo test index (panel, hvplot, panel-material-ui) +instead of the full production index, reducing test time from 5-15 minutes +to 1-3 minutes. Uses a fixed path (~/.holoviz-mcp-test/) for CI caching. +""" + +import asyncio +import logging +import os +from pathlib import Path + +import pytest + +from holoviz_mcp.config.loader import ConfigLoader +from holoviz_mcp.config.models import HoloVizMCPConfig + +logger = logging.getLogger(__name__) + +# Fixed test directory (not tmp_path) so CI can cache the index across runs. +# Override with HOLOVIZ_MCP_TEST_DIR for custom locations or concurrent sessions. +TEST_DATA_DIR = Path(os.environ["HOLOVIZ_MCP_TEST_DIR"]) if "HOLOVIZ_MCP_TEST_DIR" in os.environ else Path.home() / ".holoviz-mcp-test" + +# Only these projects are needed by test assertions +TEST_PROJECTS = ("panel", "hvplot", "panel-material-ui") + + +def _build_test_config(): + """Build a HoloVizMCPConfig with only 3 repos and test-specific paths.""" + # Load ONLY the package default config — skip user config entirely so + # user-configured repos don't leak into the test index. + env = HoloVizMCPConfig(user_dir=TEST_DATA_DIR / ".no_user_config") + loader = ConfigLoader(config=env) + default_config = loader.load_config() + + # Validate that all expected test repos exist in the default config + missing = set(TEST_PROJECTS) - set(default_config.docs.repositories) + if missing: + raise RuntimeError(f"TEST_PROJECTS not found in default config: {missing}") + test_repos = {name: default_config.docs.repositories[name] for name in TEST_PROJECTS} + + test_docs = default_config.docs.model_copy(update={"repositories": test_repos}) + test_server = default_config.server.model_copy(update={"vector_db_path": TEST_DATA_DIR / "vector_db" / "chroma"}) + + return default_config.model_copy( + update={ + "docs": test_docs, + "server": test_server, + "user_dir": TEST_DATA_DIR, + "repos_dir": TEST_DATA_DIR / "repos", + } + ) + + +@pytest.fixture(scope="package", autouse=True) +def docs_test_config(): + """Patch get_config() to use a minimal 3-repo test config for the session. + + This fixture: + 1. Builds a config with only panel, hvplot, panel-material-ui + 2. Redirects all data paths to ~/.holoviz-mcp-test/ + 3. Resets the server's _indexer singleton so it picks up the test config + 4. Pre-builds the index if it doesn't exist (avoids a deadlock in + ensure_indexed when called from within db_lock in search/list_projects) + + Since ~/.holoviz-mcp-test/ persists between runs, subsequent runs skip indexing. + """ + import holoviz_mcp.config.loader as loader_module + import holoviz_mcp.holoviz_mcp.server as server_module + + config = _build_test_config() + + # Pre-load the config into the loader (bypasses file loading and env overrides) + test_loader = ConfigLoader() + test_loader._loaded_config = config + + # Save originals + original_loader = loader_module._config_loader + original_indexer = server_module._indexer + + # Patch global config loader and reset indexer singleton + loader_module._config_loader = test_loader + server_module._indexer = None + + logger.info( + "Docs test config: %d repos (%s), vector_db=%s", + len(config.docs.repositories), + ", ".join(config.docs.repositories.keys()), + config.server.vector_db_path, + ) + + # Pre-build the index if it doesn't exist. This MUST happen here (outside + # any db_lock) because the MCP tools call ensure_indexed() from within + # db_lock, and index_documentation() also acquires db_lock — causing a + # deadlock when the index is empty. By building here, ensure_indexed() + # always finds is_indexed()==True and skips the build. + indexer = server_module.get_indexer() + if not indexer.is_indexed(): + logger.info("Building test index for %s ...", ", ".join(TEST_PROJECTS)) + asyncio.run(indexer.ensure_indexed()) + # Reset the db_lock so tests create a fresh one in their event loop + indexer._db_lock = None + + yield config + + # Restore originals + loader_module._config_loader = original_loader + server_module._indexer = original_indexer diff --git a/tests/docs_mcp/test_docs_mcp.py b/tests/docs_mcp/test_docs_mcp.py index be03198..6084019 100644 --- a/tests/docs_mcp/test_docs_mcp.py +++ b/tests/docs_mcp/test_docs_mcp.py @@ -19,6 +19,7 @@ async def test_skills_resource(): assert result.data +@pytest.mark.integration @pytest.mark.skip(reason="this test is very slow") @pytest.mark.asyncio async def test_update_index(): @@ -29,6 +30,7 @@ async def test_update_index(): assert result.data +@pytest.mark.integration @pytest.mark.asyncio async def test_list_projects(): """Test that all projects are listed correctly.""" @@ -40,6 +42,7 @@ async def test_list_projects(): assert "panel" in result.data +@pytest.mark.integration @pytest.mark.asyncio async def test_semantic_search(): """Test the search tool.""" @@ -63,6 +66,7 @@ async def test_semantic_search(): assert "content" in document +@pytest.mark.integration @pytest.mark.asyncio async def test_search_by_project(): """Test the search tool with project filtering.""" @@ -78,6 +82,7 @@ async def test_search_by_project(): assert document["project"] == "hvplot" +@pytest.mark.integration @pytest.mark.asyncio async def test_search_with_custom_max_results(): """Test the search tool with custom max_results parameter.""" @@ -95,6 +100,7 @@ async def test_search_with_custom_max_results(): assert document["project"] == "panel" +@pytest.mark.integration @pytest.mark.asyncio async def test_search_without_content(): """Test the search tool with content=False for metadata only.""" @@ -116,6 +122,7 @@ async def test_search_without_content(): assert document.get("content") is None +@pytest.mark.integration @pytest.mark.asyncio async def test_search_material_ui_specific(): """Test the search tool with Material UI specific query.""" @@ -131,6 +138,7 @@ async def test_search_material_ui_specific(): assert document["project"] == "panel-material-ui" +@pytest.mark.integration @pytest.mark.asyncio async def test_search_empty_query(): """Test the search tool with edge cases.""" @@ -142,6 +150,7 @@ async def test_search_empty_query(): assert isinstance(result.data, list) +@pytest.mark.integration @pytest.mark.asyncio async def test_search_invalid_project(): """Test the search tool with invalid project name.""" @@ -154,6 +163,7 @@ async def test_search_invalid_project(): assert len(result.data) == 0 +@pytest.mark.integration @pytest.mark.asyncio async def test_search_with_project_filter(): """Test the search tool with project filtering.""" diff --git a/tests/docs_mcp/test_docs_mcp_reference_guide.py b/tests/docs_mcp/test_docs_mcp_reference_guide.py index 129df85..302e19c 100644 --- a/tests/docs_mcp/test_docs_mcp_reference_guide.py +++ b/tests/docs_mcp/test_docs_mcp_reference_guide.py @@ -9,6 +9,8 @@ from holoviz_mcp.holoviz_mcp.server import mcp +pytestmark = pytest.mark.integration + @pytest.mark.asyncio async def test_get_reference_guide_button_no_project(): diff --git a/tests/docs_mcp/test_tabulator_truncation.py b/tests/docs_mcp/test_tabulator_truncation.py index 7da9557..5588950 100644 --- a/tests/docs_mcp/test_tabulator_truncation.py +++ b/tests/docs_mcp/test_tabulator_truncation.py @@ -14,6 +14,8 @@ from holoviz_mcp.holoviz_mcp.data import DocumentationIndexer from holoviz_mcp.holoviz_mcp.data import truncate_content +pytestmark = pytest.mark.integration + class TestTabulatorTruncation: """Test suite for Tabulator reference guide context-aware truncation."""