From 0e49d3a206231e43fa1313e07ba5abeaaafe6eac Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Fri, 20 Feb 2026 10:21:22 +0000 Subject: [PATCH 1/3] improve test --- tests/docs_mcp/conftest.py | 103 +++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 tests/docs_mcp/conftest.py diff --git a/tests/docs_mcp/conftest.py b/tests/docs_mcp/conftest.py new file mode 100644 index 0000000..9284fbe --- /dev/null +++ b/tests/docs_mcp/conftest.py @@ -0,0 +1,103 @@ +"""Test fixtures for documentation MCP tests. + +Configures a minimal 3-repo test index (panel, hvplot, panel-material-ui) +instead of the full production index, reducing test time from 5-15 minutes +to 1-3 minutes. Uses a fixed path (~/.holoviz-mcp-test/) for CI caching. +""" + +import asyncio +import logging +from pathlib import Path + +import pytest + +from holoviz_mcp.config.loader import ConfigLoader +from holoviz_mcp.config.models import HoloVizMCPConfig + +logger = logging.getLogger(__name__) + +# Fixed test directory (not tmp_path) so CI can cache the index across runs +TEST_DATA_DIR = Path.home() / ".holoviz-mcp-test" + +# Only these projects are needed by test assertions +TEST_PROJECTS = ("panel", "hvplot", "panel-material-ui") + + +def _build_test_config(): + """Build a HoloVizMCPConfig with only 3 repos and test-specific paths.""" + # Load ONLY the package default config — skip user config entirely so + # user-configured repos don't leak into the test index. + env = HoloVizMCPConfig(user_dir=TEST_DATA_DIR / ".no_user_config") + loader = ConfigLoader(config=env) + default_config = loader.load_config() + + # Keep only the repos needed for tests + test_repos = {name: default_config.docs.repositories[name] for name in TEST_PROJECTS if name in default_config.docs.repositories} + + test_docs = default_config.docs.model_copy(update={"repositories": test_repos}) + test_server = default_config.server.model_copy(update={"vector_db_path": TEST_DATA_DIR / "vector_db" / "chroma"}) + + return default_config.model_copy( + update={ + "docs": test_docs, + "server": test_server, + "user_dir": TEST_DATA_DIR, + "repos_dir": TEST_DATA_DIR / "repos", + } + ) + + +@pytest.fixture(scope="session", autouse=True) +def docs_test_config(): + """Patch get_config() to use a minimal 3-repo test config for the session. + + This fixture: + 1. Builds a config with only panel, hvplot, panel-material-ui + 2. Redirects all data paths to ~/.holoviz-mcp-test/ + 3. Resets the server's _indexer singleton so it picks up the test config + 4. Pre-builds the index if it doesn't exist (avoids a deadlock in + ensure_indexed when called from within db_lock in search/list_projects) + + Since ~/.holoviz-mcp-test/ persists between runs, subsequent runs skip indexing. + """ + import holoviz_mcp.config.loader as loader_module + import holoviz_mcp.holoviz_mcp.server as server_module + + config = _build_test_config() + + # Pre-load the config into the loader (bypasses file loading and env overrides) + test_loader = ConfigLoader() + test_loader._loaded_config = config + + # Save originals + original_loader = loader_module._config_loader + original_indexer = server_module._indexer + + # Patch global config loader and reset indexer singleton + loader_module._config_loader = test_loader + server_module._indexer = None + + logger.info( + "Docs test config: %d repos (%s), vector_db=%s", + len(config.docs.repositories), + ", ".join(config.docs.repositories.keys()), + config.server.vector_db_path, + ) + + # Pre-build the index if it doesn't exist. This MUST happen here (outside + # any db_lock) because the MCP tools call ensure_indexed() from within + # db_lock, and index_documentation() also acquires db_lock — causing a + # deadlock when the index is empty. By building here, ensure_indexed() + # always finds is_indexed()==True and skips the build. + indexer = server_module.get_indexer() + if not indexer.is_indexed(): + logger.info("Building test index for %s ...", ", ".join(TEST_PROJECTS)) + asyncio.run(indexer.ensure_indexed()) + # Reset the db_lock so tests create a fresh one in their event loop + indexer._db_lock = None + + yield config + + # Restore originals + loader_module._config_loader = original_loader + server_module._indexer = original_indexer From 7c92e5d9d744e087726ecc615d237098c41b9baf Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Fri, 20 Feb 2026 10:21:42 +0000 Subject: [PATCH 2/3] test flow improved --- .github/workflows/ci.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 836461c..1bc30b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,6 +110,18 @@ jobs: uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659 # v0.8.1 with: environments: ${{ matrix.environment }} + - name: Get cache week key + id: cache-week + run: echo "week=$(date +%Y-W%V)" >> $GITHUB_OUTPUT + shell: bash + - name: Cache test index + uses: actions/cache@v4 + with: + path: ~/.holoviz-mcp-test + key: holoviz-mcp-test-index-${{ runner.os }}-${{ hashFiles('src/holoviz_mcp/config/config.yaml') }}-${{ steps.cache-week.outputs.week }} + restore-keys: | + holoviz-mcp-test-index-${{ runner.os }}-${{ hashFiles('src/holoviz_mcp/config/config.yaml') }}- + holoviz-mcp-test-index-${{ runner.os }}- - name: Install repository run: pixi run -e ${{ matrix.environment }} postinstall - name: Run pytest From 5edfe0ccce3733adb818b8d6dc670571fcea94fa Mon Sep 17 00:00:00 2001 From: Marc Skov Madsen Date: Fri, 20 Feb 2026 16:25:07 +0000 Subject: [PATCH 3/3] copilot feedback --- .github/workflows/ci.yml | 10 ++++++---- AGENTS.md | 4 ++-- pyproject.toml | 3 +++ tests/docs_mcp/conftest.py | 15 ++++++++++----- tests/docs_mcp/test_docs_mcp.py | 10 ++++++++++ tests/docs_mcp/test_docs_mcp_reference_guide.py | 2 ++ tests/docs_mcp/test_tabulator_truncation.py | 2 ++ 7 files changed, 35 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bc30b4..93b4e21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,14 +118,16 @@ jobs: uses: actions/cache@v4 with: path: ~/.holoviz-mcp-test - key: holoviz-mcp-test-index-${{ runner.os }}-${{ hashFiles('src/holoviz_mcp/config/config.yaml') }}-${{ steps.cache-week.outputs.week }} + key: holoviz-mcp-test-index-${{ runner.os }}-${{ hashFiles('src/holoviz_mcp/config/config.yaml', 'src/holoviz_mcp/holoviz_mcp/data.py') }}-${{ steps.cache-week.outputs.week }} restore-keys: | - holoviz-mcp-test-index-${{ runner.os }}-${{ hashFiles('src/holoviz_mcp/config/config.yaml') }}- + holoviz-mcp-test-index-${{ runner.os }}-${{ hashFiles('src/holoviz_mcp/config/config.yaml', 'src/holoviz_mcp/holoviz_mcp/data.py') }}- holoviz-mcp-test-index-${{ runner.os }}- - name: Install repository run: pixi run -e ${{ matrix.environment }} postinstall - - name: Run pytest - run: pixi run -e ${{ matrix.environment }} test-coverage --color=yes + - name: Run unit tests (fast) + run: pixi run -e ${{ matrix.environment }} pytest tests/ --ignore=tests/docs_mcp --color=yes + - name: Run docs integration tests (slow, uses cached index) + run: pixi run -e ${{ matrix.environment }} pytest tests/docs_mcp --color=yes pytest_ui: name: ui:${{ matrix.environment }}:${{ matrix.os }} diff --git a/AGENTS.md b/AGENTS.md index cc8891f..b2cbc6b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,7 +11,7 @@ A README for AI coding agents to effectively contribute to the HoloViz MCP proje **Language**: Python 3.11+ **Build System**: Pixi (conda-based) + Hatchling (PEP 517) **Framework**: FastMCP (Python MCP SDK) -**Key Dependencies**: ChromaDB, sentence-transformers, Panel, PyTorch +**Key Dependencies**: ChromaDB, sentence-transformers, Panel ### Project Statistics - **Source Files**: ~20 Python modules in `src/holoviz_mcp/` @@ -564,7 +564,7 @@ First-time indexing takes 5-10 minutes: ### Memory Usage -Heavy dependencies (PyTorch, sentence-transformers): +Heavy dependencies (sentence-transformers, ChromaDB): - **Development**: ~2-4 GB RAM - **Running server**: ~1-2 GB RAM - **Indexing docs**: ~3-5 GB RAM (temporary spike) diff --git a/pyproject.toml b/pyproject.toml index d369fc2..a3cc51d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -176,6 +176,9 @@ filterwarnings = [ "ignore::UserWarning", ] testpaths = ["tests"] +markers = [ + "integration: marks tests that require the documentation index (slow)", +] [tool.codespell] ignore-words-list = "acn" diff --git a/tests/docs_mcp/conftest.py b/tests/docs_mcp/conftest.py index 9284fbe..14d357a 100644 --- a/tests/docs_mcp/conftest.py +++ b/tests/docs_mcp/conftest.py @@ -7,6 +7,7 @@ import asyncio import logging +import os from pathlib import Path import pytest @@ -16,8 +17,9 @@ logger = logging.getLogger(__name__) -# Fixed test directory (not tmp_path) so CI can cache the index across runs -TEST_DATA_DIR = Path.home() / ".holoviz-mcp-test" +# Fixed test directory (not tmp_path) so CI can cache the index across runs. +# Override with HOLOVIZ_MCP_TEST_DIR for custom locations or concurrent sessions. +TEST_DATA_DIR = Path(os.environ["HOLOVIZ_MCP_TEST_DIR"]) if "HOLOVIZ_MCP_TEST_DIR" in os.environ else Path.home() / ".holoviz-mcp-test" # Only these projects are needed by test assertions TEST_PROJECTS = ("panel", "hvplot", "panel-material-ui") @@ -31,8 +33,11 @@ def _build_test_config(): loader = ConfigLoader(config=env) default_config = loader.load_config() - # Keep only the repos needed for tests - test_repos = {name: default_config.docs.repositories[name] for name in TEST_PROJECTS if name in default_config.docs.repositories} + # Validate that all expected test repos exist in the default config + missing = set(TEST_PROJECTS) - set(default_config.docs.repositories) + if missing: + raise RuntimeError(f"TEST_PROJECTS not found in default config: {missing}") + test_repos = {name: default_config.docs.repositories[name] for name in TEST_PROJECTS} test_docs = default_config.docs.model_copy(update={"repositories": test_repos}) test_server = default_config.server.model_copy(update={"vector_db_path": TEST_DATA_DIR / "vector_db" / "chroma"}) @@ -47,7 +52,7 @@ def _build_test_config(): ) -@pytest.fixture(scope="session", autouse=True) +@pytest.fixture(scope="package", autouse=True) def docs_test_config(): """Patch get_config() to use a minimal 3-repo test config for the session. diff --git a/tests/docs_mcp/test_docs_mcp.py b/tests/docs_mcp/test_docs_mcp.py index be03198..6084019 100644 --- a/tests/docs_mcp/test_docs_mcp.py +++ b/tests/docs_mcp/test_docs_mcp.py @@ -19,6 +19,7 @@ async def test_skills_resource(): assert result.data +@pytest.mark.integration @pytest.mark.skip(reason="this test is very slow") @pytest.mark.asyncio async def test_update_index(): @@ -29,6 +30,7 @@ async def test_update_index(): assert result.data +@pytest.mark.integration @pytest.mark.asyncio async def test_list_projects(): """Test that all projects are listed correctly.""" @@ -40,6 +42,7 @@ async def test_list_projects(): assert "panel" in result.data +@pytest.mark.integration @pytest.mark.asyncio async def test_semantic_search(): """Test the search tool.""" @@ -63,6 +66,7 @@ async def test_semantic_search(): assert "content" in document +@pytest.mark.integration @pytest.mark.asyncio async def test_search_by_project(): """Test the search tool with project filtering.""" @@ -78,6 +82,7 @@ async def test_search_by_project(): assert document["project"] == "hvplot" +@pytest.mark.integration @pytest.mark.asyncio async def test_search_with_custom_max_results(): """Test the search tool with custom max_results parameter.""" @@ -95,6 +100,7 @@ async def test_search_with_custom_max_results(): assert document["project"] == "panel" +@pytest.mark.integration @pytest.mark.asyncio async def test_search_without_content(): """Test the search tool with content=False for metadata only.""" @@ -116,6 +122,7 @@ async def test_search_without_content(): assert document.get("content") is None +@pytest.mark.integration @pytest.mark.asyncio async def test_search_material_ui_specific(): """Test the search tool with Material UI specific query.""" @@ -131,6 +138,7 @@ async def test_search_material_ui_specific(): assert document["project"] == "panel-material-ui" +@pytest.mark.integration @pytest.mark.asyncio async def test_search_empty_query(): """Test the search tool with edge cases.""" @@ -142,6 +150,7 @@ async def test_search_empty_query(): assert isinstance(result.data, list) +@pytest.mark.integration @pytest.mark.asyncio async def test_search_invalid_project(): """Test the search tool with invalid project name.""" @@ -154,6 +163,7 @@ async def test_search_invalid_project(): assert len(result.data) == 0 +@pytest.mark.integration @pytest.mark.asyncio async def test_search_with_project_filter(): """Test the search tool with project filtering.""" diff --git a/tests/docs_mcp/test_docs_mcp_reference_guide.py b/tests/docs_mcp/test_docs_mcp_reference_guide.py index 129df85..302e19c 100644 --- a/tests/docs_mcp/test_docs_mcp_reference_guide.py +++ b/tests/docs_mcp/test_docs_mcp_reference_guide.py @@ -9,6 +9,8 @@ from holoviz_mcp.holoviz_mcp.server import mcp +pytestmark = pytest.mark.integration + @pytest.mark.asyncio async def test_get_reference_guide_button_no_project(): diff --git a/tests/docs_mcp/test_tabulator_truncation.py b/tests/docs_mcp/test_tabulator_truncation.py index 7da9557..5588950 100644 --- a/tests/docs_mcp/test_tabulator_truncation.py +++ b/tests/docs_mcp/test_tabulator_truncation.py @@ -14,6 +14,8 @@ from holoviz_mcp.holoviz_mcp.data import DocumentationIndexer from holoviz_mcp.holoviz_mcp.data import truncate_content +pytestmark = pytest.mark.integration + class TestTabulatorTruncation: """Test suite for Tabulator reference guide context-aware truncation."""