feat: enhance indexing with ignore file support and dry-run option

Author: iLoveTux

README.md

@@ -132,6 +132,13 @@ th index . --in-memory
 th index . --in-memory --output json
 ```
 
+Preview which files would be indexed without parsing or writing any SQLite state:
+
+```powershell
+th index . --dry-run
+th index . --dry-run --output json
+```
+
 Watch for file changes and reindex incrementally (Ctrl-C to stop):
 
 ```powershell
@@ -149,6 +156,18 @@ th index . --sqlite-db ./.trailhead/graph.db --embed-model sentence-transformers
 
 When `sqlite-vector` can be loaded, trailhead also initializes vector search for the `vertex_embeddings.embedding` column. If extension loading is unavailable on your platform build, embeddings are still stored as Float32 BLOBs in SQLite.
 
+Source discovery respects `.gitignore` and `.trailheadignore` in the indexed directory. Both use gitignore-style patterns, and `.trailheadignore` is applied after `.gitignore` so Trailhead-specific rules take precedence. If you change either ignore file, delete `.trailhead/db.sqlite` and run `th index` again to rebuild the index with the new rules.
+
+`th index --dry-run --output json` returns a file-preview payload with this schema:
+
+```json
+{
+  "root": "C:/path/to/project",
+  "count": 2,
+  "files": ["src/app.py", "src/lib/util.py"]
+}
+```
+
 ### serve
 
 Run the warm-model API server with a background indexer. The server watches the source tree, keeps the SQLite graph fresh, and reuses the loaded embedding model across index updates. The database defaults to `.trailhead/db.sqlite` under the watched directory:

docs/architecture-blog-post.md

@@ -27,6 +27,7 @@ classifiers = [
 
 dependencies = [
   "sentence-transformers",
+  "pathspec>=0.12",
   "tree-sitter>=0.21",
   "tree-sitter-python>=0.21",
   "sqliteai-vector",

pyproject.toml

@@ -1,6 +1,7 @@
 """Tests for the indexing service (property graph model + tree-sitter parser)."""
 from __future__ import annotations
 
+import json
 import sqlite3
 
 import pytest
@@ -390,6 +391,33 @@ def test_ignores_non_python_files(self, tmp_path):
         graph = index_directory(tmp_path)
         assert len(graph.vertices("module")) == 1
 
+    def test_respects_ignore_files_with_trailheadignore_precedence(self, tmp_path):
+        from trailhead.services.indexing.walker import index_directory
+
+        (tmp_path / ".gitignore").write_text("*.py\n")
+        (tmp_path / ".trailheadignore").write_text("!keep.py\n")
+        (tmp_path / "keep.py").write_text("def keep():\n    pass\n")
+        (tmp_path / "drop.py").write_text("def drop():\n    pass\n")
+
+        graph = index_directory(tmp_path)
+
+        module_names = {v.properties["name"] for v in graph.vertices("module")}
+        assert module_names == {"keep"}
+
+    def test_ignores_directories_listed_in_ignore_files(self, tmp_path):
+        from trailhead.services.indexing.walker import index_directory
+
+        generated = tmp_path / "generated"
+        generated.mkdir()
+        (tmp_path / ".gitignore").write_text("generated/\n")
+        (tmp_path / "keep.py").write_text("def keep():\n    pass\n")
+        (generated / "skip.py").write_text("def skip():\n    pass\n")
+
+        graph = index_directory(tmp_path)
+
+        module_names = {v.properties["name"] for v in graph.vertices("module")}
+        assert module_names == {"keep"}
+
     def test_accepts_existing_graph(self, tmp_path):
         from trailhead.services.indexing.graph import PropertyGraph
         from trailhead.services.indexing.walker import index_directory
@@ -517,6 +545,23 @@ def test_persist_append_mode_keeps_existing_rows(self, tmp_path):
         assert v_count == 4
         assert e_count == 2
 
+    def test_persist_indexed_files_respects_ignore_files(self, tmp_path):
+        from trailhead.services.indexing.sqlite_store import get_indexed_files
+        from trailhead.services.indexing.sqlite_store import persist_indexed_files
+
+        root = tmp_path / "src"
+        root.mkdir()
+        (root / ".gitignore").write_text("skip.py\n")
+        (root / "keep.py").write_text("def keep():\n    pass\n")
+        (root / "skip.py").write_text("def skip():\n    pass\n")
+        db = tmp_path / "graph.db"
+
+        count = persist_indexed_files(root, db)
+        indexed_files = get_indexed_files(db)
+
+        assert count == 1
+        assert set(indexed_files) == {str((root / "keep.py").resolve())}
+
     def test_persist_vertex_embeddings_writes_rows(self, tmp_path, monkeypatch):
         from trailhead.services.indexing.graph import PropertyGraph
         from trailhead.services.indexing.sqlite_store import persist_graph
@@ -629,3 +674,63 @@ def test_index_command_embed_model_persists_vectors(self, tmp_path, monkeypatch)
             ]
         )
         assert rc == 0
+
+
+class TestIndexCommandDryRun:
+    def test_dry_run_lists_candidates_without_creating_db(self, tmp_path, capsys):
+        from trailhead.cli import app as cli
+
+        src = tmp_path / "src"
+        src.mkdir()
+        (src / ".gitignore").write_text("*.py\n")
+        (src / ".trailheadignore").write_text("!keep.py\n")
+        (src / "keep.py").write_text("def keep():\n    pass\n")
+        (src / "drop.py").write_text("def drop():\n    pass\n")
+
+        rc = cli.main(["index", str(src), "--dry-run"])
+        output = capsys.readouterr().out
+
+        assert rc == 0
+        assert "Would index 1 file(s)" in output
+        assert "keep.py" in output
+        assert "drop.py" not in output
+        assert not (src / ".trailhead").exists()
+
+    def test_dry_run_json_uses_preview_schema(self, tmp_path, capsys):
+        from trailhead.cli import app as cli
+
+        src = tmp_path / "src"
+        src.mkdir()
+        (src / "keep.py").write_text("def keep():\n    pass\n")
+
+        rc = cli.main(["index", str(src), "--dry-run", "--output", "json"])
+        payload = json.loads(capsys.readouterr().out)
+
+        assert rc == 0
+        assert payload == {
+            "root": str(src.resolve()),
+            "count": 1,
+            "files": ["keep.py"],
+        }
+
+    @pytest.mark.parametrize(
+        "argv",
+        [
+            ["index", "src", "--dry-run", "--watch"],
+            ["index", "src", "--dry-run", "--in-memory"],
+            ["index", "src", "--dry-run", "--sqlite-db", "graph.db"],
+            ["index", "src", "--dry-run", "--embed-model", "sentence-transformers/all-MiniLM-L6-v2"],
+        ],
+    )
+    def test_dry_run_rejects_incompatible_flags(self, tmp_path, argv):
+        from trailhead.cli import app as cli
+
+        src = tmp_path / "src"
+        src.mkdir()
+        (src / "keep.py").write_text("def keep():\n    pass\n")
+
+        normalized_argv = [str(src) if arg == "src" else arg for arg in argv]
+
+        rc = cli.main(normalized_argv)
+
+        assert rc == 1

src/tests/test_indexing.py

@@ -116,6 +116,34 @@ def fake_reindex_paths(paths):
         assert len(reindexed) == 1
         assert any(p.name == "foo.py" for p in reindexed[0])
 
+    def test_watcher_ignores_ignore_file_updates(self, tmp_path, monkeypatch):
+        from trailhead.services.indexing.live_indexer import LiveIndexer
+
+        root = tmp_path / "src"
+        root.mkdir()
+        db = tmp_path / "graph.db"
+
+        reindexed: list[set] = []
+
+        def fake_reindex_paths(paths):
+            reindexed.append(paths)
+
+        fake_changes = [[(1, str(root / ".trailheadignore")), (1, str(root / ".gitignore"))]]
+
+        monkeypatch.setattr(
+            "trailhead.services.indexing.live_indexer.watch",
+            lambda *a, **kw: iter(fake_changes),
+        )
+
+        service = LiveIndexer(root=root, db_path=db)
+        monkeypatch.setattr(service, "reindex_paths", fake_reindex_paths)
+        service.start()
+        if service._watch_thread is not None:
+            service._watch_thread.join(timeout=5)
+        service.stop()
+
+        assert reindexed == []
+
 
 class TestLiveIndexer:
     def test_synchronize_reindexes_changed_new_and_deleted_files(self, tmp_path, monkeypatch):
@@ -158,3 +186,28 @@ def fake_reindex_file(db_path, path, **kwargs):
             new_file.resolve(),
             deleted_file.resolve(),
         }
+
+    def test_reindex_paths_skips_ignored_files_using_current_rules(self, tmp_path, monkeypatch):
+        from trailhead.services.indexing.live_indexer import LiveIndexer
+
+        root = tmp_path / "src"
+        root.mkdir()
+        keep = root / "keep.py"
+        ignored = root / "ignored.py"
+        keep.write_text("def keep():\n    pass\n")
+        ignored.write_text("def ignored():\n    pass\n")
+        (root / ".gitignore").write_text("*.py\n")
+        (root / ".trailheadignore").write_text("!keep.py\n")
+
+        seen_paths: list[Path] = []
+
+        def fake_reindex_file(db_path, path, **kwargs):
+            seen_paths.append(path.resolve())
+            return (0, 0)
+
+        monkeypatch.setattr("trailhead.services.indexing.live_indexer.reindex_file", fake_reindex_file)
+
+        service = LiveIndexer(root=root, db_path=tmp_path / "graph.db")
+        service.reindex_paths({keep, ignored})
+
+        assert seen_paths == [keep.resolve()]

src/tests/test_live_indexer.py

@@ -10,6 +10,7 @@
 from trailhead.services.config import ALLOWED_MODELS
 from trailhead.services.config import get_cache_dir
 from trailhead.services.config import is_model_allowed
+from trailhead.services.indexing import discover_source_files
 from trailhead.services.indexing import LiveIndexer
 from trailhead.services.indexing.graph import PropertyGraph
 from trailhead.services.indexing.sqlite_store import get_index_model
@@ -56,6 +57,15 @@ def configure_parser(subparsers: argparse._SubParsersAction) -> None:
             "Incompatible with --watch."
         ),
     )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help=(
+            "List the source files that would be indexed and exit without parsing "
+            "or writing SQLite state. Incompatible with --watch, --in-memory, "
+            "--sqlite-db, and --embed-model."
+        ),
+    )
     parser.add_argument(
         "--watch",
         action="store_true",
@@ -69,7 +79,7 @@ def configure_parser(subparsers: argparse._SubParsersAction) -> None:
         "--output",
         choices=["summary", "json"],
         default="summary",
-        help="Output format when --in-memory is set (default: summary).",
+        help="Output format when --in-memory or --dry-run is set (default: summary).",
     )
     parser.add_argument(
         "--embed-model",
@@ -113,10 +123,26 @@ def run(args: argparse.Namespace) -> int:
         logger.error("--in-memory and --watch are mutually exclusive.")
         return 1
 
+    if args.dry_run and args.watch:
+        logger.error("--dry-run and --watch are mutually exclusive.")
+        return 1
+
+    if args.dry_run and args.in_memory:
+        logger.error("--dry-run and --in-memory are mutually exclusive.")
+        return 1
+
+    if args.dry_run and args.sqlite_db:
+        logger.error("--dry-run does not accept --sqlite-db.")
+        return 1
+
     if args.embed_model and args.in_memory:
         logger.error("--embed-model requires SQLite persistence; remove --in-memory.")
         return 1
 
+    if args.embed_model and args.dry_run:
+        logger.error("--embed-model requires SQLite persistence; remove --dry-run.")
+        return 1
+
     if args.embed_model and not args.sqlite_db:
         logger.error("--embed-model requires --sqlite-db.")
         return 1
@@ -130,6 +156,18 @@ def run(args: argparse.Namespace) -> int:
         )
         return 1
 
+    # ------------------------------------------------------------------
+    # Dry-run path: enumerate files only, print preview, then exit.
+    # ------------------------------------------------------------------
+    if args.dry_run:
+        logger.info("Previewing index candidates under %s", root)
+        files = discover_source_files(root)
+        if args.output == "json":
+            _print_dry_run_json(root, files)
+        else:
+            _print_dry_run_summary(root, files)
+        return 0
+
     # ------------------------------------------------------------------
     # In-memory path: one-shot build, print summary/JSON, then exit.
     # ------------------------------------------------------------------
@@ -241,3 +279,21 @@ def _print_summary(graph: PropertyGraph) -> None:
         if imports:
             names = sorted(e.in_v.properties["name"] for e in imports)
             print(f"    imports: {', '.join(names)}")
+
+
+def _print_dry_run_json(root: Path, files: list[Path]) -> None:
+    data = {
+        "root": str(root),
+        "count": len(files),
+        "files": [path.relative_to(root).as_posix() for path in files],
+    }
+    print(json.dumps(data, indent=2))
+
+
+def _print_dry_run_summary(root: Path, files: list[Path]) -> None:
+    print(f"Would index {len(files)} file(s) under {root}.")
+    if not files:
+        return
+    print()
+    for path in files:
+        print(path.relative_to(root).as_posix())

src/trailhead/cli/commands/index.py

@@ -1,5 +1,8 @@
 """Code indexing service — property graph construction from source trees."""
 
+from trailhead.services.indexing.discovery import discover_source_files
+from trailhead.services.indexing.discovery import load_ignore_spec
+from trailhead.services.indexing.discovery import should_index_path
 from trailhead.services.indexing.graph import PropertyGraph
 from trailhead.services.indexing.graph_query import search_vertices
 from trailhead.services.indexing.graph_query import traverse_graph
@@ -18,11 +21,14 @@
 __all__ = [
     "PropertyGraph",
     "LiveIndexer",
+    "discover_source_files",
     "index_directory",
+    "load_ignore_spec",
     "persist_graph",
     "persist_indexed_files",
     "persist_vertex_embeddings",
     "reindex_file",
+    "should_index_path",
     "vector_to_blob",
     "get_indexed_files",
     "load_graph",