Add Step 2: ChessBase MegaDatabase indexer and query

megabase/indexer.py: streams a PGN export into a SQLite index with
indexes on white/black player name columns. Handles multi-GB files
without loading into memory; batches inserts; skips games with no
player names.

megabase/query.py: case-insensitive partial-name lookup across both
colour columns; returns PGN or JSON; supports --limit; read-only DB
connection for safety.

12 new tests covering indexing, schema, field storage, white/black
lookup, partial match, case insensitivity, limit, date ordering,
and PGN presence in results.

README updated with Step 2 usage docs and Python API example.

https://claude.ai/code/session_01VQfqug9MDEyRFydkmES4n2

Author: claude

README.md

@@ -98,13 +98,46 @@ inspect the HTML to identify the right selector to add:
 python scraper.py Challenge34 --save-html page.html
 ```
 
+## Step 2 — ChessBase MegaDatabase
+
+Export the MegaDatabase from ChessBase once (**File → Export → Export Database as PGN**),
+then build a local SQLite index for fast per-player lookups.
+
+### Build the index (once)
+
+```bash
+python -m megabase.indexer mega.pgn
+python -m megabase.indexer mega.pgn --db /data/megabase.db   # custom path
+```
+
+Streams the PGN — never loads the whole file into memory. Progress is printed every 10,000 games.
+
+### Query by player name
+
+```bash
+python -m megabase.query "Kasparov, Garry"
+python -m megabase.query "Kasparov, Garry" --output json
+python -m megabase.query "Kasparov" --limit 50              # partial name match
+python -m megabase.query "Kasparov, Garry" --db /data/megabase.db
+```
+
+Returns PGN (default) or JSON. Matching is case-insensitive and covers both White and Black.
+
+### Python API
+
+```python
+from megabase.query import get_player_games
+
+games = get_player_games("Kasparov, Garry", db_path="megabase.db")
+for game in games:
+    print(game["event"], game["date"], game["result"])
+    print(game["pgn"])
+```
+
 ## Roadmap
 
 - [x] Step 1 — Scrape tournament entry lists (kingregistration, chessaction)
-- [ ] Step 2 — Index ChessBase MegaDatabase for fast player lookups
-  - 2a. Export MegaDatabase to PGN from ChessBase (File → Export → Export Database as PGN)
-  - 2b. Stream PGN into a local SQLite index keyed on player name
-  - 2c. Query index by name to retrieve all games as PGN
+- [x] Step 2 — Index ChessBase MegaDatabase for fast player lookups
 - [ ] Step 3 — Look up each player on USCF / chess.com / Lichess
 - [ ] Step 4 — Analyse openings and tendencies
 - [ ] Step 5 — Generate per-opponent dossier report

megabase/__init__.py

@@ -0,0 +1,122 @@
+"""
+Build a SQLite index from a ChessBase MegaDatabase PGN export.
+
+Run once after exporting from ChessBase (File → Export → Export Database as PGN).
+
+Usage:
+  python -m megabase.indexer <pgn_file> [--db megabase.db] [--batch 1000]
+
+Example:
+  python -m megabase.indexer mega.pgn
+  python -m megabase.indexer mega.pgn --db /data/megabase.db
+"""
+
+import io
+import sys
+import argparse
+import sqlite3
+
+import chess.pgn
+
+
+DEFAULT_DB = "megabase.db"
+DEFAULT_BATCH = 1_000
+PROGRESS_EVERY = 10_000
+
+
+def create_schema(conn: sqlite3.Connection) -> None:
+    conn.executescript("""
+        CREATE TABLE IF NOT EXISTS games (
+            id      INTEGER PRIMARY KEY,
+            white   TEXT NOT NULL,
+            black   TEXT NOT NULL,
+            date    TEXT,
+            event   TEXT,
+            result  TEXT,
+            pgn     TEXT NOT NULL
+        );
+        CREATE INDEX IF NOT EXISTS idx_white ON games (white COLLATE NOCASE);
+        CREATE INDEX IF NOT EXISTS idx_black ON games (black COLLATE NOCASE);
+    """)
+
+
+def _pgn_text(game: chess.pgn.Game) -> str:
+    buf = io.StringIO()
+    exporter = chess.pgn.FileExporter(buf)
+    game.accept(exporter)
+    return buf.getvalue().strip()
+
+
+def build_index(pgn_path: str, db_path: str = DEFAULT_DB,
+                batch_size: int = DEFAULT_BATCH) -> int:
+    """Stream pgn_path into db_path. Returns total games indexed."""
+    conn = sqlite3.connect(db_path)
+    create_schema(conn)
+
+    total = 0
+    batch: list[tuple] = []
+
+    print(f"Indexing {pgn_path} → {db_path}", file=sys.stderr)
+
+    with open(pgn_path, encoding="utf-8", errors="replace") as fh:
+        while True:
+            try:
+                game = chess.pgn.read_game(fh)
+            except Exception:
+                continue
+            if game is None:
+                break
+
+            headers = game.headers
+            white = headers.get("White", "?")
+            black = headers.get("Black", "?")
+            if white == "?" and black == "?":
+                continue
+
+            batch.append((
+                white,
+                black,
+                headers.get("Date"),
+                headers.get("Event"),
+                headers.get("Result"),
+                _pgn_text(game),
+            ))
+
+            if len(batch) >= batch_size:
+                _flush(conn, batch)
+                total += len(batch)
+                batch = []
+                if total % PROGRESS_EVERY == 0:
+                    print(f"  {total:,} games indexed…", file=sys.stderr)
+
+    if batch:
+        _flush(conn, batch)
+        total += len(batch)
+
+    conn.close()
+    print(f"Done. {total:,} games indexed into {db_path}.", file=sys.stderr)
+    return total
+
+
+def _flush(conn: sqlite3.Connection, batch: list[tuple]) -> None:
+    conn.executemany(
+        "INSERT INTO games (white, black, date, event, result, pgn) "
+        "VALUES (?, ?, ?, ?, ?, ?)",
+        batch,
+    )
+    conn.commit()
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Index a MegaDatabase PGN export into SQLite.")
+    parser.add_argument("pgn", help="Path to exported PGN file")
+    parser.add_argument("--db", default=DEFAULT_DB, help=f"SQLite database path (default: {DEFAULT_DB})")
+    parser.add_argument("--batch", type=int, default=DEFAULT_BATCH,
+                        help=f"Insert batch size (default: {DEFAULT_BATCH})")
+    args = parser.parse_args()
+
+    build_index(args.pgn, db_path=args.db, batch_size=args.batch)
+
+
+if __name__ == "__main__":
+    main()

megabase/indexer.py

@@ -0,0 +1,84 @@
+"""
+Query a MegaDatabase SQLite index for a player's games.
+
+Usage:
+  python -m megabase.query <player_name> [--db megabase.db] [--output pgn|json] [--limit N]
+
+Examples:
+  python -m megabase.query "Kasparov, Garry"
+  python -m megabase.query "Kasparov, Garry" --output json
+  python -m megabase.query "Kasparov" --limit 50 --output json
+"""
+
+import sys
+import json
+import argparse
+import sqlite3
+
+DEFAULT_DB = "megabase.db"
+
+
+def get_player_games(name: str, db_path: str = DEFAULT_DB,
+                     limit: int | None = None) -> list[dict]:
+    """
+    Return all games where player name appears as White or Black.
+    Matching is case-insensitive and supports partial names.
+    """
+    conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
+    conn.row_factory = sqlite3.Row
+
+    pattern = f"%{name}%"
+    sql = """
+        SELECT white, black, date, event, result, pgn
+        FROM games
+        WHERE white LIKE ? COLLATE NOCASE
+           OR black LIKE ? COLLATE NOCASE
+        ORDER BY date DESC
+    """
+    params: tuple = (pattern, pattern)
+    if limit is not None:
+        sql += " LIMIT ?"
+        params = (pattern, pattern, limit)
+
+    rows = conn.execute(sql, params).fetchall()
+    conn.close()
+
+    return [dict(row) for row in rows]
+
+
+def output_pgn(games: list[dict]) -> None:
+    for game in games:
+        print(game["pgn"])
+        print()
+
+
+def output_json(games: list[dict]) -> None:
+    # Exclude raw pgn from JSON summary by default for readability;
+    # keep all fields since callers may need the pgn too.
+    print(json.dumps(games, indent=2, ensure_ascii=False))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Query a MegaDatabase SQLite index by player name.")
+    parser.add_argument("player", help="Player name (full or partial, case-insensitive)")
+    parser.add_argument("--db", default=DEFAULT_DB, help=f"SQLite database path (default: {DEFAULT_DB})")
+    parser.add_argument("--output", choices=["pgn", "json"], default="pgn",
+                        help="Output format (default: pgn)")
+    parser.add_argument("--limit", type=int, default=None,
+                        help="Maximum number of games to return")
+    args = parser.parse_args()
+
+    games = get_player_games(args.player, db_path=args.db, limit=args.limit)
+    print(f"Found {len(games)} game(s) for '{args.player}'.", file=sys.stderr)
+
+    if not games:
+        return
+
+    if args.output == "json":
+        output_json(games)
+    else:
+        output_pgn(games)
+
+
+if __name__ == "__main__":
+    main()

megabase/query.py

@@ -1,3 +1,4 @@
 requests>=2.31.0
 beautifulsoup4>=4.12.0
+python-chess>=1.10.0
 pytest>=8.0.0