extract_ocr.py

#!/usr/bin/env python3
"""
Electoral Roll PDF Data Extraction using OCR (Tesseract + OpenCV).

Extracts voter data from Tamil Nadu electoral roll PDFs (English + Tamil pairs)
into structured CSV files. Uses PyMuPDF for image extraction, OpenCV for grid
detection, and Tesseract OCR for text recognition.

Usage:
    python extract_ocr.py AC-188                          # Process one AC directory
    python extract_ocr.py AC-188 --validate               # Test on 1 pair
    python extract_ocr.py --all --workers 4               # Process all AC directories
    python extract_ocr.py AC-188 --dry-run                # Show pairs only
    python extract_ocr.py AC-188 --part 101                # Process only Part 101
    python extract_ocr.py AC-188 --part 50-100             # Process Parts 50 to 100
    python extract_ocr.py AC-188 --part 1,5,10-20          # Specific parts and ranges
    python extract_ocr.py AC-188 --reset                   # Reset checkpoint (all parts)
    python extract_ocr.py AC-188 --reset --part 101        # Reset only Part 101
    python extract_ocr.py AC-188 --reset --part 50-100     # Reset Parts 50 to 100
    python extract_ocr.py AC-188 --limit 10                # Process only 10 pairs
    python extract_ocr.py                                  # Interactive prompt
"""

import argparse
import csv
import json
import logging
import os
import re
import shutil
import sys
import traceback
import unicodedata
from collections import Counter, defaultdict
from concurrent.futures import ProcessPoolExecutor, as_completed
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
from typing import Optional

import cv2
import fitz  # PyMuPDF
import numpy as np
from PIL import Image
import io

try:
    import pytesseract
    # Set Tesseract path for Windows (default install location)
    _tess_path = r"C:\Program Files\Tesseract-OCR\tesseract.exe"
    if os.path.exists(_tess_path):
        pytesseract.pytesseract.tesseract_cmd = _tess_path
except ImportError:
    pytesseract = None

# ----- Configuration -----

OCR_DIR = Path(__file__).parent          # ER_OCR/
INPUT_BASE = OCR_DIR / "Input" / "split_files"
OUTPUT_BASE = OCR_DIR / "output" / "split_files"
MERGED_BASE = OCR_DIR / "output" / "merged_files" / "parts"
AC_MERGED_DIR = OCR_DIR / "output" / "merged_files" / "ac"
LOG_DIR = OCR_DIR / "logs"

# Legacy paths (for backward compatibility with old checkpoint location)
_LEGACY_CHECKPOINT_DIR = OCR_DIR / "checkpoints"

CSV_HEADERS = [
    "AC No", "Part No", "Serial No", "EPIC ID",
    "Name (English)", "Name (Tamil)",
    "Relation Name (English)", "Relation Name (Tamil)",
    "Relation Type", "House No", "Age", "Gender",
    "DOB", "ContactNo",
]

# Extra columns appended when --cross-check (or --validate) is active.
# Default runs always produce the 14-column CSV above.
CSV_HEADERS_CROSSCHECK = CSV_HEADERS + ["Cross_Check", "Cross_Check_Notes"]

# ----- Logging -----

log = logging.getLogger("extract_ocr")


def setup_logging(log_file: Path = None):
    """Configure dual logging: console + file."""
    log.setLevel(logging.INFO)
    fmt = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s", datefmt="%H:%M:%S")

    console = logging.StreamHandler(sys.stdout)
    console.setFormatter(fmt)
    log.addHandler(console)

    if log_file:
        log_file.parent.mkdir(parents=True, exist_ok=True)
        fh = logging.FileHandler(log_file, encoding="utf-8")
        fh.setFormatter(fmt)
        log.addHandler(fh)


# ----- Part Range Parsing -----

def parse_part_range(part_str: str) -> set[int]:
    """Parse a --part argument into a set of part numbers.

    Accepts: single part (101), range (50-100), or comma-separated (1,5,10-20).
    """
    parts = set()
    for segment in part_str.split(","):
        segment = segment.strip()
        if "-" in segment:
            start, end = segment.split("-", 1)
            parts.update(range(int(start), int(end) + 1))
        else:
            parts.add(int(segment))
    return parts


# ----- Filename Parsing -----

FILENAME_RE = re.compile(
    r"^(\d+)-EROLLGEN-S(\d+)-(\d+)-(\w+)-FinalRoll-Revision(\d+)"
    r"-ENG-(\d+)-WI_page_(\d+)\.pdf$"
)


def parse_filename(filename: str) -> Optional[dict]:
    """Extract metadata from an English PDF filename."""
    m = FILENAME_RE.match(filename)
    if not m:
        return None
    return {
        "year": m.group(1),
        "state_code": m.group(2),
        "ac_no": m.group(3),
        "ac_abbr": m.group(4),
        "revision": m.group(5),
        "part_no": m.group(6),
        "page_no": int(m.group(7)),
        "filename": filename,
    }


def discover_tamil_files(tamil_dir: Path) -> dict[str, dict[int, Path]]:
    """
    Index all Tamil PDF files by (part_no, page_no).
    Returns dict: part_no -> {page_no: Path}
    """
    TAM_RE = re.compile(
        r"^(\d+)-EROLLGEN-S(\d+)-(\d+)-(\w+)-FinalRoll-Revision(\d+)"
        r"-TAM-(\d+)-WI_page_(\d+)\.pdf$"
    )
    index = defaultdict(dict)
    if not tamil_dir.exists():
        return index
    for f in tamil_dir.iterdir():
        if not f.name.endswith(".pdf"):
            continue
        m = TAM_RE.match(f.name)
        if m:
            part_no = m.group(6)
            page_no = int(m.group(7))
            index[part_no][page_no] = f
    return index


def discover_pairs(directory: Path) -> list[dict]:
    """Discover all English PDF files in a directory.

    Tamil matching is done later at processing time using EPIC ID matching,
    since page number offsets between English and Tamil vary by part.
    We still store all Tamil files indexed by part for efficient lookup.
    """
    eng_dir = directory / "english"
    tam_dir = directory / "tamil"

    if not eng_dir.exists():
        log.error(f"English directory not found: {eng_dir}")
        return []

    # Index all Tamil files
    tamil_index = discover_tamil_files(tam_dir)

    pairs = []
    for f in sorted(eng_dir.iterdir()):
        if not f.name.endswith(".pdf"):
            continue
        meta = parse_filename(f.name)
        if meta is None:
            log.warning(f"Skipping unrecognized filename: {f.name}")
            continue

        # Collect all Tamil pages for this part (for later EPIC matching)
        tamil_pages = tamil_index.get(meta["part_no"], {})

        pairs.append({
            "english_path": str(f),
            "tamil_pages": {pg: str(p) for pg, p in tamil_pages.items()},
            "ac_no": meta["ac_no"],
            "part_no": meta["part_no"],
            "page_no": meta["page_no"],
            "key": f.name,
        })

    return pairs


# ----- Checkpoint -----

def _checkpoint_path(dir_name: str) -> Path:
    """Return checkpoint file path inside the split_files/AC-xxx/ directory."""
    cp_path = INPUT_BASE / dir_name / "checkpoint.json"
    cp_path.parent.mkdir(parents=True, exist_ok=True)

    # Auto-migrate from legacy checkpoints/ location
    if not cp_path.exists():
        legacy_path = _LEGACY_CHECKPOINT_DIR / f"{dir_name}.json"
        if legacy_path.exists():
            shutil.copy2(legacy_path, cp_path)
            log.info(f"Migrated checkpoint from {legacy_path} to {cp_path}")

    return cp_path


def load_checkpoint(dir_name: str) -> dict:
    """Load checkpoint data for a directory."""
    cp_file = _checkpoint_path(dir_name)
    if cp_file.exists():
        with open(cp_file, "r") as f:
            return json.load(f)
    return {"processed": [], "batch_number": 0}


def save_checkpoint(dir_name: str, data: dict):
    """Save checkpoint data for a directory."""
    cp_file = _checkpoint_path(dir_name)
    with open(cp_file, "w") as f:
        json.dump(data, f, indent=2)


# ----- Image Extraction -----

def extract_image_from_pdf(pdf_path: str) -> np.ndarray:
    """Extract the embedded PNG image from a single-page PDF."""
    doc = fitz.open(pdf_path)
    try:
        page = doc[0]
        images = page.get_images(full=True)
        if not images:
            raise ValueError(f"No images found in {pdf_path}")
        xref = images[0][0]
        img_data = doc.extract_image(xref)
        img_bytes = img_data["image"]
        img_pil = Image.open(io.BytesIO(img_bytes))
        return np.array(img_pil)
    finally:
        doc.close()


# ----- Image Preprocessing -----

def preprocess_for_ocr(gray: np.ndarray, use_adaptive: bool = True) -> np.ndarray:
    """
    Preprocess a grayscale image for better OCR accuracy.
    Applies CLAHE contrast enhancement, noise removal, and adaptive thresholding.
    """
    # CLAHE contrast enhancement
    clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
    enhanced = clahe.apply(gray)

    # Noise removal
    denoised = cv2.fastNlMeansDenoising(enhanced, None, 10, 7, 21)

    if use_adaptive:
        # Adaptive thresholding handles uneven lighting/scanning
        thresh = cv2.adaptiveThreshold(
            denoised, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
            cv2.THRESH_BINARY, 31, 10
        )
    else:
        _, thresh = cv2.threshold(denoised, 150, 255, cv2.THRESH_BINARY)

    return thresh


# ----- Grid Detection -----

def detect_grid(image: np.ndarray) -> list[tuple[int, int, int, int]]:
    """
    Detect voter entry cell boundaries in the electoral roll page.

    Returns list of (x1, y1, x2, y2) tuples for each cell,
    ordered left-to-right, top-to-bottom (col-major within each row).
    """
    h, w = image.shape[:2]

    # Convert to grayscale
    if len(image.shape) == 3:
        gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
    else:
        gray = image.copy()

    # Binary threshold
    _, binary = cv2.threshold(gray, 128, 255, cv2.THRESH_BINARY_INV)

    # Multi-scale horizontal line detection — try decreasing kernel widths
    h_groups = []
    for kernel_width in [w // 4, w // 6, w // 8, w // 12]:
        h_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (kernel_width, 1))
        h_lines = cv2.morphologyEx(binary, cv2.MORPH_OPEN, h_kernel)
        h_proj = np.sum(h_lines, axis=1)
        h_positions = np.where(h_proj > w * 0.3 * 255)[0]
        h_groups = _group_positions(h_positions, min_gap=15)
        if len(h_groups) >= 2:
            break

    # If morphological detection failed, try Hough Line Transform
    if len(h_groups) < 2:
        h_groups = _detect_lines_hough(binary, h, w, direction="horizontal")

    if len(h_groups) < 2:
        log.warning("Grid detection failed: not enough horizontal lines")
        # Try contour-based detection before proportional fallback
        contour_cells = _detect_grid_contours(binary, h, w)
        if contour_cells:
            return contour_cells
        return _fallback_grid(h, w)

    # Detect vertical lines for column boundary validation
    v_groups = []
    for kernel_height in [h // 4, h // 6, h // 8]:
        v_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1, kernel_height))
        v_lines = cv2.morphologyEx(binary, cv2.MORPH_OPEN, v_kernel)
        v_proj = np.sum(v_lines, axis=0)
        v_positions = np.where(v_proj > h * 0.2 * 255)[0]
        v_groups = _group_positions(v_positions, min_gap=15)
        if len(v_groups) >= 2:
            break

    # Determine column boundaries
    col_boundaries = _detect_column_boundaries(binary, h_groups)

    # If vertical lines detected, use them to validate/replace column boundaries
    if len(v_groups) >= 4:
        # Vertical lines give us direct column boundaries
        col_boundaries = _validate_column_boundaries(v_groups, w)
    elif len(col_boundaries) < 2:
        # Fallback: assume 3 equal columns
        col_boundaries = [
            int(w * 0.02),  # left margin
            int(w * 0.34),
            int(w * 0.66),
            int(w * 0.98),  # right margin
        ]

    # Validate: electoral rolls always have 3 data columns (4 boundaries)
    col_boundaries = _enforce_three_columns(col_boundaries, w)

    # P4-1: Sanity check — if detected boundaries don't span at least 85% of page
    # width, the column detection collapsed (common on partial pages with <10 voters).
    # Fall back to proportional columns.
    col_span = col_boundaries[-1] - col_boundaries[0]
    if col_span < w * 0.85:
        log.info(f"Column boundaries span only {col_span/w:.0%} of page width — using proportional fallback")
        col_boundaries = [int(w * 0.02), int(w * 0.34), int(w * 0.66), int(w * 0.98)]

    # Build cell list
    cells = []
    for row_idx in range(len(h_groups) - 1):
        y1 = h_groups[row_idx]
        y2 = h_groups[row_idx + 1]
        for col_idx in range(len(col_boundaries) - 1):
            x1 = col_boundaries[col_idx]
            x2 = col_boundaries[col_idx + 1]
            cells.append((x1, y1, x2, y2))

    return cells


def _detect_lines_hough(binary: np.ndarray, h: int, w: int, direction: str = "horizontal") -> list[int]:
    """Detect line positions using Hough Line Transform (catches lines with gaps)."""
    lines = cv2.HoughLinesP(binary, 1, np.pi / 180, threshold=100,
                            minLineLength=w // 6 if direction == "horizontal" else h // 6,
                            maxLineGap=20)
    if lines is None:
        return []

    positions = []
    for line in lines:
        x1, y1, x2, y2 = line[0]
        if direction == "horizontal":
            # Horizontal lines: small y difference, large x span
            if abs(y2 - y1) < 10 and abs(x2 - x1) > w * 0.2:
                positions.append((y1 + y2) // 2)
        else:
            # Vertical lines: small x difference, large y span
            if abs(x2 - x1) < 10 and abs(y2 - y1) > h * 0.2:
                positions.append((x1 + x2) // 2)

    if not positions:
        return []

    return _group_positions(np.array(sorted(positions)), min_gap=15)


def _detect_grid_contours(binary: np.ndarray, h: int, w: int) -> list[tuple[int, int, int, int]]:
    """Detect grid cells using contour detection as fallback."""
    contours, _ = cv2.findContours(binary, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

    # Expected cell size
    expected_area = (w / 3) * (h / 10)
    min_area = expected_area * 0.3
    max_area = expected_area * 2.5

    cells = []
    for contour in contours:
        x, y, cw, ch = cv2.boundingRect(contour)
        area = cw * ch
        aspect = cw / ch if ch > 0 else 0

        if min_area < area < max_area and 0.5 < aspect < 4.0:
            cells.append((x, y, x + cw, y + ch))

    if len(cells) < 5:
        return []

    # Sort by y then x (row-major order)
    cells.sort(key=lambda c: (c[1], c[0]))
    return cells


def _validate_column_boundaries(v_groups: list[int], w: int) -> list[int]:
    """Use detected vertical lines to determine column boundaries."""
    # Filter to boundaries that span a reasonable portion of the page width
    # Expect: left margin, col1/col2 border, col2/col3 border, right margin
    if len(v_groups) >= 4:
        # Take the leftmost, rightmost, and 2 middle ones
        boundaries = [v_groups[0]]
        inner = v_groups[1:-1]
        if len(inner) >= 2:
            # Take the two that best split into thirds
            third = w / 3
            best_pair = None
            best_score = float("inf")
            for i in range(len(inner)):
                for j in range(i + 1, len(inner)):
                    score = abs(inner[i] - third) + abs(inner[j] - 2 * third)
                    if score < best_score:
                        best_score = score
                        best_pair = (inner[i], inner[j])
            if best_pair:
                boundaries.extend(best_pair)
        elif len(inner) == 1:
            boundaries.append(inner[0])
        boundaries.append(v_groups[-1])
        return sorted(boundaries)
    return []


def _enforce_three_columns(boundaries: list[int], w: int) -> list[int]:
    """Ensure exactly 4 boundaries (3 columns) for electoral roll pages."""
    if len(boundaries) == 4:
        return boundaries
    if len(boundaries) > 4:
        # Merge closest pair until we have 4
        while len(boundaries) > 4:
            min_gap = float("inf")
            merge_idx = 0
            for i in range(len(boundaries) - 1):
                gap = boundaries[i + 1] - boundaries[i]
                if gap < min_gap:
                    min_gap = gap
                    merge_idx = i
            merged = (boundaries[merge_idx] + boundaries[merge_idx + 1]) // 2
            boundaries = boundaries[:merge_idx] + [merged] + boundaries[merge_idx + 2:]
        return boundaries
    if len(boundaries) < 4 and len(boundaries) >= 2:
        # Split widest span until we have 4
        while len(boundaries) < 4:
            max_gap = 0
            split_idx = 0
            for i in range(len(boundaries) - 1):
                gap = boundaries[i + 1] - boundaries[i]
                if gap > max_gap:
                    max_gap = gap
                    split_idx = i
            mid = (boundaries[split_idx] + boundaries[split_idx + 1]) // 2
            boundaries = boundaries[:split_idx + 1] + [mid] + boundaries[split_idx + 1:]
        return boundaries
    # Not enough data — use default proportional
    return [int(w * 0.02), int(w * 0.34), int(w * 0.66), int(w * 0.98)]


def _detect_column_boundaries(binary: np.ndarray, h_groups: list[int]) -> list[int]:
    """Detect column boundaries by analyzing vertical density in the data rows."""
    h, w = binary.shape

    # Use the middle rows for analysis (avoid header/footer)
    if len(h_groups) >= 3:
        y_start = h_groups[1]
        y_end = h_groups[-2]
    else:
        y_start = h_groups[0]
        y_end = h_groups[-1]

    row_slice = binary[y_start:y_end, :]
    col_density = np.sum(row_slice, axis=0) / 255

    # Find low-density gaps (column separators)
    # Look for regions where density drops near zero
    threshold = 10
    gaps = []
    in_gap = False
    gap_start = 0

    for x in range(w):
        if col_density[x] < threshold:
            if not in_gap:
                gap_start = x
                in_gap = True
        else:
            if in_gap:
                gap_end = x
                gap_width = gap_end - gap_start
                if gap_width >= 3:
                    gaps.append((gap_start, gap_end))
                in_gap = False

    if in_gap:
        gaps.append((gap_start, w))

    if len(gaps) < 2:
        return []

    # First gap is left margin, last gap is right margin
    # Gaps in between are column separators
    boundaries = []
    boundaries.append(gaps[0][1])  # right edge of left margin = start of col 1

    for g in gaps[1:-1]:
        mid = (g[0] + g[1]) // 2
        boundaries.append(mid)

    boundaries.append(gaps[-1][0])  # left edge of right margin = end of last col

    return boundaries


def _group_positions(positions: np.ndarray, min_gap: int = 15) -> list[int]:
    """Group nearby pixel positions into single line positions."""
    if len(positions) == 0:
        return []
    groups = []
    current = [positions[0]]
    for p in positions[1:]:
        if p - current[-1] <= min_gap:
            current.append(p)
        else:
            groups.append(int(np.mean(current)))
            current = [p]
    groups.append(int(np.mean(current)))
    return groups


def _fallback_grid(h: int, w: int) -> list[tuple[int, int, int, int]]:
    """Fallback: return proportionally-split 3x10 grid."""
    margin_x = int(w * 0.02)
    margin_top = int(h * 0.033)
    margin_bottom = int(h * 0.03)
    content_w = w - 2 * margin_x
    content_h = h - margin_top - margin_bottom
    col_w = content_w // 3
    row_h = content_h // 10

    cells = []
    for row in range(10):
        y1 = margin_top + row * row_h
        y2 = y1 + row_h
        for col in range(3):
            x1 = margin_x + col * col_w
            x2 = x1 + col_w
            cells.append((x1, y1, x2, y2))
    return cells


# ----- Cell Content Detection -----

def _is_cell_empty(cell_img: np.ndarray, threshold: float = 0.02) -> bool:
    """
    Check if a cell image is effectively empty (no voter data) by analyzing ink density.
    Real voter cells have 5-10%+ ink coverage; empty cells have <1%.
    Crops out border 5% to exclude grid lines before measuring.
    """
    h, w = cell_img.shape[:2]
    if h < 10 or w < 10:
        return True

    # Convert to grayscale
    if len(cell_img.shape) == 3:
        gray = cv2.cvtColor(cell_img, cv2.COLOR_RGB2GRAY)
    else:
        gray = cell_img

    # Binary threshold (same as grid detection)
    _, binary = cv2.threshold(gray, 128, 255, cv2.THRESH_BINARY_INV)

    # Crop out border 5% on each side to exclude grid lines
    border_y = max(int(h * 0.05), 2)
    border_x = max(int(w * 0.05), 2)
    interior = binary[border_y:h - border_y, border_x:w - border_x]

    if interior.size == 0:
        return True

    dark_pixels = np.count_nonzero(interior)
    total_pixels = interior.size
    ink_ratio = dark_pixels / total_pixels

    # Absolute minimum: fewer than 200 dark pixels means no readable text
    if dark_pixels < 200:
        log.debug(f"Empty cell: dark_pixels={dark_pixels}, ink_ratio={ink_ratio:.4f}")
        return True

    if ink_ratio < threshold:
        log.debug(f"Empty cell: ink_ratio={ink_ratio:.4f} < {threshold}")
        return True

    return False


def _is_record_valid(record: dict) -> bool:
    """
    Check if an OCR'd record has enough valid signals to be real voter data.
    A real voter entry always has 3+ valid fields; noise from empty cells rarely
    produces 2+ valid signals simultaneously.
    """
    signals = 0
    if record.get("name_english") and len(record["name_english"]) >= 3:
        signals += 1
    if record.get("epic_id") and re.match(r"^[A-Z]{3}\d{7}$", record["epic_id"]):
        signals += 1
    if record.get("serial_no"):
        signals += 1
    if record.get("age") and record.get("gender"):
        signals += 1
    if record.get("house_no"):
        signals += 1
    return signals >= 2


def _trim_trailing_empty_rows(records: list[dict], num_cols: int = 3) -> list[dict]:
    """
    Remove records from trailing rows where no cell passed validation.
    Prevents noise from bottom empty rows from entering serial inference.
    Records must have '_cell_index' set.
    """
    if not records or num_cols < 1:
        return records

    max_cell = max(r.get("_cell_index", 0) for r in records)
    num_rows = (max_cell // num_cols) + 1

    # Find the last row that has at least one valid record
    last_valid_row = -1
    for row in range(num_rows - 1, -1, -1):
        row_start = row * num_cols
        row_end = row_start + num_cols
        row_records = [r for r in records if row_start <= r.get("_cell_index", -1) < row_end]
        if any(_is_record_valid(r) for r in row_records):
            last_valid_row = row
            break

    if last_valid_row < 0:
        return records

    # Keep records up to and including last_valid_row
    cutoff = (last_valid_row + 1) * num_cols
    trimmed = [r for r in records if r.get("_cell_index", 0) < cutoff]

    if len(trimmed) < len(records):
        log.info(f"Trimmed {len(records) - len(trimmed)} records from trailing empty rows")

    return trimmed


# ----- OCR -----

def ocr_serial_targeted(cell_img: np.ndarray) -> str:
    """
    Targeted serial number extraction from the top-left corner of a cell.
    Serial numbers appear in a small bordered box at the top-left.
    Uses multi-strategy preprocessing with voting for accuracy.
    """
    h, w = cell_img.shape[:2]
    # Serial number box is in the top-left ~25% height, ~20% width of the cell
    roi = cell_img[0:int(h * 0.25), 0:int(w * 0.20)]

    scale = 4
    upscaled = cv2.resize(roi, None, fx=scale, fy=scale, interpolation=cv2.INTER_LANCZOS4)

    if len(upscaled.shape) == 3:
        gray = cv2.cvtColor(upscaled, cv2.COLOR_RGB2GRAY)
    else:
        gray = upscaled

    config = "--psm 7 --oem 1 --dpi 300 -c tessedit_char_whitelist=0123456789#"

    # Multi-strategy: try different thresholds and pick by voting
    candidates = []

    # Strategy 1: Fixed threshold at 150 (original)
    _, thresh1 = cv2.threshold(gray, 150, 255, cv2.THRESH_BINARY)
    text = pytesseract.image_to_string(thresh1, lang="eng", config=config).strip()
    m = re.search(r"(\d{1,4})", text)
    if m:
        candidates.append(m.group(1))

    # Strategy 2: Otsu's threshold
    _, thresh2 = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
    text = pytesseract.image_to_string(thresh2, lang="eng", config=config).strip()
    m = re.search(r"(\d{1,4})", text)
    if m:
        candidates.append(m.group(1))

    # Strategy 3: Lower threshold (120) for faint text
    _, thresh3 = cv2.threshold(gray, 120, 255, cv2.THRESH_BINARY)
    text = pytesseract.image_to_string(thresh3, lang="eng", config=config).strip()
    m = re.search(r"(\d{1,4})", text)
    if m:
        candidates.append(m.group(1))

    if not candidates:
        return ""

    # Voting: if 2+ strategies agree, use that; otherwise use first result
    counts = Counter(candidates)
    best, count = counts.most_common(1)[0]
    if count >= 2:
        return best

    # No consensus — return the first candidate
    return candidates[0]


def _extract_age_gender_targeted(cell_img: np.ndarray, record: dict) -> None:
    """
    Targeted age/gender extraction from the bottom portion of a cell.
    Age and Gender labels sit at the bottom of voter cells. When full-cell OCR
    misses them (e.g., due to noise or misalignment), this re-OCRs just the
    bottom region with focused settings.
    """
    h, w = cell_img.shape[:2]
    # Bottom 25% of the cell where Age/Gender typically appears
    roi = cell_img[int(h * 0.75):, :]
    if roi.size == 0:
        return

    scale = 4
    upscaled = cv2.resize(roi, None, fx=scale, fy=scale, interpolation=cv2.INTER_LANCZOS4)

    if len(upscaled.shape) == 3:
        gray = cv2.cvtColor(upscaled, cv2.COLOR_RGB2GRAY)
    else:
        gray = upscaled

    thresh = preprocess_for_ocr(gray)
    text = pytesseract.image_to_string(thresh, lang="eng", config="--psm 6 --oem 1 --dpi 300")

    if not record["age"]:
        ag_match = AGE_GENDER_RE.search(text)
        if ag_match:
            try:
                age_val = int(ag_match.group(1))
                if 18 <= age_val <= 120:
                    record["age"] = ag_match.group(1)
            except ValueError:
                pass
            if not record["gender"]:
                record["gender"] = normalize_gender(ag_match.group(2))
        else:
            age_match = AGE_RE.search(text)
            if age_match:
                try:
                    age_val = int(age_match.group(1))
                    if 18 <= age_val <= 120:
                        record["age"] = age_match.group(1)
                except ValueError:
                    pass

    if not record["gender"]:
        gender_match = GENDER_RE.search(text)
        if gender_match:
            record["gender"] = normalize_gender(gender_match.group(1))


def ocr_cell_english(cell_img: np.ndarray) -> dict:
    """OCR a single cell from the English PDF and parse fields."""
    # Upscale 4x with Lanczos for sharper text
    scale = 4
    upscaled = cv2.resize(cell_img, None, fx=scale, fy=scale, interpolation=cv2.INTER_LANCZOS4)

    # Convert to grayscale if needed
    if len(upscaled.shape) == 3:
        gray = cv2.cvtColor(upscaled, cv2.COLOR_RGB2GRAY)
    else:
        gray = upscaled

    # Preprocessed thresholding (adaptive + CLAHE + denoising)
    thresh = preprocess_for_ocr(gray)

    # OCR with PSM 4 (single column of text) — handles the serial number box + text layout
    # better than PSM 6 which treats it as one flat block
    text = pytesseract.image_to_string(thresh, lang="eng", config="--psm 4 --oem 1 --dpi 300")

    return parse_english_text(text)


def _ocr_tamil_with_preprocessing(cell_img: np.ndarray, crop_ratio: float,
                                    use_otsu: bool = False) -> dict:
    """OCR a Tamil cell with specified crop ratio and preprocessing strategy."""
    h, w = cell_img.shape[:2]
    cropped = cell_img[0:int(h * crop_ratio), :]

    scale = 4
    upscaled = cv2.resize(cropped, None, fx=scale, fy=scale, interpolation=cv2.INTER_LANCZOS4)

    if len(upscaled.shape) == 3:
        gray = cv2.cvtColor(upscaled, cv2.COLOR_RGB2GRAY)
    else:
        gray = upscaled

    if use_otsu:
        _, thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
    else:
        thresh = preprocess_for_ocr(gray)

    text = pytesseract.image_to_string(thresh, lang="tam+eng", config="--psm 6 --oem 1 --dpi 300")
    return parse_tamil_text(text)


def ocr_cell_tamil(cell_img: np.ndarray) -> dict:
    """OCR a single cell from the Tamil PDF and extract Tamil names.
    Retries with alternative preprocessing if initial result has poor Tamil names."""
    # Primary attempt: crop bottom 15%, CLAHE preprocessing
    result = _ocr_tamil_with_preprocessing(cell_img, crop_ratio=0.85)

    # If name is valid, return immediately (no retry needed)
    if _is_valid_tamil_name(result.get("name_tamil", "")):
        return result

    # Retry with less aggressive crops and alternative preprocessing
    best = result
    best_count = _count_tamil_chars(result.get("name_tamil", ""))

    retry_configs = [
        (0.90, False),  # Less crop, same preprocessing
        (0.95, False),  # Minimal crop, same preprocessing
        (0.85, True),   # Same crop, Otsu threshold
        (0.90, True),   # Less crop, Otsu threshold
    ]

    for crop_ratio, use_otsu in retry_configs:
        attempt = _ocr_tamil_with_preprocessing(cell_img, crop_ratio, use_otsu)
        name = attempt.get("name_tamil", "")
        if _is_valid_tamil_name(name):
            count = _count_tamil_chars(name)
            if count > best_count:
                best = attempt
                best_count = count
            # If we got a good result, no need to try more
            if best_count >= 3:
                break

    return best


def ocr_epic_id_targeted(cell_img: np.ndarray) -> Optional[str]:
    """
    Targeted EPIC ID extraction from the top-right area of a cell.
    Uses character allowlist for better accuracy.
    Tries multiple ROI sizes to handle shifted cell boundaries.
    """
    h, w = cell_img.shape[:2]
    config = "--psm 7 --oem 1 --dpi 300 -c tessedit_char_whitelist=ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"

    # Try multiple ROI regions: primary tight, then progressively wider
    rois = [
        (0, int(h * 0.3), int(w * 0.4), w),         # Primary: top 30%, right 60%
        (0, int(h * 0.35), int(w * 0.3), w),         # Wider: top 35%, right 70%
        (0, int(h * 0.4), int(w * 0.2), w),          # Widest: top 40%, right 80%
    ]

    for y1, y2, x1, x2 in rois:
        roi = cell_img[y1:y2, x1:x2]
        if roi.size == 0:
            continue

        scale = 4
        upscaled = cv2.resize(roi, None, fx=scale, fy=scale, interpolation=cv2.INTER_LANCZOS4)

        if len(upscaled.shape) == 3:
            gray = cv2.cvtColor(upscaled, cv2.COLOR_RGB2GRAY)
        else:
            gray = upscaled

        thresh = preprocess_for_ocr(gray)
        text = pytesseract.image_to_string(thresh, lang="eng", config=config).strip()

        epic_id = fix_epic_id(text)
        if epic_id and re.match(r"^[A-Z]{3}\d{7}$", epic_id):
            return epic_id

    return None


def _retry_epic_id_alt_preprocess(cell_img: np.ndarray) -> Optional[str]:
    """Retry EPIC ID extraction with alternative preprocessing strategies.

    Uses different binarization and sharpening approaches that may work
    when the standard CLAHE + adaptive threshold pipeline fails.
    """
    h, w = cell_img.shape[:2]
    config = "--psm 7 --oem 1 --dpi 300 -c tessedit_char_whitelist=ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"

    # ROI: top portion where EPIC ID appears
    rois = [
        (0, int(h * 0.3), int(w * 0.4), w),
        (0, int(h * 0.35), int(w * 0.3), w),
    ]

    for y1, y2, x1, x2 in rois:
        roi = cell_img[y1:y2, x1:x2]
        if roi.size == 0:
            continue

        scale = 4
        upscaled = cv2.resize(roi, None, fx=scale, fy=scale, interpolation=cv2.INTER_LANCZOS4)

        if len(upscaled.shape) == 3:
            gray = cv2.cvtColor(upscaled, cv2.COLOR_RGB2GRAY)
        else:
            gray = upscaled

        # Strategy 1: Otsu's threshold (better for high-contrast text)
        _, otsu = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
        text = pytesseract.image_to_string(otsu, lang="eng", config=config).strip()
        epic_id = fix_epic_id(text)
        if epic_id and re.match(r"^[A-Z]{3}\d{7}$", epic_id):
            return epic_id

        # Strategy 2: Sharpen + simple threshold (helps with faint text)
        sharpen_kernel = np.array([[-1, -1, -1], [-1, 9, -1], [-1, -1, -1]])
        sharpened = cv2.filter2D(gray, -1, sharpen_kernel)
        _, sharp_thresh = cv2.threshold(sharpened, 140, 255, cv2.THRESH_BINARY)
        text = pytesseract.image_to_string(sharp_thresh, lang="eng", config=config).strip()
        epic_id = fix_epic_id(text)
        if epic_id and re.match(r"^[A-Z]{3}\d{7}$", epic_id):
            return epic_id

    return None


def _ocr_epic_id_with_confidence(cell_img: np.ndarray) -> tuple[str, float]:
    """
    Multi-strategy EPIC ID extraction with confidence scoring.
    Runs all preprocessing strategies and uses voting/confidence to pick the best.
    Returns (epic_id, confidence) where confidence is 0-100.
    """
    h, w = cell_img.shape[:2]
    config = "--psm 7 --oem 1 --dpi 300 -c tessedit_char_whitelist=ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"

    # Primary ROI: top-right area where EPIC ID appears
    roi = cell_img[0:int(h * 0.35), int(w * 0.3):w]
    if roi.size == 0:
        return ("", 0.0)

    scale = 4
    upscaled = cv2.resize(roi, None, fx=scale, fy=scale, interpolation=cv2.INTER_LANCZOS4)
    if len(upscaled.shape) == 3:
        gray = cv2.cvtColor(upscaled, cv2.COLOR_RGB2GRAY)
    else:
        gray = upscaled

    # Three preprocessing strategies
    strategies = {}

    # Strategy 1: CLAHE + adaptive threshold (default)
    thresh1 = preprocess_for_ocr(gray)
    strategies["clahe"] = thresh1

    # Strategy 2: Otsu's threshold
    _, thresh2 = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
    strategies["otsu"] = thresh2

    # Strategy 3: Sharpen + simple threshold
    sharpen_kernel = np.array([[-1, -1, -1], [-1, 9, -1], [-1, -1, -1]])
    sharpened = cv2.filter2D(gray, -1, sharpen_kernel)
    _, thresh3 = cv2.threshold(sharpened, 140, 255, cv2.THRESH_BINARY)
    strategies["sharpen"] = thresh3

    candidates = []  # list of (epic_id, confidence)

    for name, thresh in strategies.items():
        try:
            data = pytesseract.image_to_data(thresh, lang="eng", config=config,
                                              output_type=pytesseract.Output.DICT)
            words = []
            confidences = []
            for j, text_word in enumerate(data["text"]):
                text_word = text_word.strip()
                if text_word and int(data["conf"][j]) > 0:
                    words.append(text_word)
                    confidences.append(int(data["conf"][j]))

            full_text = "".join(words)
            epic_id = fix_epic_id(full_text)
            if epic_id and re.match(r"^[A-Z]{3}\d{7}$", epic_id):
                avg_conf = sum(confidences) / len(confidences) if confidences else 0
                candidates.append((epic_id, avg_conf))
        except Exception:
            continue

    if not candidates:
        return ("", 0.0)

    # If multiple strategies agree on the same EPIC, boost confidence
    epic_counts = Counter(epic for epic, _ in candidates)
    most_common_epic, count = epic_counts.most_common(1)[0]

    if count >= 2:
        # Consensus: 2+ strategies agree — high confidence
        best_conf = max(conf for epic, conf in candidates if epic == most_common_epic)
        return (most_common_epic, min(best_conf + 10, 100))
    else:
        # No consensus: return highest confidence candidate
        candidates.sort(key=lambda x: x[1], reverse=True)
        return candidates[0]


def _ocr_epic_fast(cell_img: np.ndarray) -> str:
    """
    Single-strategy EPIC ID extraction for cross-validation purposes.
    Faster than _ocr_epic_id_with_confidence (1 pass instead of 3).
    Returns matched EPIC string or "" if nothing valid found.
    """
    h, w = cell_img.shape[:2]
    roi = cell_img[0:int(h * 0.35), int(w * 0.3):w]
    if roi.size == 0:
        return ""
    scale = 4
    upscaled = cv2.resize(roi, None, fx=scale, fy=scale, interpolation=cv2.INTER_LANCZOS4)
    gray = cv2.cvtColor(upscaled, cv2.COLOR_RGB2GRAY) if len(upscaled.shape) == 3 else upscaled
    thresh = preprocess_for_ocr(gray)
    config = "--psm 7 --oem 1 --dpi 300 -c tessedit_char_whitelist=ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
    try:
        text = pytesseract.image_to_string(thresh, lang="eng", config=config).strip()
        candidate = fix_epic_id(text)
        if candidate and re.match(r"^[A-Z]{3}\d{7}$", candidate):
            return candidate
    except Exception:
        pass
    return ""


def _ocr_full_cell_english(cell_img: np.ndarray) -> str:
    """
    Quick English OCR on a full cell image.
    Used in cross-check mode to extract House No from Tamil cells
    (field labels and values are in English even on Tamil pages).
    """
    scale = 4
    upscaled = cv2.resize(cell_img, None, fx=scale, fy=scale, interpolation=cv2.INTER_LANCZOS4)
    gray = cv2.cvtColor(upscaled, cv2.COLOR_RGB2GRAY) if len(upscaled.shape) == 3 else upscaled
    thresh = preprocess_for_ocr(gray)
    try:
        return pytesseract.image_to_string(thresh, lang="eng", config="--psm 6 --oem 1 --dpi 300")
    except Exception:
        return ""


# ----- Text Parsing -----

# Regex patterns with fuzzy label matching for OCR errors
# Serial can be preceded by OCR artifacts like =, |, #, etc.
SERIAL_RE = re.compile(r"^[#=|:;\s]*(\d{1,4})\b")
EPIC_RE = re.compile(r"([A-Z]{2,4}\s?\d{6,8})")
NAME_RE = re.compile(r"(?:Name|Nama|Nane|Narne)\s*[:.|\-]?\s*(.+?)(?:\s*-\s*)?$", re.IGNORECASE)
RELATION_RE = re.compile(
    r"(Father|Husband|Mother|Other|Falher|Fathor|Fatner|Fath|Husb|Othor)['\s]*(?:s\s*)?(?:Name|Nama|Nane|Narne)?\s*[:.|\-]?\s*(.+?)(?:\s*-\s*)?$",
    re.IGNORECASE,
)
# Fallback: matches lines like "Name: Boominathan" that appear after relation type line
RELATION_NAME_ONLY_RE = re.compile(
    r"^(?:Name|Nama|Nane|Narne)\s*[:.|\-]?\s*(.+?)(?:\s*-\s*)?$",
    re.IGNORECASE,
)
HOUSE_RE = re.compile(
    r"(?:House|Housc|Hovse|Houso)\s*(?:Number|Numbe|Numoor|Numb|No)\s*[:.|\-]?\s*(.+?)$",
    re.IGNORECASE,
)
AGE_GENDER_RE = re.compile(
    r"(?:Age|Ago|Aga)\s*[:.=|\-]?\s*(\d{2,3})\s*(?:Gender|Gend|Gencer|Gander|Gerder)\s*[:.=|\-]?\s*(Male|Female|Ma\w*|Fe\w*)",
    re.IGNORECASE,
)
AGE_RE = re.compile(r"(?:Age|Ago)\s*[:.=|\-]?\s*(\d{2,3})", re.IGNORECASE)
GENDER_RE = re.compile(r"(?:Gender|Gend|Gerder)\s*[:.=|\-]?\s*(Male|Female|Ma\w*|Fe\w*)", re.IGNORECASE)

# Tamil patterns
TAMIL_NAME_RE = re.compile(r"(?:பெயர்)\s*[:.\-]?\s*(.+?)(?:\s*-\s*)?$")
TAMIL_RELATION_RE = re.compile(
    r"(?:தந்தையின்|கணவரின்|கணவர்|தாயின்|இதரர்"
    r"|கட்தையின்|கற்தையின்|க்தையின்|சந்தையின்"  # OCR misreads of தந்தையின்
    r")\s*(?:பெயர்)?\s*[:.|\-]?\s*(.+?)(?:\s*-\s*)?$"
)
# Tamil form label noise that can contaminate name fields
TAMIL_LABEL_NOISE_RE = re.compile(
    r"\d+\s*பாலினம்\s*:\s*(?:ஆண்|பெண்)"     # "28 பாலினம் : ஆண்" = Age + Gender label
    r"|பாலினம்\s*:\s*(?:ஆண்|பெண்)"            # "பாலினம் : ஆண்" = Gender label alone
    r"|வயது\s*[:：]\s*\d+"                      # "வயது : 28" = Age label
    r"|வீட்டு\s*எண்"                            # "வீட்டு எண்" = House Number label
    r"|புகைப்படம்"                               # "புகைப்படம்" = Photo label
    r"|கையொப்பம்"                               # "கையொப்பம்" = Signature label
)

# Tamil words that should never appear as standalone names
TAMIL_GENDER_LABELS = {"ஆண்", "பெண்", "ஆண", "பெண"}
TAMIL_NOISE_WORDS = {
    "வயது", "வீட்டு", "எண்", "புகைப்படம்", "பாலினம்",
    "கையொப்பம்", "available", "photo",
}


def _count_tamil_chars(text: str) -> int:
    """Count the number of Tamil Unicode characters in text."""
    return sum(1 for c in text if '\u0B80' <= c <= '\u0BFF')


def _is_valid_tamil_name(text: str) -> bool:
    """
    Validate a Tamil name: must have at least 3 Tamil characters and not be
    a gender label or other noise word.
    """
    if not text:
        return False
    stripped = text.strip()
    if stripped in TAMIL_GENDER_LABELS or stripped in TAMIL_NOISE_WORDS:
        return False
    if _count_tamil_chars(stripped) < 3:
        return False
    return True


def parse_english_text(text: str) -> dict:
    """Parse OCR text from an English cell into structured fields."""
    result = {
        "serial_no": "",
        "epic_id": "",
        "name_english": "",
        "relation_name_english": "",
        "relation_type": "",
        "house_no": "",
        "age": "",
        "gender": "",
    }

    # Clean lines: strip artifacts like "Photo", "Available", "Photo Available"
    lines = []
    for l in text.split("\n"):
        l = l.strip()
        if not l:
            continue
        # Skip pure artifact lines
        l_lower = l.lower()
        if l_lower in ("available", "photo", "photo available", "photo is available"):
            continue
        # Remove trailing "Photo" / "Available" from data lines
        l = re.sub(r"\s*Photo\s*$", "", l, flags=re.IGNORECASE).strip()
        l = re.sub(r"\s*Available\s*$", "", l, flags=re.IGNORECASE).strip()
        if l:
            lines.append(l)

    if not lines:
        return result

    # First 2 lines may have serial number and/or EPIC ID (e.g., "211 RVJ1612993")
    for check_line in lines[:2]:
        serial_match = SERIAL_RE.search(check_line)
        if serial_match:
            result["serial_no"] = serial_match.group(1)
            break

    # Look for EPIC ID in all lines (can appear on first line, or separate line)
    for line in lines:
        epic_match = EPIC_RE.search(line)
        if epic_match:
            candidate = fix_epic_id(epic_match.group(1))
            if re.match(r"^[A-Z]{3}\d{7}$", candidate):
                result["epic_id"] = candidate
                break

    # If no valid EPIC found, try with looser matching
    if not result["epic_id"]:
        for line in lines[:4]:
            epic_match = EPIC_RE.search(line)
            if epic_match:
                result["epic_id"] = fix_epic_id(epic_match.group(1))
                break

    # Parse remaining fields from all lines
    found_name = False
    found_relation_type = False
    for line in lines:
        # Name (must come before relation to avoid confusion)
        if not found_name:
            name_match = NAME_RE.search(line)
            if name_match and not RELATION_RE.search(line):
                result["name_english"] = clean_name(name_match.group(1))
                found_name = True
                continue

        # Relation (e.g., "Father Name: Meyyappan -" or "Husband Name: Ganesan")
        rel_match = RELATION_RE.search(line)
        if rel_match:
            result["relation_type"] = normalize_relation_type(rel_match.group(1))
            rel_name = clean_name(rel_match.group(2))
            # Strip any "Name:" prefix that leaked into the captured value
            rel_name = re.sub(r"^(?:Name|Nama|Nane|Narne)\s*[:.|\-]?\s*", "", rel_name, flags=re.IGNORECASE)
            result["relation_name_english"] = rel_name.strip()
            found_relation_type = True
            continue

        # Standalone relation type line (e.g., just "Father" or "Husband" on its own)
        if found_name and not result["relation_type"]:
            line_stripped = line.strip().rstrip("'s").strip()
            if re.match(r"^(Father|Husband|Mother|Other)(\s|$)", line_stripped, re.IGNORECASE):
                result["relation_type"] = normalize_relation_type(line_stripped.split()[0])
                found_relation_type = True
                continue

        # Fallback: line with just "Name: <value>" after we already found the voter name
        # This happens when OCR splits "Father Name:" across lines, producing:
        #   Line N: "Father" (or relation type alone)
        #   Line N+1: "Name: Boominathan"
        if found_name and not result["relation_name_english"]:
            rel_name_match = RELATION_NAME_ONLY_RE.search(line)
            if rel_name_match:
                result["relation_name_english"] = clean_name(rel_name_match.group(1))
                continue

        # House number
        house_match = HOUSE_RE.search(line)
        if house_match:
            house_val = house_match.group(1).strip()
            # Remove trailing "Photo" artifacts
            house_val = re.sub(r"\s*Photo.*$", "", house_val, flags=re.IGNORECASE).strip()
            # Remove location name suffixes (e.g., "20 Saraswathi Vasagasala")
            house_val = re.sub(r"\s+[A-Z][a-z]{2,}.*$", "", house_val)
            # Truncate overly long values (house numbers should be short)
            if len(house_val) > 15:
                house_val = house_val.split()[0] if house_val.split() else house_val
            # Apply systematic cleanup: confusable mapping + whitelist
            result["house_no"] = clean_house_no(house_val)
            continue

        # Age and Gender (often on same line)
        ag_match = AGE_GENDER_RE.search(line)
        if ag_match:
            age_val = ag_match.group(1)
            try:
                if 18 <= int(age_val) <= 120:
                    result["age"] = age_val
            except ValueError:
                pass
            result["gender"] = normalize_gender(ag_match.group(2))
            continue

        # Age alone
        if not result["age"]:
            age_match = AGE_RE.search(line)
            if age_match:
                age_val = age_match.group(1)
                try:
                    if 18 <= int(age_val) <= 120:
                        result["age"] = age_val
                except ValueError:
                    pass

        # Gender alone
        if not result["gender"]:
            gender_match = GENDER_RE.search(line)
            if gender_match:
                result["gender"] = normalize_gender(gender_match.group(1))

    # Last-resort age extraction: if we have gender but no age, scan all lines
    # for a bare 2-3 digit number in the valid range (18-120).
    # OCR sometimes drops the "Age:" label but still captures the number.
    if not result["age"]:
        for line in lines:
            # Skip lines already parsed as name/relation/house/EPIC
            if any(kw in line.lower() for kw in ["name", "house", "father", "husband", "mother"]):
                continue
            # Look for bare numbers that could be age
            for m in re.finditer(r"\b(\d{2,3})\b", line):
                try:
                    val = int(m.group(1))
                    if 18 <= val <= 120:
                        result["age"] = m.group(1)
                        break
                except ValueError:
                    pass
            if result["age"]:
                break

    return result


def _clean_tamil_text(text: str) -> str:
    """Clean Tamil OCR text: remove zero-width chars, label prefixes, trailing dashes, label noise, ASCII contamination."""
    # Remove zero-width non-joiner and similar invisible chars
    text = text.replace("\u200c", "").replace("\u200d", "").replace("\u200b", "")
    # Remove Tamil label prefixes that leak into the value
    text = re.sub(r"^.*பெயர்\s*[:：]?\s*", "", text)
    text = re.sub(r"^[\s:.\-|]+", "", text)
    text = text.strip().rstrip("-").strip()
    # Filter out form label noise (e.g., "28 பாலினம் : ஆண்" = Age+Gender)
    if TAMIL_LABEL_NOISE_RE.search(text):
        return ""
    # Strip ASCII contamination: remove English letters, digits, and common noise chars
    # Keep Tamil Unicode (U+0B80–U+0BFF), spaces, hyphens, periods, and other Indic scripts
    cleaned = []
    for ch in text:
        if "\u0B80" <= ch <= "\u0BFF":   # Tamil Unicode block
            cleaned.append(ch)
        elif ch in " .-'":               # Allowed punctuation
            cleaned.append(ch)
        elif ord(ch) > 127:              # Other non-ASCII (keep Tamil punctuation, etc.)
            cleaned.append(ch)
        # ASCII letters, digits, and symbols are dropped
    text = "".join(cleaned)
    # Collapse multiple spaces and clean up
    text = re.sub(r"\s{2,}", " ", text)
    text = text.strip(" .-'")
    return text


def parse_tamil_text(text: str) -> dict:
    """Parse OCR text from a Tamil cell to extract Tamil names."""
    result = {
        "serial_no": "",
        "name_tamil": "",
        "relation_name_tamil": "",
    }

    lines = [l.strip() for l in text.split("\n") if l.strip()]
    if not lines:
        return result

    # Serial number from first line
    serial_match = SERIAL_RE.search(lines[0])
    if serial_match:
        result["serial_no"] = serial_match.group(1)

    for line in lines:
        # Skip non-data lines
        l_lower = line.lower()
        if any(skip in l_lower for skip in ["available", "photo"]):
            continue
        # Skip lines that are just gender labels (bare "ஆண்" or "பெண்")
        stripped = line.strip()
        if stripped in TAMIL_GENDER_LABELS or stripped in TAMIL_NOISE_WORDS:
            continue

        # Tamil name: line containing பெயர் (name) but NOT தந்தையின்/கணவரின்/தாயின் (relation)
        name_match = TAMIL_NAME_RE.search(line)
        if name_match and not result["name_tamil"]:
            if not TAMIL_RELATION_RE.search(line):
                candidate = _clean_tamil_text(name_match.group(1))
                if _is_valid_tamil_name(candidate):
                    result["name_tamil"] = candidate
                continue

        # Tamil relation name
        rel_match = TAMIL_RELATION_RE.search(line)
        if rel_match:
            candidate = _clean_tamil_text(rel_match.group(1))
            if _is_valid_tamil_name(candidate):
                result["relation_name_tamil"] = candidate
            continue

    # Fallback for relation name: if name was found but relation regex failed,
    # look for any line after the name line that contains பெயர் (indicating a
    # relation label with garbled prefix) and extract the value after it.
    if result["name_tamil"] and not result["relation_name_tamil"]:
        found_name_line = False
        for line in lines:
            if not found_name_line:
                if result["name_tamil"] in line or TAMIL_NAME_RE.search(line):
                    found_name_line = True
                continue
            # Skip non-data lines
            if any(skip in line.lower() for skip in ["available", "photo"]):
                continue
            if TAMIL_LABEL_NOISE_RE.search(line):
                continue
            # Look for பெயர் with any garbled prefix (OCR-damaged relation label)
            rel_fallback = re.search(r"பெயர்\s*[:.\-]?\s*(.+?)(?:\s*-\s*)?$", line)
            if rel_fallback:
                value = _clean_tamil_text(rel_fallback.group(1))
                if _is_valid_tamil_name(value):
                    result["relation_name_tamil"] = value
                    break
            # Also try colon-separated value on lines with Tamil text
            colon_match = re.search(r"[:：]\s*(.+?)(?:\s*-\s*)?$", line)
            if colon_match:
                value = _clean_tamil_text(colon_match.group(1))
                if _is_valid_tamil_name(value):
                    result["relation_name_tamil"] = value
                    break

    # If regex didn't match, try positional extraction:
    # In Tamil cells, line 2 is typically name, line 3 is relation
    if not result["name_tamil"] and len(lines) >= 2:
        for line in lines[1:]:
            if any(skip in line.lower() for skip in ["available", "photo"]):
                continue
            stripped = line.strip()
            if stripped in TAMIL_GENDER_LABELS or stripped in TAMIL_NOISE_WORDS:
                continue
            # Look for lines with Tamil characters and colon separator
            colon_match = re.search(r"[:：]\s*(.+?)(?:\s*-\s*)?$", line)
            if colon_match:
                value = _clean_tamil_text(colon_match.group(1))
                # Remove label prefix like "பெயர்" if present
                value = re.sub(r"^.*பெயர்\s*[:：]?\s*", "", value).strip()
                if _is_valid_tamil_name(value):
                    if not result["name_tamil"]:
                        result["name_tamil"] = value
                    elif not result["relation_name_tamil"]:
                        result["relation_name_tamil"] = value

    return result


def fix_epic_id(raw: str) -> str:
    """Post-process EPIC ID: fix common OCR errors like O→0 in digit positions."""
    # Remove spaces within EPIC ID (OCR sometimes adds them)
    raw = raw.replace(" ", "").strip()

    # P4-6: Uppercase the entire string first — OCR sometimes outputs lowercase
    raw = raw.upper()

    # Strip leading/trailing non-alphanumeric noise (e.g., "|RVJ1234567|")
    raw = re.sub(r"^[^A-Z0-9]+", "", raw)
    raw = re.sub(r"[^A-Z0-9]+$", "", raw)

    if len(raw) < 9:
        return raw

    # Take first 10 chars
    raw = raw[:10]

    # Fix digit positions (3-9): letters → digits
    digit_fixes = {"O": "0", "I": "1", "L": "1", "S": "5", "B": "8",
                   "G": "6", "Z": "2", "D": "0", "T": "7", "Q": "9"}
    chars = list(raw)
    for i in range(3, min(10, len(chars))):
        if chars[i] in digit_fixes:
            chars[i] = digit_fixes[chars[i]]

    # Fix letter positions (0-2): digits → letters
    letter_fixes = {"0": "O", "1": "I", "5": "S", "8": "B", "6": "G"}
    for i in range(min(3, len(chars))):
        if chars[i] in letter_fixes:
            chars[i] = letter_fixes[chars[i]]

    result = "".join(chars)

    # Fix 6-digit EPIC IDs by zero-padding (e.g., RVJ297066 → RVJ0297066)
    if re.match(r"^[A-Z]{3}\d{6}$", result):
        result = result[:3] + "0" + result[3:]

    # P4-6: Fix 9-char IDs where a digit may have been dropped
    # If we have [A-Z]{2}\d{7}, try to recover the missing first letter from known prefixes
    if len(result) == 9 and re.match(r"^[A-Z]{2}\d{7}$", result):
        two_letter = result[:2]
        recovered = _recover_two_letter_prefix(two_letter)
        if recovered:
            result = recovered + result[2:]

    # P4-3: Accept any ^[A-Z]{3}\d{7}$ as valid EPIC ID.
    # Multi-constituency voters have diverse prefixes (WRC, YLJ, XKR, etc.)
    # so prefix-based correction is removed to avoid corrupting valid IDs.

    return result


# Prefix frequency table: built up during processing to recover 2-letter EPIC prefixes
_epic_prefix_counts: dict[str, int] = defaultdict(int)


def _seed_prefix_table(output_dir: Path):
    """Seed the EPIC prefix frequency table from existing output CSVs.

    This ensures prefix recovery works even when re-processing a subset of pages,
    by loading prefix data from previously extracted records.
    """
    if not output_dir.exists():
        return
    count = 0
    for csv_file in output_dir.glob("*.csv"):
        try:
            with open(csv_file, encoding="utf-8-sig") as f:
                reader = csv.DictReader(f)
                for row in reader:
                    epic = row.get("EPIC ID", "").strip()
                    if epic and re.match(r"^[A-Z]{3}\d{7}$", epic):
                        _epic_prefix_counts[epic[:3]] += 1
                        count += 1
        except Exception:
            continue
    if count:
        top = sorted(_epic_prefix_counts.items(), key=lambda x: -x[1])[:5]
        top_str = ", ".join(f"{p}:{c}" for p, c in top)
        log.info(f"Seeded EPIC prefix table from {count} existing records (top: {top_str})")


def _track_epic_prefix(epic_id: str):
    """Record a valid 3-letter EPIC prefix for later recovery of 2-letter ones."""
    if epic_id and re.match(r"^[A-Z]{3}\d{7}$", epic_id):
        _epic_prefix_counts[epic_id[:3]] += 1


def _recover_two_letter_prefix(two_letter: str) -> Optional[str]:
    """Try to recover a 3-letter prefix from a 2-letter one using known prefix frequencies.

    E.g., if 'VJ' is seen and 'RVJ' has 2000 occurrences, return 'RVJ'.
    Only returns a match if there's a single dominant candidate (>80% of matches).
    """
    if not _epic_prefix_counts:
        return None

    # Find all 3-letter prefixes that end with this 2-letter suffix
    candidates = {p: c for p, c in _epic_prefix_counts.items() if p[1:] == two_letter}
    if not candidates:
        # Also try prefixes where the 2-letter is the first two chars (dropped last letter)
        candidates = {p: c for p, c in _epic_prefix_counts.items() if p[:2] == two_letter}

    if not candidates:
        return None

    total = sum(candidates.values())
    best_prefix, best_count = max(candidates.items(), key=lambda x: x[1])

    # Only recover if the best candidate is dominant (>80% of matching prefixes)
    if best_count / total >= 0.8 and best_count >= 5:
        log.info(f"Recovered 2-letter prefix '{two_letter}' → '{best_prefix}' "
                 f"(confidence: {best_count}/{total})")
        return best_prefix

    return None


def clean_name(name: str) -> str:
    """Clean up an extracted name.

    Uses a whitelist approach: only actual letters (any script), spaces, periods,
    hyphens, and apostrophes are valid in names. Everything else is stripped.
    Uses unicodedata to distinguish real letters from symbols like «, ¢, °, ©.
    """
    # P4-5: Normalize accented characters to ASCII equivalents (e.g., é → e).
    # Tesseract sometimes produces accented chars as encoding artifacts.
    name = unicodedata.normalize("NFD", name)
    name = "".join(ch for ch in name if unicodedata.category(ch) != "Mn" or "\u0B80" <= ch <= "\u0BFF")
    # Strip "Name:" prefix that leaks from relation label splitting
    name = re.sub(r"^(?:Name|Nama|Nane|Narne)\s*[:.|\-]?\s*", "", name, flags=re.IGNORECASE)
    # Whitelist: keep only characters that are Unicode letters (category L*),
    # spaces, periods, hyphens, and apostrophes. This excludes symbols (S*),
    # punctuation (P*), digits (N*), and other non-letter categories.
    cleaned = []
    for ch in name:
        cat = unicodedata.category(ch)
        if cat.startswith("L"):       # Any letter (Latin, Tamil, etc.)
            cleaned.append(ch)
        elif ch in " .-'":            # Allowed punctuation in names
            cleaned.append(ch)
        elif cat == "Zs":             # Any Unicode space
            cleaned.append(" ")
        # Everything else (symbols, digits, punctuation) is dropped
    name = "".join(cleaned)
    # Collapse multiple spaces
    name = re.sub(r"\s{2,}", " ", name)
    # Remove leading/trailing punctuation and whitespace
    name = name.strip(" .-'")
    return name


# OCR frequently confuses visually similar Unicode symbols with ASCII characters.
# For house numbers (which are mostly digits + occasional letters like A, B, C),
# we map symbols to the digit or letter they most resemble in that context.
_HOUSE_NO_CONFUSABLES = {
    # Digit-like symbols (most house number chars are digits)
    '§': '5', '$': '5',           # § and $ look like 5 in digit context
    '°': '', 'º': '',             # degree sign -> strip (noise)
    '|': '1', '¡': '1', '¦': '1',  # pipe/inverted-! -> 1
    '²': '2', '³': '3',
    # Letter-like symbols (for house sub-parts like "64A", "83C")
    '¢': 'C', '©': 'C', '€': 'C',  # C-like
    '®': 'R',
    '@': 'A',
    # Pure noise -> strip
    '~': '', '`': '', '™': '',
    '«': '', '»': '',
    '>': '', '<': '',
}


def clean_house_no(raw: str) -> str:
    """Clean a house number using Unicode confusable mapping + whitelist.

    House numbers contain: digits, letters (A-Z for sub-parts), hyphens, slashes,
    parentheses. Anything else is an OCR artifact that should be mapped to its
    visually similar ASCII equivalent or stripped.
    """
    # Step 1: Unicode NFKC normalization (maps compatibility chars to canonical forms)
    raw = unicodedata.normalize("NFKC", raw)

    # Step 2: Apply confusable mapping for chars that survive normalization
    result = []
    for ch in raw:
        if ch in _HOUSE_NO_CONFUSABLES:
            result.append(_HOUSE_NO_CONFUSABLES[ch])
        else:
            result.append(ch)
    raw = "".join(result)

    # Step 3: Whitelist — keep only digits, letters, hyphens, slashes, parens, spaces
    raw = re.sub(r"[^a-zA-Z0-9\-/() ]", "", raw)

    # Step 4: Clean up spacing and formatting
    raw = re.sub(r"\s{2,}", " ", raw).strip()
    raw = raw.strip("-/ ")

    # Step 4b: Remove leading noise from label bleed (e.g., "'s " from "Father's")
    raw = re.sub(r"^['\"]?s\s+", "", raw)

    # Step 4c: Remove unmatched/trailing parentheses
    # e.g., "2/1999)" → "2/1999", "(3/4581-H" → "3/4581-H"
    if "(" in raw and ")" not in raw:
        raw = raw.replace("(", "")
    if ")" in raw and "(" not in raw:
        raw = raw.replace(")", "")

    # P4-4: Reject pure-digit house numbers > 4 chars — these are serial numbers
    # bleeding into the house number field from grid parsing errors.
    if re.match(r"^\d{5,}$", raw):
        return ""

    # Step 5: Fix common OCR letter-for-digit confusion in leading position
    # House numbers start with digits. A leading single letter before digits/space
    # is likely a misread digit (e.g., "S8" should be "58", "C 92" should be "92").
    # But letters AFTER digits are valid sub-parts (e.g., "83C", "64A").
    m = re.match(r"^([A-Za-z])\s*(\d.*)$", raw)
    if m:
        letter, rest = m.group(1), m.group(2)
        # Common OCR digit-for-letter misreads at start of house number
        leading_letter_to_digit = {
            'S': '5', 's': '5', 'O': '0', 'o': '0', 'I': '1', 'l': '1',
            'B': '8', 'G': '6', 'g': '6', 'Z': '2', 'z': '2', 'T': '7',
            'C': 'C',  # C could be valid if followed by space (noise prefix)
        }
        replacement = leading_letter_to_digit.get(letter.upper())
        if replacement and replacement != letter.upper():
            raw = replacement + rest
        elif letter.upper() == 'C':
            # "C 92" -> "92" (C is noise prefix); but "C3" might be valid
            if rest and rest[0] == ' ':
                raw = rest.strip()

    return raw


def normalize_relation_type(raw: str) -> str:
    """Normalize relation type from OCR text."""
    raw_lower = raw.lower().strip()
    if raw_lower.startswith("fat") or raw_lower.startswith("fal"):
        return "Father"
    elif raw_lower.startswith("hus"):
        return "Husband"
    elif raw_lower.startswith("moth"):
        return "Mother"
    return raw.strip().title()


def normalize_gender(raw: str) -> str:
    """Normalize gender from OCR text."""
    raw_lower = raw.lower().strip()
    if raw_lower.startswith("ma"):
        return "Male"
    elif raw_lower.startswith("fe"):
        return "Female"
    return raw.strip()


# ----- Page Processing -----

def _infer_serial_numbers(records: list[dict], page_base_serial: int = 0):
    """
    Infer missing serial numbers from surrounding records.
    Electoral rolls have sequential serial numbers (e.g., 211, 212, 213, ...).
    Records are in cell order (left-to-right, top-to-bottom).

    Args:
        records: list of record dicts
        page_base_serial: expected first serial number on this page
                          (calculated from page position in part)
    """
    if not records:
        return

    # First pass: convert existing serials to int
    for rec in records:
        try:
            rec["_serial_int"] = int(rec["serial_no"]) if rec["serial_no"] else None
        except ValueError:
            rec["_serial_int"] = None

    # Find records with known serials to establish the sequence
    known = [(i, rec["_serial_int"]) for i, rec in enumerate(records) if rec["_serial_int"] is not None]

    if not known:
        # No serials detected at all — leave blank rather than guessing wrong.
        # They may be filled later from Tamil page cross-validation.
        for rec in records:
            rec.pop("_serial_int", None)
        return

    # For each record with missing serial, interpolate from neighbors
    for i, rec in enumerate(records):
        if rec["_serial_int"] is not None:
            continue

        # Find nearest known before and after
        before = None
        after = None
        for ki, kval in known:
            if ki < i:
                before = (ki, kval)
            elif ki > i and after is None:
                after = (ki, kval)
                break

        if before is not None:
            # Infer: serial = before_serial + (position_diff)
            inferred = before[1] + (i - before[0])
            rec["serial_no"] = str(inferred)
            rec["_serial_int"] = inferred
        elif after is not None:
            inferred = after[1] - (after[0] - i)
            rec["serial_no"] = str(inferred)
            rec["_serial_int"] = inferred

    # Cleanup
    for rec in records:
        rec.pop("_serial_int", None)


def _dedup_serial_numbers(records: list[dict]):
    """
    Fix duplicate serial numbers. Electoral rolls have strictly sequential serials.
    When two records share the same serial, one of them has a misread serial.

    Strategy: find gaps in the sequence (missing serials) and match them to
    duplicates. The duplicate record closest to the gap position gets reassigned.
    If no gap is found, clear the second occurrence and re-infer.
    """
    if len(records) < 2:
        return

    # Build serial -> list of record indices
    serial_indices = defaultdict(list)
    for i, rec in enumerate(records):
        s = rec.get("serial_no", "")
        if s:
            serial_indices[s].append(i)

    # Find duplicates (serials appearing more than once)
    dup_serials = {s: idxs for s, idxs in serial_indices.items() if len(idxs) > 1}
    if not dup_serials:
        return

    # Find gaps in the expected sequence
    all_serials = sorted(int(s) for s in serial_indices if s)
    if len(all_serials) < 2:
        return
    expected = set(range(all_serials[0], all_serials[-1] + 1))
    actual = set(all_serials)
    gaps = sorted(expected - actual)

    # Match each gap to the nearest duplicate
    for dup_serial_str, idxs in dup_serials.items():
        dup_val = int(dup_serial_str)
        if gaps:
            # Find the gap closest to this duplicate's value
            best_gap = min(gaps, key=lambda g: abs(g - dup_val))
            # Assign the gap to the duplicate occurrence whose position is
            # closest to where the gap should be in the record sequence
            # (gaps appear where a record is "missing" in position order)
            gap_expected_pos = best_gap - all_serials[0]  # approximate index
            best_idx = min(idxs, key=lambda i: abs(i - gap_expected_pos))
            old = records[best_idx]["serial_no"]
            records[best_idx]["serial_no"] = str(best_gap)
            gaps.remove(best_gap)
            log.info(f"Reassigned duplicate serial {old} -> {best_gap} at index {best_idx}")
        else:
            # No gap to fill — clear the last duplicate and re-infer
            for idx in idxs[1:]:
                old = records[idx]["serial_no"]
                records[idx]["serial_no"] = ""
                log.info(f"Cleared duplicate serial {old} at index {idx}")
            _infer_serial_numbers(records)


def _filter_stray_records(records: list[dict]) -> list[dict]:
    """
    Correct stray records whose serial numbers are far outside the page's range.
    Each page has ~30 sequential records. If a serial is far from the median,
    it's likely an OCR misread (e.g., 253 -> 203). Instead of removing the record,
    we clear the serial so _infer_serial_numbers can re-derive it from neighbors,
    or we attempt digit-level correction when possible.
    """
    if len(records) < 3:
        return records

    serials = []
    for r in records:
        try:
            if r.get("serial_no"):
                serials.append(int(r["serial_no"]))
        except ValueError:
            pass

    if len(serials) < 3:
        return records

    sorted_serials = sorted(serials)

    # Find the longest consecutive run to use as the anchor sequence.
    # This is more robust than median — a single misread (e.g., 509→609) can
    # poison inference to create a false majority (609,610,611,612) that outvotes
    # the correct sequence (505,506,507,508).
    best_run_start = sorted_serials[0]
    best_run_len = 1
    cur_start = sorted_serials[0]
    cur_len = 1
    for j in range(1, len(sorted_serials)):
        if sorted_serials[j] == sorted_serials[j - 1] + 1:
            cur_len += 1
        else:
            if cur_len > best_run_len:
                best_run_start = cur_start
                best_run_len = cur_len
            cur_start = sorted_serials[j]
            cur_len = 1
    if cur_len > best_run_len:
        best_run_start = cur_start
        best_run_len = cur_len

    # Use the midpoint of the longest run as anchor
    anchor_serial = best_run_start + best_run_len // 2

    # Dynamic tolerance: based on record count, centered on the anchor
    tolerance = max(len(records) + 5, 20)
    min_expected = anchor_serial - tolerance
    max_expected = anchor_serial + tolerance

    log.debug(f"Stray filter: anchor={anchor_serial} (run {best_run_start}-{best_run_start+best_run_len-1}, "
              f"len={best_run_len}), tolerance=±{tolerance}")

    corrected_any = False
    for r in records:
        try:
            s = int(r.get("serial_no", "0"))
            if s == 0:
                continue
            if abs(s - anchor_serial) > tolerance:
                # Try digit-level correction: common OCR misreads
                # e.g., 203 -> 253 (0 misread as 5), 205 -> 255
                original = r["serial_no"]
                corrected = _try_correct_serial(s, min_expected, max_expected)
                if corrected is not None:
                    log.info(f"Corrected stray serial {original} -> {corrected} (anchor: {anchor_serial})")
                    r["serial_no"] = str(corrected)
                    corrected_any = True
                else:
                    # Can't correct — clear it so inference can fill it from neighbors
                    log.info(f"Cleared stray serial {original} (too far from anchor {anchor_serial})")
                    r["serial_no"] = ""
                    corrected_any = True
        except ValueError:
            pass

    # If we cleared any serials, re-run inference to fill gaps
    if corrected_any:
        _infer_serial_numbers(records)

    return records


def _try_correct_serial(stray: int, min_expected: int, max_expected: int) -> Optional[int]:
    """
    Attempt digit-level correction of a misread serial number.
    Common OCR confusions: 0<->5, 0<->6, 0<->8, 1<->7, 3<->8, 2<->7, etc.
    Returns corrected serial if exactly one substitution puts it in range, else None.
    """
    digit_confusions = {
        '0': ['5', '6', '8', '9'],
        '1': ['7', '4'],
        '2': ['7', '3'],
        '3': ['8', '5'],
        '4': ['1', '9'],
        '5': ['0', '3', '6'],
        '6': ['0', '5', '8'],
        '7': ['1', '2'],
        '8': ['0', '3', '6'],
        '9': ['0', '4'],
    }
    s = str(stray)
    candidates = []
    for i, ch in enumerate(s):
        for replacement in digit_confusions.get(ch, []):
            candidate = int(s[:i] + replacement + s[i+1:])
            if min_expected <= candidate <= max_expected:
                candidates.append(candidate)

    # Only return if there's exactly one plausible correction
    unique = list(set(candidates))
    if len(unique) == 1:
        return unique[0]
    return None


def is_non_data_page(text: str) -> bool:
    """Check if OCR text indicates a non-data page (metadata, summary, map, legend, or signature)."""
    text_upper = text.upper()

    # --- Summary pages ---
    if "SUMMARY OF ELECTORS" in text_upper:
        return True
    if "SUMMARY" in text_upper and "TOTAL" in text_upper:
        return True

    # --- Legend page (Tamil) ---
    if "E- Expired" in text or "S- Shifted" in text:
        return True
    if "Expired" in text and "Shifted" in text and "Repeated" in text:
        return True

    # --- Metadata page (English) ---
    if "ELECTORAL ROLL" in text_upper:
        return True

    # --- Metadata page (Tamil) ---
    # Tamil header: "வாக்காளர் பட்டியல்" (Electoral Roll)
    if "வாக்காளர்" in text and "பட்டியல்" in text:
        return True

    # --- Map/photos page ---
    # Both English and Tamil map pages contain these English labels
    if "NAZRI NAKSHA" in text_upper or "GOOGLE MAP VIEW" in text_upper:
        return True
    if "POLLING STATION BUILDING" in text_upper:
        return True
    if "KEY MAP VIEW" in text_upper and "CAD VIEW" in text_upper:
        return True

    # --- Signature-only page (Tamil) or near-blank page ---
    stripped = text.strip()
    if len(stripped) < 100 and "கையொப்பம்" in text:
        return True
    # Generic catch-all: page with very little text is not voter data
    if len(stripped) < 50:
        return True

    return False


def _ocr_tamil_page(tam_path: str, cross_check: bool = False) -> dict[str, dict]:
    """
    OCR a Tamil page and return a dict of {epic_id: {name_tamil, relation_name_tamil}}.
    Also returns fallback dicts under "_by_serial" (keyed by serial_no) and
    "_by_position" (keyed by cell index) for position-based last-resort matching.

    When cross_check=True, each entry also stores:
      "_cell_img"  — the raw cell numpy array (for EPIC fast re-extraction)
      "_en_text"   — English OCR text of the full cell (for House No extraction)
      "_serial_no" — serial number extracted from Tamil cell (for serial cross-check)
    """
    result_by_epic = {}
    result_by_serial = {}
    result_by_position = {}

    try:
        tam_img = extract_image_from_pdf(tam_path)
    except Exception as e:
        log.warning(f"Failed to extract Tamil image {tam_path}: {e}")
        return result_by_epic

    tam_cells = detect_grid(tam_img)
    if not tam_cells:
        return result_by_epic

    for i, (x1, y1, x2, y2) in enumerate(tam_cells):
        cell_img = tam_img[y1:y2, x1:x2]
        if cell_img.size == 0:
            continue

        # Pre-OCR empty cell detection: skip cells with no meaningful ink
        if _is_cell_empty(cell_img):
            continue

        # Tamil OCR for names
        tam_data = ocr_cell_tamil(cell_img)

        # EPIC ID extraction: targeted ROI first, full-cell fallback for edge cases
        epic_id = ocr_epic_id_targeted(cell_img)
        if not epic_id or not re.match(r"^[A-Z]{3}\d{7}$", epic_id):
            epic_id = _extract_epic_from_cell(cell_img)

        # Serial number extraction using targeted top-left ROI (fallback matching)
        serial_no = tam_data["serial_no"] or ocr_serial_targeted(cell_img)

        tam_entry = {
            "name_tamil": tam_data["name_tamil"],
            "relation_name_tamil": tam_data["relation_name_tamil"],
        }

        # When cross-check mode is active, retain extra data for validation
        if cross_check:
            tam_entry["_cell_img"] = cell_img
            tam_entry["_en_text"] = _ocr_full_cell_english(cell_img)
            tam_entry["_serial_no"] = serial_no or ""

        if epic_id and re.match(r"^[A-Z]{3}\d{7}$", epic_id):
            result_by_epic[epic_id] = tam_entry

        if serial_no:
            result_by_serial[serial_no] = tam_entry

        # Always store by position for last-resort fallback
        result_by_position[i] = tam_entry

    result_by_epic["_by_serial"] = result_by_serial
    result_by_epic["_by_position"] = result_by_position
    return result_by_epic


def _extract_epic_from_cell(cell_img: np.ndarray) -> str:
    """Extract EPIC ID from a cell image using English OCR (works on both EN and TA pages)."""
    up = cv2.resize(cell_img, None, fx=4, fy=4, interpolation=cv2.INTER_LANCZOS4)
    if len(up.shape) == 3:
        gray = cv2.cvtColor(up, cv2.COLOR_RGB2GRAY)
    else:
        gray = up
    thresh = preprocess_for_ocr(gray)
    text = pytesseract.image_to_string(thresh, lang="eng", config="--psm 6 --oem 1 --dpi 300")
    epic_match = EPIC_RE.search(text)
    if epic_match:
        return fix_epic_id(epic_match.group(1))
    return ""


def _check_tamil_page_match(tam_path: str, eng_epic_ids: set[str]) -> int:
    """Check how many EPIC IDs from a Tamil page match the English page. Returns match count."""
    try:
        tam_img = extract_image_from_pdf(tam_path)
    except Exception:
        return 0

    tam_cells = detect_grid(tam_img)
    if not tam_cells:
        return 0

    match_count = 0
    for x1, y1, x2, y2 in tam_cells[:6]:  # Check first 6 cells
        cell_img = tam_img[y1:y2, x1:x2]
        if cell_img.size == 0:
            continue
        epic = ocr_epic_id_targeted(cell_img)
        if epic and epic in eng_epic_ids:
            match_count += 1
        if match_count >= 3:
            return match_count  # Early exit: confirmed match

    return match_count


def _find_tamil_page(eng_page_no: int, eng_epic_ids: set[str], tamil_pages: dict[int, str]) -> Optional[str]:
    """
    Find the Tamil page matching an English page. Tries likely candidates first:
    1. Same page number
    2. Page number + 1 (common offset)
    3. Scan remaining pages if needed
    Uses both EPIC ID matching and relaxed threshold for better pairing.
    """
    if not tamil_pages:
        return None

    # Try likely candidates first (fast path)
    candidates = []
    for offset in [0, 1, -1]:
        pg = eng_page_no + offset
        if pg in tamil_pages:
            candidates.append((pg, tamil_pages[pg]))

    # First pass: strict match (2+ EPIC IDs)
    if eng_epic_ids:
        for pg, tam_path in candidates:
            count = _check_tamil_page_match(tam_path, eng_epic_ids)
            if count >= 2:
                return tam_path

    # Second pass: relaxed match (1 EPIC ID match for candidates)
    if eng_epic_ids:
        for pg, tam_path in candidates:
            count = _check_tamil_page_match(tam_path, eng_epic_ids)
            if count >= 1:
                return tam_path

    # Slow path: scan all Tamil pages for this part
    if eng_epic_ids:
        for pg, tam_path in sorted(tamil_pages.items()):
            if pg in [eng_page_no, eng_page_no + 1, eng_page_no - 1]:
                continue  # Already tried
            count = _check_tamil_page_match(tam_path, eng_epic_ids)
            if count >= 1:
                return tam_path

    # Last resort: use page+1 offset (the documented standard pattern:
    # "Tamil page = English page + 1"). This catches cases where EPIC ID
    # matching failed due to OCR errors on both sides.
    pg = eng_page_no + 1
    if pg in tamil_pages:
        return tamil_pages[pg]
    if eng_page_no in tamil_pages:
        return tamil_pages[eng_page_no]

    return None


def process_page(eng_path: str, tamil_pages: dict[int, str] = None, eng_page_no: int = 0,
                 cross_check: bool = False) -> list[dict]:
    """
    Process one English PDF page and find+merge its Tamil pair.

    Args:
        eng_path: Path to English PDF
        tamil_pages: Dict of {page_no: tamil_path} for this part
        eng_page_no: Page number of the English PDF (for Tamil matching hint)

    Returns list of voter record dicts.
    """
    if tamil_pages is None:
        tamil_pages = {}

    # Extract English image
    eng_img = extract_image_from_pdf(eng_path)

    # Quick check: OCR the full page to detect summary/legend pages
    if len(eng_img.shape) == 3:
        eng_gray = cv2.cvtColor(eng_img, cv2.COLOR_RGB2GRAY)
    else:
        eng_gray = eng_img
    _, eng_thresh = cv2.threshold(eng_gray, 150, 255, cv2.THRESH_BINARY)
    full_text = pytesseract.image_to_string(eng_thresh, lang="eng", config="--psm 3")

    if is_non_data_page(full_text):
        return []

    # Detect grid on English page
    eng_cells = detect_grid(eng_img)
    if not eng_cells:
        log.warning(f"No grid cells detected in {eng_path}")
        return []

    # OCR each English cell
    eng_records = []
    for i, (x1, y1, x2, y2) in enumerate(eng_cells):
        cell_img = eng_img[y1:y2, x1:x2]
        if cell_img.size == 0:
            continue

        # Pre-OCR empty cell detection: skip cells with no meaningful ink
        if _is_cell_empty(cell_img):
            continue

        record = ocr_cell_english(cell_img)

        # Targeted age/gender extraction from bottom of cell when main OCR missed them
        if not record["age"] or not record["gender"]:
            _extract_age_gender_targeted(cell_img, record)

        # Targeted serial number extraction from the top-left box (multi-strategy)
        # Always run to cross-validate the primary serial — catches digit misreads
        targeted_serial = ocr_serial_targeted(cell_img)
        if targeted_serial:
            if not record["serial_no"]:
                record["serial_no"] = targeted_serial
            elif record["serial_no"] != targeted_serial:
                # Primary and targeted disagree — prefer targeted (multi-strategy voted)
                log.debug(f"Serial cross-validate: primary={record['serial_no']} "
                          f"targeted={targeted_serial} — using targeted")
                record["serial_no"] = targeted_serial

        # Try targeted EPIC ID extraction if regex didn't find a valid one
        if not record["epic_id"] or not re.match(r"^[A-Z]{3}\d{7}$", record["epic_id"]):
            targeted_epic = ocr_epic_id_targeted(cell_img)
            if targeted_epic:
                record["epic_id"] = targeted_epic

        # Enhanced retry: alternative preprocessing for still-missing EPIC IDs
        if not record["epic_id"] or not re.match(r"^[A-Z]{3}\d{7}$", record["epic_id"]):
            retry_epic = _retry_epic_id_alt_preprocess(cell_img)
            if retry_epic:
                record["epic_id"] = retry_epic

        # Confidence-scored EPIC: if we have an EPIC, verify with multi-strategy voting
        # Only triggers when confidence may be low (helps catch digit misreads)
        epic_confidence = 100.0  # Default high confidence for clean reads
        if record["epic_id"] and re.match(r"^[A-Z]{3}\d{7}$", record["epic_id"]):
            # Run confidence check to verify the EPIC we got
            conf_epic, conf_score = _ocr_epic_id_with_confidence(cell_img)
            if conf_epic:
                epic_confidence = conf_score
                if conf_epic != record["epic_id"] and conf_score >= 70:
                    # Multi-strategy voting disagrees with initial read — prefer voted result
                    log.debug(f"EPIC confidence override: {record['epic_id']} -> {conf_epic} (conf={conf_score:.0f})")
                    record["epic_id"] = conf_epic
            else:
                epic_confidence = 50.0  # Couldn't verify — mark as uncertain
        elif not record["epic_id"]:
            # No EPIC found at all — try confidence-scored extraction as last resort
            conf_epic, conf_score = _ocr_epic_id_with_confidence(cell_img)
            if conf_epic:
                record["epic_id"] = conf_epic
                epic_confidence = conf_score

        record["_epic_confidence"] = epic_confidence

        # Track valid EPIC prefixes for 2-letter prefix recovery
        _track_epic_prefix(record.get("epic_id", ""))

        # Skip cells without enough valid signals (defense in depth after ink check)
        if not _is_record_valid(record):
            continue

        record["_cell_index"] = i
        eng_records.append(record)

    # Trim trailing empty rows before serial inference (safety net for partial pages)
    eng_records = _trim_trailing_empty_rows(eng_records)

    # Infer missing serial numbers from surrounding records
    _infer_serial_numbers(eng_records)

    # Correct stray records: fix OCR-misread serials instead of removing records
    eng_records = _filter_stray_records(eng_records)

    # Fix duplicate serial numbers (OCR misread causing two records with same serial)
    _dedup_serial_numbers(eng_records)

    if not eng_records:
        return []

    # Record count validation: each page should have ~30 records (3 cols x 10 rows)
    expected_max = 30
    actual_count = len(eng_records)
    if actual_count < expected_max - 2:
        log.warning(f"Low record count in {Path(eng_path).name}: {actual_count}/{expected_max} expected")
    elif actual_count > expected_max + 2:
        log.warning(f"High record count in {Path(eng_path).name}: {actual_count}/{expected_max} expected")

    # Find matching Tamil page using EPIC IDs
    eng_epic_ids = {r["epic_id"] for r in eng_records if r.get("epic_id")}

    tam_path = None      # set below if Tamil page is found
    position_data = {}   # keyed by cell index; used by cross-check block

    if tamil_pages:
        tam_path = _find_tamil_page(eng_page_no, eng_epic_ids, tamil_pages)
        if tam_path:
            tam_data = _ocr_tamil_page(tam_path, cross_check=cross_check)
            serial_data = tam_data.pop("_by_serial", {})
            position_data = tam_data.pop("_by_position", {})

            # First pass: merge by EPIC ID (only when EPIC confidence >= 70)
            for record in eng_records:
                epic = record.get("epic_id", "")
                epic_conf = record.get("_epic_confidence", 100.0)
                if epic and epic in tam_data and epic_conf >= 70:
                    record["name_tamil"] = tam_data[epic]["name_tamil"]
                    record["relation_name_tamil"] = tam_data[epic]["relation_name_tamil"]
                    record["_tamil_match_method"] = "epic"

            # Second pass: fill gaps using serial number matching
            for record in eng_records:
                if record.get("name_tamil"):
                    continue
                serial = record.get("serial_no", "")
                if serial and serial in serial_data:
                    record["name_tamil"] = serial_data[serial]["name_tamil"]
                    record["relation_name_tamil"] = serial_data[serial]["relation_name_tamil"]
                    record["_tamil_match_method"] = "serial"

            # Third pass: fill remaining gaps using cell position matching
            # Both EN and TA pages share the same grid layout, so cell N maps to cell N
            for record in eng_records:
                if record.get("name_tamil"):
                    continue
                cell_idx = record.get("_cell_index")
                if cell_idx is not None and cell_idx in position_data:
                    pos_entry = position_data[cell_idx]
                    if pos_entry["name_tamil"]:
                        record["name_tamil"] = pos_entry["name_tamil"]
                        record["relation_name_tamil"] = pos_entry["relation_name_tamil"]
                        record["_tamil_match_method"] = "position"

            # Cross-validate: for low-confidence EPICs matched via EPIC,
            # check if position-based match gives a different result
            for record in eng_records:
                if record.get("_tamil_match_method") != "epic":
                    continue
                epic_conf = record.get("_epic_confidence", 100.0)
                if epic_conf >= 80:
                    continue  # High confidence — trust EPIC match
                cell_idx = record.get("_cell_index")
                if cell_idx is not None and cell_idx in position_data:
                    pos_entry = position_data[cell_idx]
                    if (pos_entry["name_tamil"] and
                            pos_entry["name_tamil"] != record["name_tamil"]):
                        log.debug(f"Tamil cross-validation: EPIC match '{record['name_tamil']}' "
                                  f"vs position match '{pos_entry['name_tamil']}' "
                                  f"(epic_conf={epic_conf:.0f}) — preferring position")
                        record["name_tamil"] = pos_entry["name_tamil"]
                        record["relation_name_tamil"] = pos_entry["relation_name_tamil"]
                        record["_tamil_match_method"] = "position_crossval"

            # Cross-validate: fill serial gaps from Tamil page serials
            for record in eng_records:
                if not record.get("serial_no"):
                    epic = record.get("epic_id", "")
                    if epic and epic in tam_data:
                        for ts, td in serial_data.items():
                            if td == tam_data.get(epic):
                                record["serial_no"] = ts
                                break

    # Cross-validate English vs Tamil cell fields (only in --cross-check / --validate mode)
    if cross_check and tam_path:
        for record in eng_records:
            notes = []

            cell_idx = record.get("_cell_index")
            pos_entry = position_data.get(cell_idx, {}) if cell_idx is not None else {}

            # 1. EPIC ID cross-check: re-extract EPIC from Tamil cell top-right ROI
            tam_cell_img = pos_entry.get("_cell_img")
            if tam_cell_img is not None:
                ta_epic = _ocr_epic_fast(tam_cell_img)
                en_epic = record.get("epic_id", "")
                if ta_epic and en_epic and ta_epic != en_epic:
                    notes.append(f"EPIC mismatch EN={en_epic} TA={ta_epic}")

            # 2. House No cross-check: apply HOUSE_RE to English OCR text of Tamil cell
            en_text = pos_entry.get("_en_text", "")
            if en_text:
                house_m = HOUSE_RE.search(en_text)
                if house_m:
                    ta_house = house_m.group(1).strip()
                    en_house = record.get("house_no", "").lstrip("'")
                    if ta_house and en_house and ta_house != en_house:
                        notes.append(f"House mismatch EN={en_house} TA={ta_house}")

            # 3. Serial number cross-check
            ta_serial = pos_entry.get("_serial_no", "")
            en_serial = record.get("serial_no", "")
            if ta_serial and en_serial and ta_serial != en_serial:
                notes.append(f"Serial mismatch EN={en_serial} TA={ta_serial}")

            # 4. Name length plausibility (Tamil chars vs English chars)
            en_name = record.get("name_english", "")
            ta_name = record.get("name_tamil", "")
            if en_name and ta_name:
                ratio = len(ta_name) / max(len(en_name), 1)
                if ratio > 5.0 or ratio < 0.2:
                    notes.append(f"Name length ratio {ratio:.1f} (EN={len(en_name)} TA={len(ta_name)})")

            record["_cross_check_status"] = "REVIEW" if notes else "OK"
            record["_cross_check_notes"] = "; ".join(notes)

    # Clean up and set defaults
    for record in eng_records:
        record.pop("_cell_index", None)
        record.pop("_epic_confidence", None)
        record.pop("_tamil_match_method", None)
        if "name_tamil" not in record:
            record["name_tamil"] = ""
        if "relation_name_tamil" not in record:
            record["relation_name_tamil"] = ""
        # P4-2: Final ASCII contamination guard — ensure Tamil fields are clean
        # regardless of which code path populated them.
        if record["name_tamil"]:
            record["name_tamil"] = _clean_tamil_text(record["name_tamil"])
        if record["relation_name_tamil"]:
            record["relation_name_tamil"] = _clean_tamil_text(record["relation_name_tamil"])

    return eng_records


# ----- Validation -----

def validate_record(record: dict) -> list[str]:
    """Validate a single record. Returns list of warnings."""
    warnings = []

    epic_id = record.get("epic_id", "")
    if epic_id and not re.match(r"^[A-Z]{3}\d{7}$", epic_id):
        warnings.append(f"Invalid EPIC ID: {epic_id}")

    age = record.get("age", "")
    if age:
        try:
            age_int = int(age)
            if age_int < 18 or age_int > 120:
                warnings.append(f"Unusual age: {age}")
        except ValueError:
            warnings.append(f"Non-numeric age: {age}")

    gender = record.get("gender", "")
    if gender and gender not in ("Male", "Female"):
        warnings.append(f"Unexpected gender: {gender}")

    return warnings


# ----- CSV Output -----

def write_pair_csv(records: list[dict], output_dir: Path, ac_no: str, part_no: str,
                   eng_filename: str, cross_check: bool = False):
    """
    Write records for one page pair to a CSV file.
    Filename matches the English input PDF (with .csv extension).
    Creates a header-only file for non-data pages (summary/legend).

    When cross_check=True, appends two extra columns:
      Cross_Check       — "OK" or "REVIEW"
      Cross_Check_Notes — semicolon-separated mismatch descriptions
    """
    output_dir.mkdir(parents=True, exist_ok=True)
    # Replace .pdf extension with .csv to match input filename
    csv_name = eng_filename.replace(".pdf", ".csv")
    csv_path = output_dir / csv_name

    # Sort by serial number
    def sort_key(r):
        try:
            return int(r.get("serial_no", 0))
        except ValueError:
            return 0

    records.sort(key=sort_key)

    headers = CSV_HEADERS_CROSSCHECK if cross_check else CSV_HEADERS

    with open(csv_path, "w", newline="", encoding="utf-8-sig") as f:
        writer = csv.writer(f)
        writer.writerow(headers)
        for rec in records:
            house_no = str(rec.get("house_no", ""))
            if house_no and not house_no.startswith("'"):
                house_no = f"'{house_no}"

            row = [
                ac_no,
                part_no,
                rec.get("serial_no", ""),
                rec.get("epic_id", ""),
                rec.get("name_english", ""),
                rec.get("name_tamil", ""),
                rec.get("relation_name_english", ""),
                rec.get("relation_name_tamil", ""),
                rec.get("relation_type", ""),
                house_no,
                rec.get("age", ""),
                rec.get("gender", ""),
                "",  # DOB
                "",  # ContactNo
            ]

            if cross_check:
                row.append(rec.get("_cross_check_status", "OK"))
                row.append(rec.get("_cross_check_notes", ""))

            writer.writerow(row)

    return csv_path


# ----- Worker Function (for multiprocessing) -----

def _process_pair_worker(args: tuple) -> tuple[str, list[dict], list[str], str, str]:
    """
    Worker function for ProcessPoolExecutor.
    Returns (key, records, warnings, ac_no, part_no).
    args tuple: (eng_path, tamil_pages, key, page_no, ac_no, part_no, cross_check)
    """
    eng_path, tamil_pages, key, page_no, ac_no, part_no, cross_check = args
    warnings = []
    try:
        records = process_page(eng_path, tamil_pages, eng_page_no=page_no, cross_check=cross_check)
        for rec in records:
            warnings.extend(validate_record(rec))
        return (key, records, warnings, ac_no, part_no)
    except Exception as e:
        return (key, [], [f"ERROR processing {key}: {e}\n{traceback.format_exc()}"], ac_no, part_no)


# ----- Directory Processing -----

def process_directory(dir_name: str, workers: int = 4, validate_only: bool = False,
                      limit: int = 0, part_filter: set[int] = None, page_filter: set[int] = None,
                      cross_check: bool = False):
    """Process all unprocessed pairs in a directory."""
    directory = INPUT_BASE / dir_name
    if not directory.exists():
        log.error(f"Directory not found: {directory}")
        return

    output_dir = OUTPUT_BASE / dir_name
    output_dir.mkdir(parents=True, exist_ok=True)

    # Discover pairs
    pairs = discover_pairs(directory)
    log.info(f"Found {len(pairs)} total pairs in {dir_name}")

    if not pairs:
        return

    # Load checkpoint
    checkpoint = load_checkpoint(dir_name)
    processed_set = set(checkpoint.get("processed", []))

    # Seed EPIC prefix table from existing output CSVs for prefix recovery
    _seed_prefix_table(output_dir)

    # Filter to unprocessed (when --page is given with --validate, include already-processed pages)
    if page_filter is not None and validate_only:
        # Allow re-validating any page regardless of checkpoint
        pending = list(pairs)
    else:
        pending = [p for p in pairs if p["key"] not in processed_set]

    # Filter by part number if specified
    if part_filter:
        pending = [p for p in pending if int(p["part_no"]) in part_filter]
        log.info(f"Part filter: processing parts {sorted(part_filter)}")

    # Filter to a specific page number (useful with --validate to target a known data page)
    if page_filter is not None:
        pending = [p for p in pending if p["page_no"] in page_filter]
        log.info(f"Page filter: targeting pages {sorted(page_filter)}")

    if validate_only:
        pending = pending[:1]
        log.info("Validate mode: processing only 1 pair")
    elif limit > 0:
        pending = pending[:limit]
        log.info(f"Limit mode: processing up to {limit} pairs")

    log.info(f"Pending: {len(pending)} pairs ({len(processed_set)} already processed)")

    if not pending:
        log.info("All pairs already processed!")
        return

    start_time = datetime.now()
    total_records = 0
    total_warnings = 0
    total_files = 0
    error_count = 0

    # Process pairs — one output CSV per input pair
    if workers > 1 and len(pending) > 1 and not validate_only:
        # Multiprocessing
        work_items = [
            (p["english_path"], p.get("tamil_pages", {}), p["key"], p["page_no"],
             p["ac_no"], p["part_no"], cross_check)
            for p in pending
        ]

        with ProcessPoolExecutor(max_workers=workers) as executor:
            futures = {
                executor.submit(_process_pair_worker, item): item
                for item in work_items
            }

            for i, future in enumerate(as_completed(futures)):
                key, records, warnings, ac_no, part_no = future.result()

                for w in warnings:
                    if w.startswith("ERROR"):
                        log.error(w)
                        error_count += 1
                    else:
                        log.warning(w)
                        total_warnings += 1

                # Write per-pair CSV (even if empty — header-only for non-data pages)
                csv_path = write_pair_csv(records, output_dir, ac_no, part_no, key,
                                          cross_check=cross_check)
                total_records += len(records)
                total_files += 1

                # Update checkpoint
                processed_set.add(key)
                checkpoint["processed"] = sorted(processed_set)
                save_checkpoint(dir_name, checkpoint)

                if (i + 1) % 10 == 0 or (i + 1) == len(pending):
                    log.info(f"Progress: {i + 1}/{len(pending)} pairs, {total_records} records, {total_files} files")
    else:
        # Sequential processing
        for i, pair in enumerate(pending):
            log.info(f"Processing {i + 1}/{len(pending)}: {pair['key']}")

            try:
                records = process_page(pair["english_path"], pair.get("tamil_pages", {}),
                                       eng_page_no=pair["page_no"], cross_check=cross_check)
                for rec in records:
                    rec_warnings = validate_record(rec)
                    for w in rec_warnings:
                        log.warning(f"{pair['key']} serial {rec.get('serial_no', '?')}: {w}")
                        total_warnings += 1

                total_records += len(records)

                # Write per-pair CSV (even if empty — header-only for non-data pages)
                csv_path = write_pair_csv(records, output_dir, pair["ac_no"], pair["part_no"],
                                          pair["key"], cross_check=cross_check)
                total_files += 1
                log.info(f"  Extracted {len(records)} records -> {csv_path.name}")

                if validate_only:
                    # Print detailed output (handle Unicode on Windows)
                    import io as _io
                    out = _io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
                    out.write("\n" + "=" * 60 + "\n")
                    out.write("VALIDATION RESULTS\n")
                    out.write("=" * 60 + "\n")
                    for rec in records:
                        out.write(f"\n  Serial: {rec.get('serial_no', '?')}\n")
                        out.write(f"  EPIC ID: {rec.get('epic_id', '')}\n")
                        out.write(f"  Name (EN): {rec.get('name_english', '')}\n")
                        out.write(f"  Name (TA): {rec.get('name_tamil', '')}\n")
                        out.write(f"  Relation (EN): {rec.get('relation_name_english', '')} ({rec.get('relation_type', '')})\n")
                        out.write(f"  Relation (TA): {rec.get('relation_name_tamil', '')}\n")
                        out.write(f"  House No: {rec.get('house_no', '')}\n")
                        out.write(f"  Age: {rec.get('age', '')}  Gender: {rec.get('gender', '')}\n")
                        if cross_check:
                            status = rec.get("_cross_check_status", "N/A")
                            notes = rec.get("_cross_check_notes", "")
                            out.write(f"  Cross-Check: {status}\n")
                            if notes:
                                out.write(f"  Notes: {notes}\n")
                    out.write(f"\nTotal records: {len(records)}\n")
                    if cross_check:
                        review_count = sum(1 for r in records if r.get("_cross_check_status") == "REVIEW")
                        out.write(f"Cross-check flags: {review_count} REVIEW / {len(records)} total\n")
                    out.flush()
                    return

            except Exception as e:
                log.error(f"Failed to process {pair['key']}: {e}")
                error_count += 1

            # Update checkpoint
            processed_set.add(pair["key"])
            checkpoint["processed"] = sorted(processed_set)
            save_checkpoint(dir_name, checkpoint)

    # Summary
    log.info(f"\n{'=' * 40}")
    log.info(f"SUMMARY for {dir_name}")
    log.info(f"  Pairs processed: {len(pending)}")
    log.info(f"  Output files: {total_files}")
    log.info(f"  Records extracted: {total_records}")
    log.info(f"  Warnings: {total_warnings}")
    log.info(f"  Errors: {error_count}")
    log.info(f"  Output: {output_dir}")
    log.info(f"{'=' * 40}")

    # Write run summary JSON
    _write_run_summary(dir_name, start_time, len(pairs), total_files,
                       total_records, total_warnings, error_count)


# ----- Main -----

def discover_ac_dirs() -> list[str]:
    """Find all AC-xxx directories under INPUT_BASE that have an english/ subdirectory."""
    if not INPUT_BASE.exists():
        return []
    dirs = []
    for d in sorted(INPUT_BASE.iterdir()):
        if d.is_dir() and d.name.startswith("AC-") and (d / "english").exists():
            dirs.append(d.name)
    return dirs


def _write_run_summary(dir_name: str, start_time: datetime, total_pairs: int,
                       processed: int, total_records: int, warnings: int, errors: int):
    """Write a JSON summary file for the run."""
    end_time = datetime.now()
    duration = (end_time - start_time).total_seconds()
    summary = {
        "directory": dir_name,
        "started": start_time.isoformat(),
        "completed": end_time.isoformat(),
        "duration_seconds": round(duration, 1),
        "total_pairs": total_pairs,
        "processed": processed,
        "total_records": total_records,
        "warnings": warnings,
        "errors": errors,
    }
    LOG_DIR.mkdir(parents=True, exist_ok=True)
    ts = start_time.strftime("%Y%m%d_%H%M%S")
    summary_path = LOG_DIR / f"extract_{dir_name}_{ts}_summary.json"
    with open(summary_path, "w") as f:
        json.dump(summary, f, indent=2)
    log.info(f"Run summary saved to: {summary_path}")


def main():
    parser = argparse.ArgumentParser(
        description="Extract electoral roll data from PDFs using OCR"
    )
    parser.add_argument(
        "directory", nargs="?",
        help="Directory to process (e.g., AC-188 or legacy AC-184-Part-1-50)"
    )
    parser.add_argument(
        "--all", action="store_true",
        help="Process all discovered AC directories sequentially"
    )
    parser.add_argument(
        "--validate", action="store_true",
        help="Process only 1 pair and print detailed output"
    )
    parser.add_argument(
        "--dry-run", action="store_true",
        help="Show file pairs without processing"
    )
    parser.add_argument(
        "--reset", action="store_true",
        help="Reset checkpoint for directory"
    )
    parser.add_argument(
        "--yes", "-y", action="store_true",
        help="Skip confirmation prompts (used by web UI)"
    )
    parser.add_argument(
        "--workers", type=int, default=4,
        help="Number of parallel workers (default: 4)"
    )
    parser.add_argument(
        "--limit", type=int, default=0,
        help="Max number of pairs to process (default: 0 = all)"
    )
    parser.add_argument(
        "--part", type=str, default=None,
        help="Part number or range to process (e.g., --part 101 or --part 50-100)"
    )
    parser.add_argument(
        "--page", type=str, default=None,
        help="Page number, range, or list (e.g., 4, 1-10, 1,5,10-20). Requires --part."
    )
    parser.add_argument(
        "--cross-check", action="store_true",
        help="Cross-validate EPIC ID, House No, and serial between English and Tamil cells. "
             "Adds Cross_Check, Cross_Check_Notes, EPIC_Confidence columns to CSV output. "
             "Also auto-enabled by --validate."
    )
    args = parser.parse_args()
    cross_check_mode = args.cross_check or args.validate

    # Verify Tesseract is available
    if pytesseract is None:
        print("ERROR: pytesseract not installed. Run: pip install pytesseract")
        sys.exit(1)

    try:
        pytesseract.get_tesseract_version()
    except Exception:
        print(
            "ERROR: Tesseract OCR not found. Install it:\n"
            "  winget install UB-Mannheim.TesseractOCR\n"
            "  Then add to PATH and install Tamil language data."
        )
        sys.exit(1)

    # Determine directories to process
    if args.all:
        dirs_to_process = discover_ac_dirs()
        if not dirs_to_process:
            print(f"No AC directories found in {INPUT_BASE}")
            sys.exit(1)
    elif args.directory:
        dirs_to_process = [args.directory]
    else:
        # Interactive prompt
        available = discover_ac_dirs()
        if available:
            print(f"\nAvailable directories in {INPUT_BASE}:")
            for d in available:
                print(f"  {d}")
            print()
        dir_name = input("Please enter the Assembly Constituency directory (e.g., AC-188): ").strip()
        if not dir_name:
            print("No directory specified.")
            sys.exit(1)
        dirs_to_process = [dir_name]

    # Setup per-run log file
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    dir_label = dirs_to_process[0] if len(dirs_to_process) == 1 else "all"
    log_file = LOG_DIR / f"extract_{dir_label}_{timestamp}.log"
    setup_logging(log_file)

    for dir_name in dirs_to_process:
        directory = INPUT_BASE / dir_name

        # Parse --part filter
        part_filter = parse_part_range(args.part) if args.part else None

        if args.dry_run:
            pairs = discover_pairs(directory)
            checkpoint = load_checkpoint(dir_name)
            processed_set = set(checkpoint.get("processed", []))
            pending = [p for p in pairs if p["key"] not in processed_set]
            if part_filter:
                pending = [p for p in pending if int(p["part_no"]) in part_filter]
            print(f"\n{dir_name}: {len(pairs)} total, {len(pending)} pending"
                  + (f" (parts: {sorted(part_filter)})" if part_filter else ""))
            for p in pending[:10]:
                print(f"  ENG: {os.path.basename(p['english_path'])}")
                tam_count = len(p.get("tamil_pages", {}))
                print(f"  Tamil pages available: {tam_count}")
                print(f"  AC={p['ac_no']} Part={p['part_no']} Page={p['page_no']}")
                print()
            if len(pending) > 10:
                print(f"  ... and {len(pending) - 10} more")
            continue

        if args.reset:
            cp_file = _checkpoint_path(dir_name)
            merged_dir = MERGED_BASE / dir_name
            if part_filter:
                # Reset only specific parts
                part_labels = sorted(part_filter)
                if args.yes:
                    confirmed = True
                else:
                    try:
                        response = input(f"Reset parts {part_labels} for {dir_name}? (y/N): ")
                        confirmed = response.lower() == "y"
                    except (EOFError, OSError):
                        confirmed = True  # non-interactive — auto-confirm
                if confirmed:
                    checkpoint = load_checkpoint(dir_name)
                    old_count = len(checkpoint.get("processed", []))
                    checkpoint["processed"] = [
                        k for k in checkpoint.get("processed", [])
                        if not any(f"ENG-{p}-" in k for p in part_labels)
                    ]
                    removed = old_count - len(checkpoint["processed"])
                    save_checkpoint(dir_name, checkpoint)
                    # Remove page-level output CSVs for these parts
                    output_dir = OUTPUT_BASE / dir_name
                    if output_dir.exists():
                        for p in part_labels:
                            for f in output_dir.glob(f"*ENG-{p}-*.csv"):
                                f.unlink()
                    # Remove merged CSVs and update merge checkpoint
                    if merged_dir.exists():
                        merge_cp_path = merged_dir / "merge_checkpoint.json"
                        merge_cp = {}
                        if merge_cp_path.exists():
                            with open(merge_cp_path) as f:
                                merge_cp = json.load(f)
                        for p in part_labels:
                            for f in merged_dir.glob(f"*ENG-{p}-*.csv"):
                                f.unlink()
                            # Remove from merge checkpoint
                            if "merged_parts" in merge_cp:
                                merge_cp["merged_parts"] = [
                                    m for m in merge_cp["merged_parts"]
                                    if f"ENG-{p}-" not in m
                                ]
                        if merge_cp:
                            with open(merge_cp_path, "w") as f:
                                json.dump(merge_cp, f, indent=2)
                    # Also delete stale AC-level merged CSV
                    ac_csv = AC_MERGED_DIR / f"{dir_name}.csv"
                    if ac_csv.exists():
                        ac_csv.unlink()
                        log.info(f"Removed stale AC-level CSV: {ac_csv.name}")
                    log.info(f"Reset {removed} entries for parts {part_labels} in {dir_name} (incl. merged)")
            else:
                if args.yes:
                    confirmed = True
                else:
                    try:
                        response = input(f"Reset checkpoint for {dir_name}? (y/N): ")
                        confirmed = response.lower() == "y"
                    except (EOFError, OSError):
                        confirmed = True  # non-interactive — auto-confirm
                if confirmed:
                    if cp_file.exists():
                        cp_file.unlink()
                    output_dir = OUTPUT_BASE / dir_name
                    if output_dir.exists():
                        for f in output_dir.glob("*.csv"):
                            f.unlink()
                    # Also reset merged output
                    if merged_dir.exists():
                        for f in merged_dir.glob("*.csv"):
                            f.unlink()
                        merge_cp = merged_dir / "merge_checkpoint.json"
                        if merge_cp.exists():
                            merge_cp.unlink()
                    # Also delete AC-level merged CSV
                    ac_csv = AC_MERGED_DIR / f"{dir_name}.csv"
                    if ac_csv.exists():
                        ac_csv.unlink()
                        log.info(f"Removed AC-level CSV: {ac_csv.name}")
                    log.info(f"Reset complete for {dir_name} (incl. merged)")
            # After reset, ask whether to continue with extraction
            if args.yes:
                continue  # web UI: reset only, no extraction
            try:
                resp = input("Continue with extraction? (y/N): ")
                if resp.lower() != "y":
                    continue
            except (EOFError, OSError):
                continue  # non-interactive — stop after reset

        log.info(f"\n{'=' * 50}")
        log.info(f"Processing: {dir_name}")
        log.info(f"{'=' * 50}")

        process_directory(
            dir_name,
            workers=args.workers,
            validate_only=args.validate,
            limit=args.limit,
            part_filter=part_filter,
            page_filter=parse_part_range(args.page) if args.page else None,
            cross_check=cross_check_mode,
        )


if __name__ == "__main__":
    main()