update dev

.github/workflows/ci.yml

@@ -17,7 +17,17 @@ jobs:
         run: |
           python -m pip install -U pip
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          if [ $GPU_COUNT -gt 0]; then pip install -r requirements-faiss-gpu.txt; else; pip install -r requirements-faiss-cpu.txt;fi
+          if command -v nvidia-smi >/dev/null 2>&1; then
+            GPU_COUNT=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)
+          else
+            GPU_COUNT=0
+          fi
+
+          if [ "$GPU_COUNT" -gt 0 ]; then
+            pip install -r requirements-faiss-gpu.txt
+          else
+            pip install -r requirements-faiss-cpu.txt
+          fi
           if [ -f requirements-test.txt ]; then pip install -r requirements-test.txt; fi
       - name: Start MCP server
         run: |

README.md

@@ -233,14 +233,12 @@ make repack-faiss FAISS_DIR=storage/faiss_science__BAAI-bge-small-en-v1.5 OUT=st
 
 **Options:**
 - `--key`: collection key used when building the index (requires matching `--chunks-dir`)
-- `--index`: path to a FAISS index directory (or `index.faiss`) when you want to point directly at a built index
 - `--k`: number of results to return from vector database
 - `--embed-model`: the model index to query (default:`BAAI/bge-small-en-v1.5`)
 - `--ce-model`: cross encoder model (default: `cross-encoder/ms-marco-MiniLM-L-6-v2`)
 - `--chunks-dir`: directory containing the chunk JSONL written by `lc_build_index`
 - `--chunks-file`: explicit path to a chunk JSONL file (skips `--chunks-dir` lookup)
-- `--chunks-dir`: directory containing chunk metadata generated at index build time (default: `<repo>/data_processed`)
-- `--index-dir`: directory containing FAISS index folders (usually the same `--index-dir` passed to `lc_build_index`)
+- `--index-dir`: root directory containing FAISS index folders (the same parent directory passed to `lc_build_index`; defaults to `<repo>/storage`)
 
 
 **Usage**:
@@ -249,8 +247,8 @@ make repack-faiss FAISS_DIR=storage/faiss_science__BAAI-bge-small-en-v1.5 OUT=st
 # Basic query
 python src/langchain/lc_ask.py ask "What is machine learning?"
 
-# Query using an explicit index directory
-python src/langchain/lc_ask.py --index storage/faiss_science__BAAI-bge-small-en-v1.5 --question "Summarise the Higgs boson"
+# Query using a specific key with a custom index directory root
+python src/langchain/lc_ask.py --key science --index-dir /mnt/vector-storage --question "Summarise the Higgs boson"
 
 # Advanced query with options
 python src/langchain/lc_ask.py ask "Explain neural networks" --content-type technical_manual_writer --key science --k 20
@@ -799,7 +797,7 @@ over MCP:
 
 ```bash
 python -m src.cli.multi_agent "Find papers on transformers and call the time tool" \
-  --key default --mcp ./tools/mcp_server.py
+  --key default --mcp ./tools/mcp_server.py --index-dir ./storage
 ```
 
 The command registers the `rag_retrieve` tool for vector search and loads any

docs/rag-writer.1

@@ -158,7 +158,6 @@ Start with specific collection:
 [\fB\-\-task\fR \fITASK\fR]
 [\fB\-\-json\fR \fIFILE\fR]
 [\fB\-\-key\fR \fIKEY\fR]
-[\fB\-\-index\fR \fIPATH\fR]
 [\fB\-\-k\fR \fINUM\fR]
 [\fB\-\-chunks-file\fR \fIFILE\fR]
 [\fB\-\-output\fR \fIFILE\fR]
@@ -175,10 +174,7 @@ The task/prompt for the LLM
 JSON file with job specification
 .TP
 .B \-\-key \fIKEY\fR
-Collection key for RAG (default: default). One of \fB--key\fR or \fB--index\fR is required.
-.TP
-.B \-\-index \fIPATH\fR
-Path to a FAISS index directory (or \fIindex.faiss\fR) when addressing an explicit index location.
+Collection key for RAG (default: default). This key must match the embedding model used when building the index.
 .TP
 .B \-\-k \fINUM\fR
 Top-k results for retrieval (default: 30)

run_rag_verification.py

@@ -38,6 +38,7 @@
 }
 
 ANSI_RE = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]")
+_FLAG_PATTERN = re.compile(r"(?<![\w-])(-{1,2}[A-Za-z0-9][A-Za-z0-9-]*)(?![\w-])")
 
 
 def _pdf_escape(text: str) -> str:
@@ -328,6 +329,16 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
         "--multi",
         help="Optional multi-turn CLI script (defaults to multi_agent.py if present)",
     )
+    parser.add_argument(
+        "--index-key",
+        default="verification",
+        help="Storage key used when building and querying the FAISS index",
+    )
+    parser.add_argument(
+        "--embed-model",
+        default="BAAI/bge-small-en-v1.5",
+        help="Embedding model name forwarded to the asker CLI",
+    )
     parser.add_argument(
         "--topk",
         type=int,
@@ -562,11 +573,30 @@ def load_questions(path: Path, *, require_answers: bool) -> list[Question]:
     return questions
 
 
+def script_base_command(script: Path) -> list[str]:
+    """Return the appropriate Python invocation for a script path."""
+
+    script = Path(script)
+    repo_root = Path(__file__).resolve().parent
+    try:
+        relative = script.relative_to(repo_root)
+    except ValueError:
+        return [sys.executable, str(script)]
+
+    if relative.parts and relative.parts[0] == "src":
+        module = ".".join(relative.with_suffix("").parts)
+        return [sys.executable, "-m", module]
+
+    return [sys.executable, str(script)]
+
+def _advertised_flags(help_text: str) -> set[str]:
+    return {match.group(1) for match in _FLAG_PATTERN.finditer(help_text)}
+
 @lru_cache(maxsize=None)
 def script_help_text(script: Path) -> str:
     try:
         proc = subprocess.run(
-            [sys.executable, str(script), "--help"],
+            [*script_base_command(script), "--help"],
             capture_output=True,
             text=True,
             timeout=10,
@@ -579,9 +609,14 @@ def script_help_text(script: Path) -> str:
 
 def determine_flag(script: Path, candidates: Sequence[str]) -> str | None:
     help_text = script_help_text(script)
+    advertised = _advertised_flags(help_text)
+    try:
+        source = script.read_text(encoding="utf-8")
+    except (OSError, UnicodeDecodeError):
+        return None
     for flag in candidates:
-        if flag and flag in help_text:
-            return flag
+        if flag and flag in source:
+          return flag
     return None
 
 
@@ -591,8 +626,9 @@ def _script_supports_flag(
     """Return the first flag advertised by the script's help output."""
 
     help_text = script_help_text(script)
+    advertised = _advertised_flags(help_text)
     for flag in flag_candidates:
-        if flag and flag in help_text:
+        if flag and flag in advertised:
             return flag
     return None
 
@@ -605,49 +641,37 @@ def build_question_command(
     If the script advertises a question flag, use it; otherwise pass the prompt positionally.
     """
 
-    argv: List[str] = [sys.executable, str(script), *base_args]
+
+    argv: List[str] = [*script_base_command(script), *base_args]
     flag = _script_supports_flag(script, ["--question", "-q"])
+
     if flag:
         argv.extend([flag, prompt])
     else:
         argv.append(prompt)
     return argv
 
 
-def determine_builder_flags(builder: Path) -> tuple[str, str]:
-    corpus_flag = os.getenv("RAG_VERIFICATION_BUILDER_CORPUS_FLAG")
-    out_flag = os.getenv("RAG_VERIFICATION_BUILDER_OUT_FLAG")
-    if corpus_flag and out_flag:
-        return corpus_flag, out_flag
-    help_text = script_help_text(builder)
-    corpus_candidates = [
-        corpus_flag,
-        "--corpus",
-        "--docs",
-        "--source",
-        "--input",
-        "--path",
-    ]
-    out_candidates = [
-        out_flag,
-        "--out",
-        "--output",
-        "--index",
-        "--dest",
+def build_builder_command(
+    *,
+    builder: Path,
+    index_key: str,
+    pdf_dir: Path,
+    chunks_dir: Path,
+    index_dir: Path,
+) -> list[str]:
+    """Construct the lc_build_index command for the verification run."""
+
+    return [
+        *script_base_command(builder),
+        index_key,
+        "--input-dir",
+        str(pdf_dir),
+        "--chunks-dir",
+        str(chunks_dir),
+        "--index-dir",
+        str(index_dir),
     ]
-    for cand in corpus_candidates:
-        if cand and cand in help_text:
-            corpus_flag = cand
-            break
-    else:
-        corpus_flag = corpus_flag or "--corpus"
-    for cand in out_candidates:
-        if cand and cand in help_text:
-            out_flag = cand
-            break
-    else:
-        out_flag = out_flag or "--out"
-    return corpus_flag, out_flag
 
 
 def write_log(
@@ -746,8 +770,10 @@ def _read_stderr() -> None:
 def run_builder(
     *,
     builder: Path,
-    corpus_dir: Path,
+    pdf_dir: Path,
+    chunks_dir: Path,
     index_dir: Path,
+    index_key: str,
     logs_dir: Path,
     timeout: float,
     verbose: bool,
@@ -758,15 +784,16 @@ def run_builder(
     if index_dir.exists():
         shutil.rmtree(index_dir)
     index_dir.mkdir(parents=True, exist_ok=True)
-    corpus_flag, out_flag = determine_builder_flags(builder)
-    command = [
-        sys.executable,
-        str(builder),
-        corpus_flag,
-        str(corpus_dir),
-        out_flag,
-        str(index_dir),
-    ]
+    if chunks_dir.exists():
+        shutil.rmtree(chunks_dir)
+    chunks_dir.mkdir(parents=True, exist_ok=True)
+    command = build_builder_command(
+        builder=builder,
+        index_key=index_key,
+        pdf_dir=pdf_dir,
+        chunks_dir=chunks_dir,
+        index_dir=index_dir,
+    )
     if verbose:
         print("$", " ".join(shlex.quote(part) for part in command))
     log_path = logs_dir / "build_index.log"
@@ -819,9 +846,12 @@ def build_question_invocation(
     *,
     question: Question,
     index_dir: Path,
+    chunks_dir: Path,
     asker: Path,
     multi: Path | None,
     topk: int | None,
+    index_key: str,
+    embed_model: str,
 ) -> tuple[list[str], str]:
     use_multi = bool(
         multi
@@ -831,11 +861,23 @@ def build_question_invocation(
     if script is None:
         raise RuntimeError("No asker script available")
     route = "multi" if use_multi else "asker"
+    def _append_flag(script_path: Path, args: list[str], candidates: list[str], value: str) -> None:
+        flag = determine_flag(script_path, candidates)
+        if not flag:
+            # Typer-based CLIs such as multi_agent.py require running via ``-m`` to
+            # expose subcommand help. When invoked as a script from the harness the
+            # ``--help`` probe used by ``determine_flag`` fails, so fall back to the
+            # primary candidate.
+            flag = candidates[0]
+        args.extend([flag, value])
+
     if not use_multi:
-        index_flag = (
-            determine_flag(asker, ["--index-dir", "--key", "--collection"]) or "--index-dir"
-        )
-        base_args: List[str] = [index_flag, str(index_dir)]
+        base_args: List[str] = []
+        _append_flag(asker, base_args, ["--key"], index_key)
+        _append_flag(asker, base_args, ["--index-dir", "--index"], str(index_dir))
+        _append_flag(asker, base_args, ["--chunks-dir"], str(chunks_dir))
+        _append_flag(asker, base_args, ["--embed-model"], embed_model)
+
         command = build_question_command(asker, question.prompt, base_args)
         docs_flag = determine_flag(asker, ["--docs", "--doc", "--gold"])
         if docs_flag:
@@ -845,7 +887,11 @@ def build_question_invocation(
             if topk_flag:
                 command.extend([topk_flag, str(topk)])
     else:
-        command = build_question_command(script, question.prompt, [])
+        base_args: list[str] = []
+        _append_flag(multi, base_args, ["--key", "-k"], index_key)
+        _append_flag(multi, base_args, ["--index-dir", "--index"], str(index_dir))
+
+        command = build_question_command(script, question.prompt, base_args)
         if question.clarify:
             clarify_flag = determine_flag(
                 multi, ["--clarify", "--followup", "--context"]
@@ -865,9 +911,12 @@ def run_question(
     *,
     question: Question,
     index_dir: Path,
+    chunks_dir: Path,
     asker: Path,
     multi: Path | None,
     topk: int | None,
+    index_key: str,
+    embed_model: str,
     timeout: float,
     outputs_dir: Path,
     logs_dir: Path,
@@ -880,9 +929,12 @@ def run_question(
     command, route = build_question_invocation(
         question=question,
         index_dir=index_dir,
+        chunks_dir=chunks_dir,
         asker=asker,
         multi=multi,
         topk=topk,
+        index_key=index_key,
+        embed_model=embed_model,
     )
     if verbose:
         print("$", " ".join(shlex.quote(part) for part in command))
@@ -973,6 +1025,8 @@ def main(argv: Sequence[str] | None = None) -> int:
     workdir = Path(args.workdir).resolve()
     workdir.mkdir(parents=True, exist_ok=True)
     index_dir = workdir / "index_dir"
+    chunks_dir = workdir / "chunks"
+    pdf_dir = workdir / "pdf_corpus"
     logs_dir = workdir / "logs"
     logs_dir.mkdir(parents=True, exist_ok=True)
     outputs_dir = ensure_outputs_dir(workdir, args.save_outputs)
@@ -1014,11 +1068,20 @@ def main(argv: Sequence[str] | None = None) -> int:
         except Exception as exc:  # noqa: BLE001
             raise SystemExit(f"Failed to load questions: {exc}") from exc
 
+        try:
+            if pdf_dir.exists():
+                shutil.rmtree(pdf_dir)
+            prepare_pdf_corpus(corpus_dir, pdf_dir)
+        except FileNotFoundError as exc:
+            raise SystemExit(str(exc)) from exc
+
         try:
             run_builder(
                 builder=builder_path,
-                corpus_dir=corpus_dir,
+                pdf_dir=pdf_dir,
+                chunks_dir=chunks_dir,
                 index_dir=index_dir,
+                index_key=args.index_key,
                 logs_dir=logs_dir,
                 timeout=args.timeout,
                 verbose=args.verbose,
@@ -1037,9 +1100,12 @@ def main(argv: Sequence[str] | None = None) -> int:
                 stdout, route, returncode = run_question(
                     question=question,
                     index_dir=index_dir,
+                    chunks_dir=chunks_dir,
                     asker=asker_path,
                     multi=multi_path,
                     topk=args.topk,
+                    index_key=args.index_key,
+                    embed_model=args.embed_model,
                     timeout=args.timeout,
                     outputs_dir=outputs_dir,
                     logs_dir=logs_dir,