diff --git a/.gitignore b/.gitignore index b7faf40..a599e53 100644 --- a/.gitignore +++ b/.gitignore @@ -205,3 +205,4 @@ cython_debug/ marimo/_static/ marimo/_lsp/ __marimo__/ +.DS_Store diff --git a/README.md b/README.md index 83e007d..39c21ec 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,16 @@ Skills are kept **flat**: a skill never contains another skill's `SKILL.md` nest The list below is kept in alphabetical order by skill name. -| Skill | Description | -| --- | --- | -| [`duct`](./skills/duct/SKILL.md) | Wrap any command with [con/duct](https://github.com/con/duct) to capture wall-clock time, CPU, and memory usage as structured logs, so agents and reviewers can inspect what a run actually consumed. | -| [`kya`](./skills/kya/SKILL.md) | Govern and review agents with [veldt-kya](https://github.com/veldtlabs/veldt-kya) (Know Your Agents) — risk-score, consensus-judge, and drift-check an agent, emit compliance evidence, and write a governance verdict that the `labnb` loop can break on. | -| [`labnb`](./skills/labnb/SKILL.md) | Create and maintain a concurrency-safe global lab notebook outside project roots, with startup summaries of related prior work, first-class idea capture and promotion, isolated experiment workspaces, focused companion skills, and append-only indexing across projects, investigations, and tasks. | -| [`labnb-idea`](./skills/labnb-idea/SKILL.md) | Record a promising but not-yet-implemented experiment idea in the shared lab notebook index. | -| [`labnb-promote`](./skills/labnb-promote/SKILL.md) | Promote a lab notebook idea into a concrete experiment with explicit budgets, source links, and provenance. | -| [`labnb-resume`](./skills/labnb-resume/SKILL.md) | Summarize prior ideas and experiments for a project slug, then choose whether to resume, promote, branch, or start new work. | -| [`labnb-run`](./skills/labnb-run/SKILL.md) | Create and run a concrete lab notebook experiment with isolated workspace, explicit budgets, and iterative logging. | +| Skill | Description | +|--------------------------------------------------------------| --- | +| [`duct`](./skills/duct/SKILL.md) | Wrap any command with [con/duct](https://github.com/con/duct) to capture wall-clock time, CPU, and memory usage as structured logs, so agents and reviewers can inspect what a run actually consumed. | +| [`kya`](./skills/kya/SKILL.md) | Govern and review agents with [veldt-kya](https://github.com/veldtlabs/veldt-kya) (Know Your Agents) — risk-score, consensus-judge, and drift-check an agent, emit compliance evidence, and write a governance verdict that the `labnb` loop can break on. | +| [`labnb`](./skills/labnb/SKILL.md) | Create and maintain a concurrency-safe global lab notebook outside project roots, with startup summaries of related prior work, first-class idea capture and promotion, isolated experiment workspaces, focused companion skills, and append-only indexing across projects, investigations, and tasks. | +| [`labnb-idea`](./skills/labnb-idea/SKILL.md) | Record a promising but not-yet-implemented experiment idea in the shared lab notebook index. | +| [`labnb-promote`](./skills/labnb-promote/SKILL.md) | Promote a lab notebook idea into a concrete experiment with explicit budgets, source links, and provenance. | +| [`labnb-resume`](./skills/labnb-resume/SKILL.md) | Summarize prior ideas and experiments for a project slug, then choose whether to resume, promote, branch, or start new work. | +| [`labnb-run`](./skills/labnb-run/SKILL.md) | Create and run a concrete lab notebook experiment with isolated workspace, explicit budgets, and iterative logging. | +| [`structsense-skills`](./skills/structsense-skills/SKILL.md) | StructSense Skills transforms unstructured text and PDFs into validated, ontology-grounded structured JSON using a model-agnostic extraction pipeline. | ## Installing Into An Agent diff --git a/skills/structsense-skills/.idea/.gitignore b/skills/structsense-skills/.idea/.gitignore new file mode 100644 index 0000000..b58b603 --- /dev/null +++ b/skills/structsense-skills/.idea/.gitignore @@ -0,0 +1,5 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/skills/structsense-skills/.idea/inspectionProfiles/Project_Default.xml b/skills/structsense-skills/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..ed2e837 --- /dev/null +++ b/skills/structsense-skills/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,14 @@ + + + + \ No newline at end of file diff --git a/skills/structsense-skills/.idea/inspectionProfiles/profiles_settings.xml b/skills/structsense-skills/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/skills/structsense-skills/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/skills/structsense-skills/.idea/misc.xml b/skills/structsense-skills/.idea/misc.xml new file mode 100644 index 0000000..169e448 --- /dev/null +++ b/skills/structsense-skills/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/skills/structsense-skills/.idea/modules.xml b/skills/structsense-skills/.idea/modules.xml new file mode 100644 index 0000000..2c61004 --- /dev/null +++ b/skills/structsense-skills/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/skills/structsense-skills/.idea/structsense-skills.iml b/skills/structsense-skills/.idea/structsense-skills.iml new file mode 100644 index 0000000..5459756 --- /dev/null +++ b/skills/structsense-skills/.idea/structsense-skills.iml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/skills/structsense-skills/CHANGELOG.md b/skills/structsense-skills/CHANGELOG.md new file mode 100644 index 0000000..e151a87 --- /dev/null +++ b/skills/structsense-skills/CHANGELOG.md @@ -0,0 +1,163 @@ +# Changelog + +All notable changes to `structsense-skills`. Versions follow semantic versioning. +The `version:` field in `SKILL.md` frontmatter is the source of truth; this file +records what changed between versions so you can tell which features your local +copy has. + +## [0.4.0] — 2026-06-05 (zero-hallucination policy) + +### Concept mapping policy (breaking) +- **Mapping is mandatory and tool-only.** The cascade no longer silently + skips alignment. Default cascade: local hybrid (`http://localhost:8000`, + verified at `/docs`) → BioPortal → ask user for an alternate URL → + **hard-stop** with an actionable error. The skill no longer falls back + to OLS by default (OLS lacks gene coverage); pass `--allow-ols-fallback` + to opt in. +- **`concept_mapping_provenance: "llm_knowledge"` is forbidden in canonical + output.** Any item carrying it is automatically demoted to `unmapped`, + with `alignment_method: "validation_failed"`. The item itself is + preserved (exhaustive extraction is not compromised); only the + fabricated mapping is dropped. + +### New: `scripts/iri_validation.py` +- Per-ontology regex patterns for UBERON, CL, NCBITaxon, MONDO, DOID, HP, + CHEBI, DRON, BTO, OBI, MP, HGNC, NCBIGene, MGI, UniProt, PR, GO, EFO, + NIFSTD, CIDO, plus a permissive structural fallback for unknown + ontologies (`_` OWL IRIs, identifiers.org, semanticweb.org). +- Accepts **legitimate cross-ontology mappings** (e.g. CIDO results that + reuse HP IRIs as synonyms) — the declared `ontology` field does NOT + have to match the IRI prefix. We only reject `llm_knowledge` provenance + and structurally-malformed IRIs. +- Adds `result["validation"]` with `passed` / `demoted` counts and + `demoted_by_reason` breakdown (`llm_knowledge_rejected`, + `malformed_iri`, `wrong_ontology_pattern`, `missing_ontology_id`). + +### Stats clarity +- **`stats.totals` block at the top** of every result. Top-level + `total_items`, `total_entity_mentions`, `total_key_term_mentions`, + `total_resources`, `unique_entities`, `unique_key_terms`. The human + summary shows `TOTAL ITEMS: …` as the second line after `task_type`. +- **`stats.task_type` auto-inferred** when missing — entities present → + `ner`; resources → `resource`; activity/items → `structured_extraction`. +- **`alignment.by_method`** no longer shows `{missing: N}` for items that + have `concept_mapping_provenance` set. The normalizer fills + `alignment_method` from the provenance (`tool` → `direct_tool_call`, + `skipped` → `skipped`). + +### Connecting / runtime +- **`connecting/claude-desktop.md`** added. Explains Claude Desktop's + **split execution model** (chat UI on your machine, code interpreter in + Anthropic's cloud sandbox), why your `localhost:8000` is unreachable + from a sandboxed code-interpreter call (`/home/claude/work/...` paths + are the giveaway), and how to bridge via an MCP server configured in + `claude_desktop_config.json`. + +### Stats fix verified against real numbers +Smoke-tested against the user's actual run (2331 entity mentions across +15 labels, 210 key terms, 922 tool-mapped + 1619 skipped). The new +output: +- `totals.total_items: 2541` (prominent at top) +- `task_type: "ner"` (was `null`) +- `alignment.by_method: {direct_tool_call: 922, skipped: 1409, missing: 210}` (was `{missing: 2541}`) + +## [0.3.1] — 2026-06-05 (later same day) + +### Concept-mapping (fixed) +- **API schema bug fixed.** `scripts/local_hybrid_map.py` was POSTing + `{terms: [...]}` but the real `/map/batch` endpoint takes + `{text: [{text, context?}], max_results}`. Verified against a live + deployment. The client now supports either a flat list of strings or + shaped dicts with optional per-term `context`. Also handles the real + response shape (`results` is a dict keyed by input text, score is + `final_score`). +- **`prompts/alignment-via-http.md`** added: a turnkey curl + jq pipeline + for hitting `/map/batch` directly. Use this when in Claude Code (Bash + available) but the Python client isn't loaded. +- **`SKILL.md` rule #10 strengthened**: "you MUST probe `curl + ${MAPPER_URL}/docs` before declaring the mapper unavailable." Plus a + runtime-reachability table (Claude Code → yes; claude.ai web → + no, needs MCP bridge; ChatGPT cloud → no, needs Action). + +### New scripts ported from structsense +- **`scripts/input_loader.py`** — PDF / CSV / TXT / MD ingestion. PDF chain: + GROBID → PyMuPDF (`fitz`) → pdfminer.six. CLI writes a sibling `.txt` + so subsequent NER stages have stable character offsets. +- **`scripts/task_detection.py`** — auto-detect task type (ner / resource / + structured_extraction / relation_extraction / keyphrase_extraction / …) + from a free-text description. Heuristic regex first; LLM fallback via + `llm_client`. +- **`scripts/model_context.py`** — model context-window registry (~50 model + families, longest-match wins) + `estimate_payload_tokens` + + `compute_downstream_chunk_size` for sizing alignment/judge chunks + programmatically. +- **`group_by_entity.unify_ontology_across_entities`** — when the same + surface form gets different ontology IDs from different chunks/models + (parallel extraction is the usual cause), unify on the best one + (tool-mapped > llm_knowledge > unmapped). Runs automatically inside + `attach_grouped_views`. Mentions are preserved; only the ontology + fields are normalized. + +## [0.3.0] — 2026-06-05 + +### Schema (breaking) +- **`paper_title` / `doi` removed from per-entity items.** They now live ONCE at + the top level under `source_metadata: { paper_title, doi, source_path }`. + `paper_location` (section / page / paragraph) stays per-entity because it + varies per mention. +- **`source_model` provenance** added to every entity item (HF model id, or + `llm_ner:` for the LLM extractor). +- **`entities_grouped[]`** (and `key_terms_grouped[]`) added: per-entity index + collapsing raw mentions by canonical `(entity.lower(), label)`. Each group has + `source_models[]`, `source_model_counts{}`, `consensus_count`, + `judge_score_max/avg/min`, deduplicated `sentences[]` with their + `paper_locations[]`, and slim per-occurrence `mentions[]`. +- **`stats` block** always embedded: totals, label histogram, + **by_source_model** histogram, alignment provenance, judge score buckets, + per-stage timings. + +### Pipeline +- **Post-processing normalizer** (`scripts/normalize_result.py`) added. Runs + automatically before every save, and is also exposed as a CLI so legacy + result files can be normalized in place. It lifts per-entity + `paper_title`/`doi` into `source_metadata`, tags missing `source_model`, + attaches `entities_grouped`, and computes `stats`. **Idempotent.** +- **Mapper cascade**: local hybrid (`http://localhost:8000`, verify via + `/docs`) → BioPortal → ask the user for an alternative URL → skip alignment. +- **HuggingFace NER ensemble** (`scripts/ner_models.py`): d4data, + BC5CDR-chem, NCBI-disease, BioBERT-genetic, BENT-PubMedBERT family + (Gene/Chemical/Disease/Anatomical/Cell-Type/Cell-Line/Organism/Bioprocess), + Clinical-AI-Apollo, blaze999. Selectable via `--ner-profile`: + `biomedical_broad` / `cns_cells` / `pharmacology` / `genetic` / `clinical` / + `minimal` / `all`. +- **Final filename** is `_final.json` by default. + +### NER prompts +- Three variants: **general**, **neuroscience**, **CNS-cells**. +- Each ships with explicit **EXHAUSTIVENESS** rules ("extract every occurrence, + do not deduplicate by surface form") and explicit **WRONG / RIGHT** example + blocks for the `source_metadata` shape. +- **Mask-recall pass** (`prompts/mask-recall-pass.md`) and **mask-verify pass** + (`prompts/mask-verify-pass.md`) for boosting recall and detecting label + errors. + +### Connecting (new) +- `connecting/claude-code.md` — install as a Claude Code skill (`~/.claude/skills/` + or `.claude/skills/`). +- `connecting/claude-skills.md` — publish as a hosted Anthropic Skill. +- `connecting/custom-gpt.md` — wire as an OpenAI Custom GPT (Instructions + + Knowledge files + Actions). +- `connecting/mcp-server.md` — expose as an MCP server. + +## [0.2.0] — 2026-06-04 + +### Added +- Multi-stage pipeline pattern (extractor → alignment → judge → human feedback). +- Three NER variants and resource / structured-extraction prompts. +- Ontology mapping clients: BioPortal, OLS, local hybrid. +- JSON repair, span validator, chunking helpers. +- Reference pipeline driver (`scripts/pipeline.py`). + +## [0.1.0] — 2026-06-03 + +- Initial skill scaffolding from the structsense pipeline patterns. diff --git a/skills/structsense-skills/README.md b/skills/structsense-skills/README.md new file mode 100644 index 0000000..ad07cfe --- /dev/null +++ b/skills/structsense-skills/README.md @@ -0,0 +1,425 @@ +# StructSense Skills + +**Current version: 0.3.1** (see [CHANGELOG.md](CHANGELOG.md)). + +A model-agnostic skill for **structured information extraction**: NER, research-resource extraction, and schema-driven JSON extraction, with optional ontology mapping (BioPortal / OLS / local hybrid), quality scoring, and human-in-the-loop review. + +Works with any LLM — Claude, GPT, Gemini, Pi, local Ollama / vLLM. The skill ships system prompts, JSON Schemas, and pure-Python helper scripts, and does not depend on the `structsense` library. + +> **Why this exists.** It distills the proven multi-stage pattern that the [`structsense`](https://docs.brainkb.org/structsense_overview.html) pipeline uses (extract → align → judge → human feedback) into a portable, drop-in skill. You can use the prompts directly with any chat/agent tool, or wire them into your own pipeline with the included helper scripts. + +--- + +## Directory map + +``` +structsense-skills/ +├── SKILL.md ← entry point with name + description + version frontmatter (load this first) +├── README.md ← you are here +├── CHANGELOG.md ← what changed in each version +├── connecting/ ← integration guides per platform +│ ├── claude-code.md +│ ├── claude-desktop.md +│ ├── claude-skills.md +│ ├── custom-gpt.md +│ ├── mcp-server.md +│ └── pi-dev.md +├── references/ ← progressive-disclosure documentation +│ ├── pipeline-pattern.md +│ ├── ner-extraction.md +│ ├── resource-extraction.md +│ ├── structured-extraction.md +│ ├── ontology-mapping.md +│ ├── chunking-strategy.md +│ ├── json-output-discipline.md +│ ├── model-selection.md +│ └── human-feedback.md +├── prompts/ ← copy-paste-ready system + user prompts +│ ├── extractor-ner-general.md ← Person / Org / Location / Product / Event / … +│ ├── extractor-ner-neuroscience.md ← BrainRegion / Gene / Protein / Drug / Method / … +│ ├── extractor-ner-cns-cells.md ← CellType / CellSubtype / LineageMarker / Ephys / … +│ ├── mask-recall-pass.md ← pass-2: catch mentions pass-1 missed +│ ├── mask-verify-pass.md ← per-item cloze label check +│ ├── extractor-resource.md ← Model / Dataset / Tool / Benchmark / … +│ ├── extractor-structured.md ← user-supplied JSON Schema +│ ├── alignment.md ← ontology mapping +│ ├── judge.md ← per-item quality scoring +│ └── humanfeedback.md ← apply reviewer edits +├── schemas/ ← JSON Schemas for outputs (drop into structured-outputs APIs) +│ ├── ner-output.schema.json +│ ├── resource-output.schema.json +│ ├── aligned-item.schema.json +│ └── judged-item.schema.json +├── scripts/ ← pure-Python runnable helpers +│ ├── chunking.py +│ ├── json_repair.py +│ ├── span_validator.py +│ ├── mask_pass.py +│ ├── bioportal_map.py +│ ├── ols_map.py +│ ├── local_hybrid_map.py +│ ├── llm_client.py ← OpenAI / OpenRouter / Anthropic / Ollama / Gemini +│ ├── ner_models.py ← HF biomedical NER ensemble (d4data, BC5CDR, NCBI-disease, +│ │ BioBERT-genetic, BENT-PubMedBERT family, …) + provenance +│ ├── group_by_entity.py ← merge mentions into entities_grouped[] (per-entity index) +│ ├── stats.py ← totals, label histogram, by-source-model breakdown, +│ │ alignment provenance, judge score buckets, timings +│ ├── normalize_result.py ← idempotent post-processor. Lifts paper_title/doi to +│ │ source_metadata, tags source_model, attaches grouped +│ │ + stats. Runs automatically in pipeline.py; also CLI. +│ ├── input_loader.py ← PDF/CSV/TXT ingestion with GROBID → PyMuPDF → +│ │ pdfminer fallback chain. CLI writes .txt. +│ ├── task_detection.py ← auto-detect task type (ner/resource/structured/…) +│ │ from a free-text description. Heuristic + LLM. +│ ├── model_context.py ← model context-window registry + downstream +│ │ chunk-size math. +│ └── pipeline.py ← reference driver: ensemble + LLM extract → mask-recall → +│ align (cascade) → judge → normalize → group → stats +└── examples/ ← worked end-to-end examples + ├── ner-example.md + ├── resource-example.md + └── reproschema-example.md +``` + +The entry point an LLM should load is **`SKILL.md`**. Everything else loads on demand from there. + +--- + +## Using this skill (platform-specific setup) + +Different ways to wire this skill into the tool you actually use day-to-day. Pick the one that matches your workflow: + +| You want to use it with… | Read | +|---|---| +| **Claude Code** (CLI) | [connecting/claude-code.md](connecting/claude-code.md) — drop into `~/.claude/skills/` (user-global) or `.claude/skills/` (per-project). Auto-discovered. | +| **Pi** (CLI, pi.dev) | [connecting/pi-dev.md](connecting/pi-dev.md) — drop into `~/.pi/agent/skills/`, `~/.agents/skills/`, or `.pi/skills/`. Native Agent Skills support; runs the pipeline via its built-in `bash` tool. | +| **Claude Desktop** | [connecting/claude-desktop.md](connecting/claude-desktop.md) — split execution model; bridge `localhost` services via MCP. | +| **Hosted Claude Skills** on claude.ai / Claude Agent SDK | [connecting/claude-skills.md](connecting/claude-skills.md) | +| **ChatGPT Custom GPT** | [connecting/custom-gpt.md](connecting/custom-gpt.md) — Instructions + Knowledge files + (optional) server-side Action calling the pipeline. | +| **MCP-aware clients** (Claude Code, Cursor, ChatGPT desktop, custom agents) | [connecting/mcp-server.md](connecting/mcp-server.md) — expose `pipeline.py` as an MCP server. | +| Python script / direct API calls | The "Usage with the Anthropic SDK" / "Usage with OpenAI" sections below. | + +**Got legacy output that has `paper_title`/`doi` on every entity, missing `source_metadata`, or no `entities_grouped`?** Don't edit by hand. Run the normalizer: + +```bash +python -m scripts.normalize_result paper_final.json \ + --input paper.txt \ + --llm-model openrouter/anthropic/claude-sonnet-4-6 +``` + +It's **idempotent** — safe to run on already-canonical files. It also runs automatically as the last step in every `scripts/pipeline.py` invocation, so any new run produces the canonical shape regardless of what the LLM emitted. + +## Quick start + +### 1. Decide what you want to extract + +| User intent | Prompt | +|---|---| +| Entities + key terms from general text | `prompts/extractor-ner-general.md` | +| Entities + key terms from neuroscience text | `prompts/extractor-ner-neuroscience.md` | +| CNS cell-typing extraction (atlases, patch-seq, scRNA-seq) | `prompts/extractor-ner-cns-cells.md` | +| Pull tools / datasets / models / benchmarks from a paper | `prompts/extractor-resource.md` | +| Convert a PDF to a target JSON schema (e.g. ReproSchema) | `prompts/extractor-structured.md` | + +### 2. Run pass-1, then pass-2 (mask-recall) for exhaustive yield + +For NER, **always run mask-recall on top of pass-1** unless cost is critical. Typical recovery on neuroscience text: **+30–80% mentions**. See `references/ner-extraction.md` → "Two-pass strategy: mask-mode". + +### 3. (Optional) align, judge, review + +| Stage | Prompt | +|---|---| +| Ontology alignment | `prompts/alignment.md` (or skip the LLM and call the mapping helper directly — see `scripts/bioportal_map.py`, `ols_map.py`, `local_hybrid_map.py`) | +| Quality scoring | `prompts/judge.md` | +| Human review | `prompts/humanfeedback.md` | + +--- + +## Usage with Claude Code + +### As a project skill + +```bash +cp -r structsense-skills/ /path/to/your/project/.claude/skills/structsense-skills/ +``` + +Claude Code will discover it automatically from the `name` + `description` in `SKILL.md`'s frontmatter. Invoke implicitly by mentioning the task ("extract entities from this paper") or explicitly ("use the structsense-skills skill"). + +### As a user-global skill (auto-loaded on every project) + +```bash +cp -r structsense-skills/ ~/.claude/skills/structsense-skills/ +``` + +### Verify it loaded + +In Claude Code: + +``` +/skills +``` + +You should see `structsense-skills` in the list with its one-line description. + +--- + +## Usage with the Anthropic SDK + +The skill files are plain Markdown / JSON / Python — there's no SDK-specific format. Load `SKILL.md` (or a specific prompt file) as a system prompt: + +```python +from pathlib import Path +from anthropic import Anthropic + +SKILL_DIR = Path("structsense-skills") +client = Anthropic() + +# Load the NER extractor prompt (neuroscience variant) +system_prompt = (SKILL_DIR / "prompts" / "extractor-ner-neuroscience.md").read_text() + +# Or compose: SKILL.md + the specific prompt(s) the task needs +system_prompt = "\n\n---\n\n".join([ + (SKILL_DIR / "SKILL.md").read_text(), + (SKILL_DIR / "prompts" / "extractor-ner-neuroscience.md").read_text(), +]) + +response = client.messages.create( + model="claude-sonnet-4-6", + max_tokens=4096, + system=system_prompt, + messages=[{"role": "user", "content": f"INPUT TEXT:\n<<<\n{paper_text}\n>>>"}], +) +print(response.content[0].text) +``` + +For the **full pipeline** (extract → mask-recall → align → judge), use `scripts/pipeline.py` as a reference implementation. + +For **structured outputs**, pass `schemas/ner-output.schema.json` (or any other schema) via tool-use with a JSON-typed input or via prompt-side schema embedding. + +--- + +## Usage with OpenAI / GPT + +### ChatGPT custom GPTs + +1. Create a new GPT. +2. Paste the contents of `SKILL.md` + the relevant `prompts/extractor-*.md` into the "Instructions" field. +3. Upload `schemas/ner-output.schema.json` (and any other schemas you need) as a Knowledge file. +4. Save and invoke by asking it to extract from text. + +### OpenAI Assistants API or `chat.completions` + +Same pattern as the Anthropic example: + +```python +from pathlib import Path +from openai import OpenAI + +SKILL_DIR = Path("structsense-skills") +client = OpenAI() + +system_prompt = (SKILL_DIR / "prompts" / "extractor-ner-neuroscience.md").read_text() +schema = (SKILL_DIR / "schemas" / "ner-output.schema.json").read_text() + +resp = client.chat.completions.create( + model="gpt-4.1", + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": f"INPUT TEXT:\n<<<\n{paper_text}\n>>>"}, + ], + response_format={"type": "json_schema", + "json_schema": {"name": "ner", "schema": eval(schema)["properties"], "strict": True}}, + temperature=0, +) +``` + +For best results with GPT, use **OpenAI structured outputs** with the JSON Schemas in `schemas/` — it eliminates the need for `json_repair.py`. + +### Through OpenRouter + +Same as OpenAI, but the model string is `openrouter//` (e.g. `openrouter/anthropic/claude-sonnet-4-6`) and `base_url="https://openrouter.ai/api/v1"`. `scripts/llm_client.py` handles this transparently. + +--- + +## Usage with Pi, Gemini, Mistral, etc. + +The prompts work with any model that takes a system message. For providers without JSON-mode: + +1. Paste the prompt as the system instruction. +2. Add to the system prompt: `"Output JSON only. Do not wrap in markdown fences."` +3. Run the response through `scripts/json_repair.py:parse_or_repair()` — it strips fences, repairs trailing commas, and handles truncations. + +```python +from scripts.json_repair import parse_or_repair +raw_text = some_other_model.complete(system=prompt, user=text) +parsed = parse_or_repair(raw_text) +``` + +For Pi specifically: paste the prompt into the chat, then send the text in the next message. + +--- + +## Usage with local models (Ollama / vLLM) + +`scripts/llm_client.py` supports Ollama out of the box: + +```python +from scripts.llm_client import call + +raw = call( + model="ollama/qwen2.5:14b", + system=Path("structsense-skills/prompts/extractor-ner-neuroscience.md").read_text(), + user=f"INPUT TEXT:\n<<<\n{paper_text}\n>>>", + base_url="http://localhost:11434", + json_mode=True, + temperature=0, +) +``` + +For vLLM (OpenAI-compatible API), use `model="openai/"` and set `base_url` to your vLLM server. + +Small local models often produce malformed JSON; pair them with `scripts/json_repair.py` and consider constrained decoding (Outlines, llguidance, vLLM's `guided_json`). + +--- + +## Running the reference pipeline end-to-end + +`scripts/pipeline.py` wires together: (optional) HF NER ensemble + LLM extraction → mask-recall → ontology-mapping cascade → judging → grouping → stats. Standalone, no framework: + +```bash +export OPENROUTER_API_KEY=sk-or-v1-... + +python -m scripts.pipeline \ + --task ner \ + --input paper.txt \ + --extractor openrouter/anthropic/claude-sonnet-4-6 \ + --judge openrouter/openai/gpt-4o-mini \ + --mapper local \ + --mapper-url http://localhost:8000 \ + --ner-profile biomedical_broad \ + --chunk-size 2000 --max-workers 8 +# writes paper_final.json (the input stem + _final.json) and prints stats to stderr. +``` + +### Flags + +| Flag | Default | Description | +|---|---|---| +| `--task` | `ner` | `ner` / `resource` / `structured`. Picks the matching `extractor-*.md` prompt. | +| `--input` | (required) | Path to a `.txt` input. | +| `--extractor` | (required) | Model string for LLM extraction (e.g. `openrouter/anthropic/claude-sonnet-4-6`). | +| `--judge` | none = auto-approve | Model string for the judge stage. Omit to auto-approve (saves cost). | +| `--mapper` | `local` | Preferred concept-mapping backend: `local` / `bioportal` / `ols` / `none`. With `local`, the cascade is **local → BioPortal → ask user for URL → skip**. | +| `--mapper-url` | `http://localhost:8000` | Local hybrid service URL. Verify it's up at `/docs`. | +| `--non-interactive` | off | Disable the user prompt when the mapper cascade fails. Fail fast instead. | +| `--ner-profile` | none | Enable the HF ensemble: `biomedical_broad` / `cns_cells` / `pharmacology` / `genetic` / `clinical` / `minimal` / `all`. Requires `pip install transformers torch`. | +| `--ner-models` | none | Comma-separated explicit list of HF model IDs (overrides `--ner-profile`). | +| `--ner-device` | `-1` (CPU) | CUDA device index for HF models. | +| `--chunk-size` | `2000` | Characters per chunk for parallel extraction. | +| `--max-workers` | `8` | Parallel workers. | +| `--out` | `_final.json` | Output JSON path. Defaults to `_final.json` (e.g. `paper.pdf` → `paper_final.json`). | + +### What you get in the output + +The pipeline always writes a `_final.json` containing: + +```jsonc +{ + "source_metadata": { + "paper_title": "…", // ONCE per run, not per entity + "doi": "…", + "source_path": "paper.txt" + }, + + "entities": [ … ], // raw mentions — one per occurrence (exhaustive) + // each item carries source_model, judge_score, ontology_* + "key_terms": [ … ], + + "entities_grouped": [ … ], // canonical entities — mentions collapsed by (entity, label) + // each group lists source_models, sentences merged from all + // locations, mention_count, consensus_count, judge_score_max/avg/min + "key_terms_grouped": [ … ], + + "ensemble_models": [ // per-HF-model summary (skipped/loaded) + {"source_model": "d4data/biomedical-ner-all", "count": 215, "skipped_reason": null}, + {"source_model": "pruas/BENT-PubMedBERT-NER-Gene", "count": 124, "skipped_reason": null}, + "..." + ], + + "stats": { // run summary — see scripts/stats.py + "task_type": "ner", + "elapsed_seconds": {"total": 42.3, "extraction": 18.4, "ensemble_ner": 12.1, "alignment": 5.2, "judge": 6.6}, + "input": {"char_count": 28430, "chunk_count": 14, "chunk_size_chars": 2000, "input_path": "paper.txt"}, + "entities": { + "total_mentions": 1132, + "unique_surface_forms": 287, + "mentions_per_unique": 3.94, + "by_label": {"Gene": 312, "Protein": 198, "BrainRegion": 87, "…": "…"}, + "by_source_model": {"alvaroalon2/biobert_genetic_ner": 412, + "llm_ner:openrouter/anthropic/claude-sonnet-4-6": 287, + "d4data/biomedical-ner-all": 215, "…": "…"} + }, + "alignment": {"mapper_used": "local_hybrid", "mapper_url": "http://localhost:8000", + "fallback_triggered": false, "cascade_history": ["local_hybrid@http://localhost:8000"]}, + "judge": {"method": "llm", "score_buckets": {"1.00": 412, "0.85-0.99": 487, "…": "…"}} + } +} +``` + +A human-readable summary of the stats block is also printed to stderr after the run. + +See `examples/ner-example.md`, `examples/resource-example.md`, and `examples/reproschema-example.md` for worked walkthroughs. + +--- + +## Helper-script dependencies + +Pure-Python deps, install only what you use: + +```bash +# Always useful +pip install requests + +# For json_repair.py Tier-3 fallback (highly recommended) +pip install json-repair + +# For repair_to_schema() +pip install jsonschema + +# For llm_client.py — install whichever providers you actually use +pip install openai # OpenAI / OpenRouter / vLLM +pip install anthropic # Anthropic direct +pip install google-generativeai # Gemini + +# For the HuggingFace NER ensemble (ner_models.py) — required only if you +# use --ner-profile or pass ner_ensemble_profile=... to run() +pip install transformers torch + +# For sentence-aligned chunking with spaCy (optional; chunking.py falls back to regex) +pip install "spacy>=3.7" && python -m spacy download en_core_web_sm +``` + +None of the scripts depend on `structsense` or `crewai` — they're standalone. + +--- + +## Important rules (full list in `SKILL.md`) + +1. **Strict JSON only**, no markdown fences. `temperature: 0` for extraction and alignment. +2. **Extract EXHAUSTIVELY.** Emit every occurrence of every mention as its own item with its own `start`/`end`. Never deduplicate by surface form. If yield feels low, run the mask-recall pass. +3. **Preserve fields downstream.** Alignment, judge, and human-feedback stages ADD fields. They never remove existing ones. +4. **Record provenance** — every mapped item carries `concept_mapping_provenance` (`tool` / `llm_knowledge` / `unmapped` / `skipped`), `alignment_method`, and `source_model` (which NER model — HF or LLM — surfaced this mention). +5. **Chunk and merge** for inputs > model context (or > 25k chars for safety on 128k models). Re-merge by stable identifiers, not order. +6. **Don't invent placeholders.** Pipe stage outputs verbatim — extractor JSON → alignment, alignment JSON → judge. +7. **Validate before returning.** Parse JSON; repair-then-retry on failure; validate against the schema in `schemas/`. +8. **Always emit a `stats` block** at the top level (totals, label/source_model histograms, alignment provenance, judge score buckets, timings). +9. **Final-result filename:** `_final.json` (e.g. `paper.pdf` → `paper_final.json`). Honor `--out` only when the user provides one. +10. **Mapping cascade:** local hybrid (`http://localhost:8000`, verify at `/docs`) → BioPortal → **ask the user** for an alternative URL → skip alignment only if they decline. Don't hardcode the URL. +11. **Document metadata at top, not per entity.** `paper_title` / `doi` / `source_path` go ONCE in `source_metadata`. `paper_location` stays per-entity because it varies. +12. **Always emit both `entities[]` (raw) and `entities_grouped[]` (per-entity index).** The raw list is one-per-occurrence; the grouped list collapses by canonical (entity, label) with merged sentences + every contributing `source_model`. + +--- + +## Licence + +Apache 2.0 (matches `structsense`'s licence). diff --git a/skills/structsense-skills/SKILL.md b/skills/structsense-skills/SKILL.md new file mode 100644 index 0000000..b273bc0 --- /dev/null +++ b/skills/structsense-skills/SKILL.md @@ -0,0 +1,172 @@ +--- +name: structsense-skills +version: 0.4.0 +description: Extract structured information (named entities, key terms, resources like tools/datasets/models/benchmarks, or any target JSON schema) from unstructured text and PDFs using a model-agnostic multi-stage pipeline (extract → align to ontologies → judge → optional human feedback). Use this skill when the user asks to do NER, pull resources out of papers, convert documents to a target JSON schema (e.g. ReproSchema), or map terms to ontologies (BioPortal, OLS, OBO, BTO, CL, UBERON, NCBITaxon, etc.). Works with any LLM (Claude, GPT, Gemini, Pi, local Ollama/vLLM) — no library dependency. +license: Apache-2.0 +--- + +> **Skill version 0.4.0.** Concept mapping is now **mandatory and tool-only**. The pipeline refuses to silently skip alignment; it cascades local hybrid → BioPortal → ask the user for an alternate URL → hard-stop. Items with `concept_mapping_provenance: "llm_knowledge"` are automatically demoted to `unmapped` because hallucinated IRIs are not acceptable output. The pipeline now also tracks `validation` counts (passed / demoted / by_reason) and surfaces a prominent `totals` block in `stats`. Legacy outputs can be brought up to spec via `python -m scripts.normalize_result --input --llm-model ` (idempotent). See [CHANGELOG.md](CHANGELOG.md). + + + +# StructSense Skills — structured information extraction + +A reusable methodology for turning unstructured text and PDFs into clean, schema-conformant JSON, with optional ontology grounding and quality scoring. The patterns here are model-agnostic: they work with Claude, GPT, Gemini, Pi, or any local model. + +## When to invoke this skill + +Trigger when the user asks to: + +- Extract **named entities + key terms** (NER) from biomedical, neuroscience, or scientific text. +- Pull **resources** out of papers — tools, datasets, models, benchmarks, leaderboards. +- Convert a document into a **target JSON schema** (e.g. ReproSchema, Croissant, a custom schema the user supplies). +- **Map extracted terms to ontologies** (BioPortal, OLS, OBO, BTO, CL, UBERON, NCBITaxon, MESH, …). +- **Score or judge** the quality of an existing extraction. +- Process a **long document** that needs chunking and parallel runs. + +## The core pattern + +Four cooperating roles, run sequentially. Each role's output is the next role's input. Any role can use a different model. + +``` +┌───────────┐ raw text ┌────────────┐ extracted ┌───────┐ aligned ┌──────────────┐ +│ EXTRACTOR │ ──────────────►│ ALIGNMENT │──────────────►│ JUDGE │────────────►│ HUMAN FB │ +│ (LLM) │ │ (LLM+tool) │ │ (LLM) │ │ (optional) │ +└───────────┘ └────────────┘ └───────┘ └──────────────┘ + strict JSON + ontology fields + judge_score + corrections + + provenance + remarks + revised JSON +``` + +| Stage | Job | Reads | Writes | +|---|---|---|---| +| **Extractor** | Find entities/resources/fields. Output strict JSON. | raw text | items with `entity`/`name`, `label`/`type`, `sentence`, `start`, `end` (etc.) | +| **Alignment** | Map each item to an ontology IRI. | extractor output | adds `ontology_id`, `ontology_label`, `ontology`, `concept_mapping_provenance` (`tool` or `llm_knowledge`) | +| **Judge** | Score quality of each item (0–1). | alignment output | adds `judge_score`, `remarks` | +| **Human feedback** | Apply corrections from a human reviewer. | judge output + user feedback | revised JSON | + +You can run any subset — see `references/pipeline-pattern.md`. + +## Quick decision flow + +1. **What kind of extraction?** + - Entities + key terms (NER) → load `references/ner-extraction.md`, then pick the extractor prompt by domain: + - General-domain text (news, finance, biographies, generic web pages, mixed text) → `prompts/extractor-ner-general.md`. + - Neuroscience text — broad (behavior + systems + cellular + molecular + computational) → `prompts/extractor-ner-neuroscience.md`. + - CNS-cell-focused text (cell atlases, patch-seq, scRNA-seq cell typing, BICCN-style cell census — anything where cell types + markers + morphology + ephys are the subject) → `prompts/extractor-ner-cns-cells.md`. + - Tools / datasets / models / benchmarks → load `references/resource-extraction.md` and `prompts/extractor-resource.md`. + - User has a target JSON schema → load `references/structured-extraction.md` and `prompts/extractor-structured.md`. +2. **Want exhaustive recall? (almost always yes for NER)** → after pass-1 extraction, run the **mask-recall pass** with `prompts/mask-recall-pass.md` + `scripts/mask_pass.py`. Optionally also run **mask-verify** (`prompts/mask-verify-pass.md`) for per-item label sanity. See `references/ner-extraction.md` → "Two-pass strategy: mask-mode". +2b. **Biomedical text? Enable the HuggingFace NER ensemble.** Pass `--ner-profile biomedical_broad` (or `cns_cells` / `pharmacology` / `genetic` / `clinical` / `minimal` / `all`) to run specialist models alongside the LLM extractor. Every mention carries a `source_model` field; the grouped view records `consensus_count` (how many models agreed). See `references/ner-models.md`. Skip the ensemble for non-biomedical text or when `transformers` isn't installed. +3. **Need ontology mapping?** → load `references/ontology-mapping.md`. Default cascade: **local hybrid** at `http://localhost:8000` (verify at `/docs`) → **BioPortal** → **ask the user** for an alternative URL → skip alignment only if declined. Don't hardcode the URL; the port and host vary across deployments. +4. **Long document (>10 pages or > model context)?** → load `references/chunking-strategy.md`. Chunk → run extractor in parallel → merge → run downstream stages. +5. **Need quality scoring?** → load `prompts/judge.md`. +6. **Multiple models for cost?** Use the cheapest capable model for extraction (often a small open model), a stronger model for alignment if you don't have a mapping tool, and a fast model for judging. See `references/model-selection.md`. + +## Hard rules + +These prevent the most common failures. + +1. **Strict JSON output, no markdown fences.** Every prompt must include `"Output strict JSON only. No prose. No markdown fences."` in the system message. Set `temperature: 0` for extraction and alignment. +2. **Extract EXHAUSTIVELY.** For NER, emit every occurrence of every mention as a distinct item with its own `start`/`end`. Never deduplicate by surface form. A multi-page neuroscience paper should yield hundreds to thousands of entity items, not a few hundred. If yield feels low, run the mask-recall pass (`prompts/mask-recall-pass.md` + `scripts/mask_pass.py`) — typical recovery is +30–80%. +3. **Preserve fields downstream.** Alignment, judge, and human-feedback stages **add** fields. They never remove existing fields and never re-key existing items. +4. **Record provenance.** Every mapped item carries `concept_mapping_provenance: "tool" | "llm_knowledge"`. Never hide where a mapping came from. +5. **Chunk and merge** for inputs longer than the model's context window (or `> 25,000` chars for safety on 128k models). Always re-merge by stable identifiers (sentence + char span, or item `id`). +6. **Don't invent placeholders.** The agent communication contract is: extractor input is the raw text; alignment input is the extractor's JSON; judge input is the alignment's JSON. Pipe outputs cleanly — don't re-wrap or paraphrase between stages. +7. **Validate before returning.** Parse the JSON; if parsing fails, repair-then-retry (see `references/json-output-discipline.md`). Validate against the task's JSON schema in `schemas/`. +8. **Always emit a `stats` block.** Every final result must embed a `stats` block at the top level (totals, label histogram, alignment provenance, judge score buckets, per-stage elapsed times) and print a human-readable summary to stderr. Use `scripts/stats.py`. This is the answer to "did the run do what it was supposed to?" — a healthy NER run on a paper has hundreds-to-thousands of entity mentions and `mentions_per_unique > 1`. A summary with 230 mentions and `mentions_per_unique ≈ 1` is the symptom of surface-form deduplication; re-run with the mask-recall pass and double-check no upstream step is collapsing duplicates. +9. **Final-result filename convention.** When writing the result to disk, name it **`_final.json`** (e.g. `paper.pdf` → `paper_final.json`, `note.txt` → `note_final.json`). Honor an explicit `--out` only when the user provides one. The reference helper is `scripts/pipeline.py::default_output_path`. +10. **Concept-mapping cascade — and you MUST probe before declaring unavailable.** + Default mapper is the local hybrid service at **`http://localhost:8000`**. Before saying "no mapper available" you MUST run at least one probe in your current runtime: + ```bash + curl -s -o /dev/null -w '%{http_code}\n' http://localhost:8000/docs + ``` + If the probe returns 200, USE the mapper. The real API schema is **`{ "max_results": N, "text": [{"text": "...", "context": "..."}] }`** — NOT `terms:[...]`. See `prompts/alignment-via-http.md` for a turnkey curl + jq pipeline. + - On connection refused: try BioPortal (`BIOPORTAL_API_KEY`). + - On further failure: **ask the user** for an alternative URL (ports 8001 / 8080 / 9000 / reverse-proxied paths are common) — do not give up silently. + - Only after the user declines should you skip alignment (`concept_mapping_provenance: "skipped"`). + - **If your runtime can't reach the user's `localhost`** (claude.ai web app, Anthropic Skills hosted, ChatGPT cloud), say so explicitly and direct the user to the MCP bridge or tunnel options in `connecting/mcp-server.md`. Don't pretend the service is unreachable when the user has it running — be explicit that the runtime is the constraint. +11. **Always tag `source_model` provenance.** Every entity item carries `source_model` (HF model id like `d4data/biomedical-ner-all`, or `llm_ner:` for LLM-extracted items). The grouped view (`entities_grouped[]`) lists all contributing models per entity and the `consensus_count`. Never strip or merge these fields. See `references/ner-models.md`. +12. **Document metadata lives at the top, not on every entity.** `paper_title` / `doi` / `source_path` go into a top-level **`source_metadata`** block — ONCE per run. `paper_location` (section / page / paragraph) stays per-entity because it varies. Never repeat document-level metadata on hundreds of items. +13. **Always emit both `entities[]` (raw, one per occurrence) and `entities_grouped[]` (canonical, with merged sentences from every location).** The raw list is the authoritative record for exhaustive extraction; the grouped list is what downstream consumers navigate by. Use `scripts/group_by_entity.py:attach_grouped_views` — it's automatic in `pipeline.run()`. +14. **Canonical-shape guarantee via normalizer.** `scripts/normalize_result.py` runs automatically before every save and produces the canonical shape regardless of what the LLM emitted: top-level `source_metadata`, stripped per-entity `paper_title`/`doi`, tagged `source_model` on every item, `entities_grouped` attached, `stats` embedded. **Idempotent** — safe to run on already-canonical results. It's also exposed as a CLI to fix legacy result files in place. If you ever see legacy output, do not panic and do not edit by hand — run `python -m scripts.normalize_result ` and the file is brought up to spec. +15. **Concept mapping is MANDATORY and TOOL-ONLY. Zero hallucination.** + - The pipeline **never silently skips** alignment. Default cascade: **local hybrid** (`http://localhost:8000`, verify at `/docs`) → **BioPortal** (`BIOPORTAL_API_KEY`) → **ask the user** for an alternate URL → **hard-stop with a clear error**. OLS is no longer in the auto-cascade (it has no gene coverage); the user must opt in explicitly via `--allow-ols-fallback`. + - `concept_mapping_provenance: "llm_knowledge"` is **strictly forbidden** in canonical output. Any item carrying it is automatically demoted to `unmapped` by `scripts/iri_validation.py` and marked `alignment_method: "validation_failed"`. The item itself is preserved (exhaustive extraction is not compromised); only the fabricated mapping is dropped. + - Every `ontology_id` is **structurally validated** against known IRI patterns (OBO PURLs, identifiers.org, BioPortal PURLs, EBI OLS, semanticweb.org, generic `_` OWL IRIs). Malformed strings get demoted too. + - The output's `stats.validation` block reports `passed` / `demoted` counts and the breakdown by failure reason — so you can tell at a glance whether the LLM tried to hallucinate. + - If you cannot reach a real mapping tool, **say so to the user** and let them decide. Never invent IRIs to make the run "look complete." + +## File map (load on demand) + +The files below are intentionally separated so you only load what the current task needs. + +### `references/` +- `pipeline-pattern.md` — multi-stage agent pattern, how to chain stages, when to skip, resume from a saved stage. +- `ner-extraction.md` — NER methodology: entity types, output keys, edge cases, exhaustive extraction, mask-recall pass, grouped view. +- `ner-models.md` — HuggingFace + LLM ensemble: model roster, profiles by domain, `source_model` provenance, consensus count, when to skip. +- `resource-extraction.md` — resource extraction methodology (tools/datasets/models/benchmarks/leaderboards). +- `structured-extraction.md` — generic schema-driven extraction (PDF → user-supplied JSON schema). +- `ontology-mapping.md` — BioPortal REST API, OLS REST API, embedding-based hybrid retrieval, LLM-only fallback. Picking and combining backends. +- `chunking-strategy.md` — sentence-aligned chunking, parallel extraction, merge by stable key, context window math. +- `json-output-discipline.md` — schema-locked prompting, JSON repair, validation. +- `model-selection.md` — picking models per stage; OpenRouter / Ollama / vLLM / Claude / GPT / Gemini configuration. +- `human-feedback.md` — designing the human-in-the-loop review step. + +### `prompts/` +- `extractor-ner-general.md` — general-domain NER (Person, Org, Location, Product, Event, Date, …). +- `extractor-ner-neuroscience.md` — broad neuroscience NER (BrainRegion, CellType, Gene, Protein, Drug, Behavior, Disease, Method, Stimulus, Measurement, …). +- `extractor-ner-cns-cells.md` — CNS-cell-focused NER (CellClass / CellType / CellSubtype with lineage markers, morphology, ephys, layer, projection, atlas references, profiling method). +- `mask-recall-pass.md` — **pass-2** that surfaces mentions pass-1 missed. Run on any of the three NER prompts above. Big recall booster (typical +30–80%). +- `mask-verify-pass.md` — per-item label sanity check via cloze (mask one entity, predict label from context). Optional precision booster. +- `extractor-resource.md` — research resource extractor (one primary Model/Dataset/Tool/Benchmark per source, with `mentions` for secondaries). +- `extractor-structured.md` — schema-driven extractor (PDF → user-supplied JSON Schema). +- `alignment.md` — ontology alignment (LLM + concept-mapping tool). +- `alignment-via-http.md` — turnkey curl + jq pipeline for calling the local hybrid `/map/batch` endpoint directly. Use when you have Bash + network access but no Python client (Claude Code is the common case). +- `judge.md` — per-item quality judge. +- `humanfeedback.md` — apply human reviewer edits. + +### `schemas/` +- `ner-output.schema.json` — JSON Schema for NER output. +- `resource-output.schema.json` — JSON Schema for resource output. +- `aligned-item.schema.json` — fragment schema for any aligned item (adds ontology + provenance fields). +- `judged-item.schema.json` — fragment schema for any judged item (adds judge_score + remarks). + +### `scripts/` (runnable helpers) +- `chunking.py` — sentence-aligned chunking, span re-anchoring, deduplication. +- `json_repair.py` — four-tier JSON repair (strict → fences → json-repair → truncate-to-balanced) + schema-driven LLM repair. +- `span_validator.py` — validate `text[start:end] == entity`, repair from sentence context. +- `mask_pass.py` — build masked text for the mask-recall pass (offset-preserving placeholders), translate offsets back, and mask single items for the mask-verify pass. +- `stats.py` — compute the `stats` block (totals, label histogram, by-source-model histogram, alignment provenance, judge score buckets, per-stage timings) and a human-readable summary. +- `group_by_entity.py` — collapse raw mentions into `entities_grouped[]` keyed by canonical `(entity, label)`: collects all `mentions`, deduplicates `sentences` across locations, lists every contributing `source_model`, and reports `consensus_count`. Also exposes `unify_ontology_across_entities()` — when the same surface form gets different ontology IDs from different chunks/models, pick the best one (tool-mapped > LLM > unmapped) and apply it to every occurrence. Both run automatically in `attach_grouped_views()`. +- `ner_models.py` — HuggingFace NER ensemble (biomedical + clinical specialists). Default roster: d4data, BC5CDR-chem, NCBI-disease, BioBERT-genetic, plus the BENT-PubMedBERT family (Gene / Chemical / Disease / Anatomical / Cell-Type / Cell-Line / Organism / Bioprocess). Picked by profile: `biomedical_broad` / `cns_cells` / `pharmacology` / `genetic` / `clinical` / `minimal` / `all`. Every emitted item carries a `source_model` provenance field. +- `normalize_result.py` — **idempotent post-processor**. Lifts per-entity `paper_title`/`doi` into top-level `source_metadata`, strips the per-entity dupes, tags missing `source_model` + `alignment_method`, infers `task_type`, runs **strict IRI validation** (demotes `llm_knowledge` and malformed IRIs), attaches `entities_grouped[]`, and computes `stats` (with prominent `totals` block at the top). **Runs automatically** in `pipeline.py` before saving. Also exposed as a CLI: `python -m scripts.normalize_result legacy.json --input paper.txt --llm-model …`. This is the safety net that guarantees the canonical shape even when the LLM ignores the prompt. +- `iri_validation.py` — **strict IRI validator**. Per-ontology regex patterns + permissive structural fallback. Rejects `concept_mapping_provenance: "llm_knowledge"` outright (zero hallucination policy), demotes malformed IRIs to `unmapped`, accepts legitimate cross-ontology mappings (e.g. CIDO results that reuse HP IRIs). Adds `result["validation"]` with `passed` / `demoted` counts and `demoted_by_reason` breakdown. +- `input_loader.py` — **PDF / CSV / TXT / MD** ingestion. PDF backends in order: GROBID (if reachable) → PyMuPDF (`fitz`) → pdfminer.six. Graceful fallback so it works with whichever library is installed. Also writes a sibling `.txt` so subsequent NER stages have stable character offsets. CLI: `python -m scripts.input_loader paper.pdf [--no-grobid] [--grobid-url …]`. +- `task_detection.py` — **auto-detect task type** from a free-text task description: heuristic regex first (fast, no LLM), then LLM fallback via `llm_client`. Returns a `TaskDetection` with `task_type` (`ner`/`resource`/`structured_extraction`/`relation_extraction`/`keyphrase_extraction`/…), `confidence`, `labels`, `rationale`. +- `model_context.py` — **model context-window registry** (~50 model families, longest-match wins) + token-aware `compute_downstream_chunk_size(...)` for sizing alignment/judge/humanfeedback chunks. CLI: `python -m scripts.model_context openrouter/anthropic/claude-sonnet-4-6 --items 2000 --workers 8`. +- `bioportal_map.py` — throttled + LRU-cached BioPortal client. +- `ols_map.py` — EBI OLS client (no API key). +- `local_hybrid_map.py` — client for a self-hosted BM25+dense mapping service (one POST, many terms). +- `llm_client.py` — provider-agnostic LLM call (OpenAI / OpenRouter / Anthropic / Ollama / Gemini). +- `pipeline.py` — reference end-to-end pipeline (extract → align → judge) wiring the helpers together. + +### `examples/` +- `ner-example.md` — end-to-end NER worked example. +- `resource-example.md` — end-to-end resource extraction worked example. +- `reproschema-example.md` — end-to-end PDF → ReproSchema worked example. + +### `connecting/` (how to wire the skill into different LLM platforms) +- `claude-code.md` — install as a Claude Code skill (`~/.claude/skills/` or `.claude/skills/`). Auto-discovery via the `SKILL.md` frontmatter. +- `pi-dev.md` — install as a [Pi](https://pi.dev) skill (`~/.pi/agent/skills/`, `~/.agents/skills/`, or `.pi/skills/`). Pi is a CLI coding agent with native Agent Skills support and a built-in `bash` tool, so it runs the pipeline directly — same story as Claude Code. +- `claude-desktop.md` — **Claude Desktop has a split execution model**: chat UI on your machine, code interpreter in Anthropic's cloud sandbox (so it cannot reach your `localhost:8000` directly). Use the MCP server config in this guide to bridge. +- `claude-skills.md` — upload as a hosted Anthropic Skill on claude.ai or use with the Claude Agent SDK. +- `custom-gpt.md` — wire as an OpenAI Custom GPT (Instructions + Knowledge files + optional server-side Action). +- `mcp-server.md` — expose the pipeline as an MCP server so any MCP-aware client (Claude Code, Cursor, ChatGPT desktop, custom agents) can call it. + +## Minimal mental model + +If you remember nothing else, remember this: + +> Three short, focused prompts in sequence (extract, align, judge), each emitting strict JSON that the next prompt parses. Add a concept-mapping tool call inside alignment when you need real ontology IRIs. Chunk long inputs at sentence boundaries and merge by stable identifiers. Validate against a JSON schema. + +That's the entire skill. The references and prompts here are the careful version of that one sentence. diff --git a/skills/structsense-skills/connecting/claude-code.md b/skills/structsense-skills/connecting/claude-code.md new file mode 100644 index 0000000..c4ffd47 --- /dev/null +++ b/skills/structsense-skills/connecting/claude-code.md @@ -0,0 +1,137 @@ +# Connecting structsense-skills to Claude Code + +Claude Code (the CLI tool) auto-discovers skills via the `name` + `description` in `SKILL.md`'s YAML frontmatter. There's nothing to configure — drop the folder in the right place and Claude Code will load it. + +## 1. Install + +Pick a scope: + +### User-global (auto-loaded on every project) + +```bash +mkdir -p ~/.claude/skills +cp -r /path/to/structsense-skills ~/.claude/skills/ +``` + +After this, every Claude Code session on this machine has the skill available. + +### Per-project (versioned with the repo) + +```bash +cd /path/to/your/project +mkdir -p .claude/skills +cp -r /path/to/structsense-skills .claude/skills/ +git add .claude/skills/structsense-skills && git commit -m "Add structsense-skills" +``` + +Per-project takes precedence over user-global with the same name. + +## 2. Verify it loaded + +``` +/skills +``` + +You should see `structsense-skills` listed with its description. If you don't: + +- The folder must be `structsense-skills/` containing `SKILL.md` at its root (not `SKILL.md` inside a subfolder). +- The frontmatter MUST have both `name:` and `description:` (and they MUST match the YAML format — three dashes, key/value lines, three dashes). +- Restart Claude Code (some installations cache the skill registry). + +## 3. Invoke + +Either implicitly (Claude picks it up from your message): + +> "Extract all the gene mentions from this paper and map them to ontology IDs." + +> "Pull the tools and datasets out of these three READMEs as a structured table." + +Or explicitly: + +> "Use the structsense-skills skill to extract entities from this PDF and write the result as paper_final.json." + +## 4. What Claude will do + +When invoked, Claude reads `SKILL.md`, then loads only the reference / prompt files relevant to the current task (progressive disclosure). For NER on a biomedical paper, that's typically: + +- `references/ner-extraction.md` +- `prompts/extractor-ner-neuroscience.md` (or `extractor-ner-cns-cells.md`) +- `prompts/mask-recall-pass.md` +- `prompts/judge.md` +- `references/ontology-mapping.md` +- `references/ner-models.md` + +For the Python pipeline driver, Claude will call `scripts/pipeline.py` via Bash. Make sure your project's working directory has Python + dependencies available (`pip install requests openai anthropic json-repair jsonschema` — install the providers you actually use). + +## 5. API keys + +Set keys in your shell environment **before** launching Claude Code, or in a `.env` file in your project root: + +```bash +export OPENROUTER_API_KEY=sk-or-v1-... +export ANTHROPIC_API_KEY=sk-ant-... +export OPENAI_API_KEY=sk-... +export BIOPORTAL_API_KEY=... +``` + +Claude Code inherits the env from the shell that launched it. **Do not paste API keys into chat** — Claude will warn you. + +## 6. Picking models inside Claude Code + +When Claude calls `scripts/pipeline.py`, it can pass model strings: + +```bash +python -m scripts.pipeline \ + --task ner --input paper.txt \ + --extractor openrouter/anthropic/claude-sonnet-4-6 \ + --judge openrouter/openai/gpt-4o-mini \ + --mapper local --mapper-url http://localhost:8000 \ + --ner-profile biomedical_broad +``` + +You can tell Claude in plain English: *"use claude-sonnet-4-6 for extraction and gpt-4o-mini for the judge, with the cns_cells NER profile"*. Claude will translate that into the right CLI flags. + +## 7. Working with the local concept-mapping service + +If you run the [search_hybrid](https://github.com/sensein/search_hybrid) service locally for ontology mapping, **make sure it's reachable before invoking the skill**. Claude will tell you if the cascade falls through to BioPortal or asks you for an alternate URL. + +```bash +# Verify the service is up before running the skill +curl -s http://localhost:8000/docs > /dev/null && echo OK +``` + +## 8. Updating the skill + +When you pull a new version of the skill: + +```bash +cd ~/.claude/skills/structsense-skills # or .claude/skills/structsense-skills +git pull # if it's a checkout +# OR +rm -rf structsense-skills/ && cp -r /path/to/new/structsense-skills . +``` + +Check the version with: + +```bash +head -5 ~/.claude/skills/structsense-skills/SKILL.md +``` + +Look for `version: 0.3.0` (or higher). If you have output JSON from an older version, run: + +```bash +python -m scripts.normalize_result old_output.json --input paper.txt \ + --llm-model openrouter/anthropic/claude-sonnet-4-6 +``` + +to bring it up to the current canonical shape (idempotent). + +## 9. Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| `/skills` doesn't list the skill | Wrong folder name or missing frontmatter | Check `name:` + `description:` in `SKILL.md`. | +| Claude ignores the skill in a conversation | Claude didn't see a trigger for it | Mention the task explicitly ("extract entities", "map to ontologies") or invoke by name. | +| Output still has `paper_title` on every entity | LLM ignored the prompt; or you're calling prompts directly without the pipeline | Run `python -m scripts.normalize_result ` — it's idempotent and lifts the legacy fields to the canonical shape. | +| Pipeline can't find the concept mapping service | Service not running at the expected port | Start `search_hybrid`, then re-run; or accept the interactive prompt and supply your URL. | +| HF NER models fail with "transformers not installed" | `transformers` package missing | `pip install transformers torch`, or omit `--ner-profile` to skip the ensemble. | diff --git a/skills/structsense-skills/connecting/claude-desktop.md b/skills/structsense-skills/connecting/claude-desktop.md new file mode 100644 index 0000000..c1f2e0d --- /dev/null +++ b/skills/structsense-skills/connecting/claude-desktop.md @@ -0,0 +1,122 @@ +# Connecting structsense-skills to Claude Desktop + +Claude Desktop is the third major Claude runtime, alongside Claude Code (CLI) and claude.ai (web). It looks like a local app on your machine, but it has a **split execution model** that surprises most users — and is the reason your local `http://localhost:8000` mapping service appears "unreachable" even when you can hit it with curl in another terminal. + +## The split execution model — read this first + +| What runs where | Location | Can reach your `localhost`? | +|---|---|---| +| **The Claude chat UI** | Your machine (the app window) | (Doesn't make HTTP calls) | +| **The conversation LLM** | Anthropic's cloud | No | +| **Python / Bash / code interpreter** | Anthropic's cloud sandbox (look for paths like `/home/claude/work/...`) | **No** | +| **File uploads & downloads** | Your machine ↔ the cloud sandbox over the API | (Files, not network) | +| **MCP servers you configure** | **Your machine** | **Yes** | + +The giveaway in your skill output: + +``` +"input_path": "/home/claude/work/paper.txt" +``` + +That `/home/claude/work/` path is Anthropic's cloud sandbox — not your machine. Anything that runs in that sandbox cannot dial back to your `localhost:8000` because the sandbox is in a different network namespace. + +**The fix is not to "make the sandbox reach my localhost"** (you can't). The fix is to **expose the mapper via an MCP server that runs on your machine**, so Claude Desktop calls your machine through MCP instead of through the sandbox. + +## What you have to set up + +Claude Desktop reads MCP server configuration from a JSON file. On macOS: + +``` +~/Library/Application Support/Claude/claude_desktop_config.json +``` + +On Windows: + +``` +%APPDATA%\Claude\claude_desktop_config.json +``` + +Add an entry that launches a small MCP server which proxies HTTP calls to your local `/map/batch`. The pattern is the one in [`connecting/mcp-server.md`](mcp-server.md) — here's the Claude-Desktop-specific configuration: + +```json +{ + "mcpServers": { + "structsense-skills": { + "command": "python3", + "args": [ + "/absolute/path/to/structsense-skills/connecting/mcp_server.py" + ], + "env": { + "LOCAL_CONCEPT_MAPPING_URL": "http://localhost:8000", + "OPENROUTER_API_KEY": "sk-or-v1-...", + "BIOPORTAL_API_KEY": "..." + } + } + } +} +``` + +Then in Claude Desktop: + +1. Quit the app entirely (Cmd-Q / right-click tray → Quit). Don't just close the window. +2. Reopen. Look for the 🔌 icon in the input field — clicking it should list `structsense-skills` and its tools. +3. Ask Claude to use the tool: *"use structsense-skills/extract_ner on this paper, with the local mapper at http://localhost:8000"*. + +Claude will now call the MCP server (on your machine) instead of trying to dial localhost from the cloud sandbox. The MCP server has network access to localhost and will return the mapped concepts. + +## Why this is necessary even for the simple curl case + +Looking at your existing output: + +``` +The skill's default local hybrid mapper at http://localhost:8000 isn't +reachable from this runtime (probe returned connection-refused), and no +BioPortal key is set. I used the no-key EBI OLS client instead… +``` + +That message is correct and honest. Claude in the cloud sandbox genuinely cannot reach your localhost. It fell back to OLS (which is on the public internet, so the sandbox CAN reach it) and was upfront about the substitution. The judge scores were the heuristic ones because no LLM API key was set in the sandbox. + +To get the full pipeline behavior you want — local mapper + full LLM judge — you have three options, in order of effort: + +1. **MCP bridge** (recommended) — the config above. Five minutes to set up; Claude Desktop calls your machine's services natively. +2. **Tunnel** (`ngrok http 8000`, `cloudflared tunnel`, Tailscale Funnel) — the cloud sandbox CAN reach a public URL. Set `LOCAL_CONCEPT_MAPPING_URL=https://.ngrok.io` and the existing skill code Just Works. +3. **Switch runtime to Claude Code** (CLI) — Claude Code runs Bash on your machine, no sandbox in between. The skill's existing `curl ${MAPPER_URL}/docs` probe works directly. + +For a one-off run, option 2 (a tunnel) is fastest. For ongoing use, option 1 (MCP) is cleanest. + +## What works without any of the above + +Even without an MCP bridge or tunnel, the skill still produces useful output from Claude Desktop: + +- All extraction (LLM-driven NER, mask-recall) works fully — the LLM is in the cloud already, doesn't need your machine. +- The OLS fallback (public, no key) provides ontology mapping for anatomy / cells / diseases / chemicals — anything OLS hosts. +- Heuristic judge scores (span-validation + ontology grounding + specificity) work — no API key needed. +- The normalizer canonicalizes the result (`paper_final.json` with `source_metadata`, `entities_grouped`, `stats`, `totals` block). + +What you lose without the bridge: + +- The local hybrid mapper's richer coverage (it tends to be tuned to your domain and has higher-fidelity matching for unusual terms). +- Gene mappings (OLS doesn't host HGNC; BioPortal or your local service does). +- Real LLM-judge scores (only matters when you specifically want a separate model to score the alignment). + +## Recognizing a Claude Desktop run in your output + +If you're not sure which runtime produced an output file, look for these signs: + +| Signal | Runtime | +|---|---| +| `input_path: /home/claude/work/...` | Claude Desktop or claude.ai web (cloud sandbox) | +| `input_path: /Users//...` or `/home//...` | Claude Code (your machine) | +| `source_model: llm_ner:claude-opus-4-8` (or similar Anthropic model) and no `source_model: ` items | LLM-only extraction, no HF ensemble (the HF models require `transformers` + weights, usually only present on your machine) | +| `alignment.cascade_history: ["ols"]` | Public fallback used; the local mapper wasn't reachable | +| `alignment.mapper_used: "local_hybrid"` with `mapper_url: "http://localhost:..."` | Local mapper reached (via Claude Code or MCP bridge) | + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| The MCP server doesn't show up after editing the config | App not fully quit, or JSON syntax error | Quit and reopen; validate JSON with `jq . < claude_desktop_config.json`. | +| MCP server appears but tools fail with "connection refused" | The MCP process can't reach the local mapper either (firewall, wrong port) | From a terminal: `curl http://localhost:8000/docs` — if that fails, the mapper isn't running where you think. | +| Claude says "local mapper unreachable" even with MCP configured | Claude is still trying the sandbox path. Be explicit in your message. | "Use the structsense-skills/extract_ner MCP tool" — naming the tool forces the MCP route. | +| You get the OLS-fallback output anyway | Same as above + the model wasn't told to insist on the local mapper | Add to your message: "Do NOT fall back to OLS. If the MCP tool fails, ask me for an alternate URL." | +| Output has `paper_title`/`doi` on every entity | Old skill version, or the model emitted the legacy shape | Run `python -m scripts.normalize_result --input --llm-model ` — idempotent. | diff --git a/skills/structsense-skills/connecting/claude-skills.md b/skills/structsense-skills/connecting/claude-skills.md new file mode 100644 index 0000000..c1cfc53 --- /dev/null +++ b/skills/structsense-skills/connecting/claude-skills.md @@ -0,0 +1,99 @@ +# Connecting structsense-skills as a hosted Claude Skill + +Anthropic Skills (in claude.ai and the Claude Agent SDK) accept the **same `SKILL.md` + supporting files** layout this repo uses. There's nothing to change in the skill itself — just upload or reference it. + +## Option A: Upload to claude.ai as a personal skill + +1. Go to claude.ai → Settings → Skills (or the analogous menu in your workspace). +2. Click "Create Skill" / "Upload Skill". +3. Upload the **entire `structsense-skills/` folder** (or zip it first). +4. Claude reads `SKILL.md`'s `name:` + `description:` for discovery and `version:` for change-tracking. The frontmatter is already correct. +5. The skill becomes available in any conversation by mention ("use the structsense-skills skill", or by invoking it implicitly with a triggering request). + +Files Claude loads on demand from the skill: +- `references/*.md` — methodology docs (loaded only when relevant). +- `prompts/*.md` — system prompts (used directly as message content). +- `schemas/*.json` — JSON Schemas for validating outputs. +- `scripts/*.py` — Python helpers (executed via the code interpreter / sandbox if your workspace allows it). + +## Option B: Use with the Claude Agent SDK + +```python +from anthropic import Anthropic +from pathlib import Path + +client = Anthropic() +SKILL = Path("structsense-skills") + +# Load SKILL.md + the prompts the task needs as the system message +system = "\n\n---\n\n".join([ + (SKILL / "SKILL.md").read_text(), + (SKILL / "prompts" / "extractor-ner-neuroscience.md").read_text(), +]) + +response = client.messages.create( + model="claude-sonnet-4-6", + max_tokens=8192, + system=system, + messages=[ + {"role": "user", "content": f"INPUT TEXT:\n<<<\n{paper_text}\n>>>"} + ], +) +``` + +Or, if the Skills API in the SDK takes a folder path directly: + +```python +response = client.messages.create( + model="claude-sonnet-4-6", + skills=[{"path": "structsense-skills"}], # SDK-specific + messages=[...], +) +``` + +(Check the latest SDK docs — the exact parameter name evolves.) + +## Option C: Share the skill via a shareable link + +If you've made the skill installable, use the `ShareOnboardingGuide` tool or the platform's skill-sharing feature to generate a short link. Teammates open the link in Claude Code or claude.ai and the skill loads automatically. + +## What Claude does with the skill + +Once loaded, Claude: + +1. Reads `SKILL.md` to understand when to trigger the skill. +2. When triggered (NER request, ontology mapping, schema-driven extraction, etc.), loads only the relevant `references/` and `prompts/` files (progressive disclosure — the rest is ignored). +3. For runnable work, calls `scripts/pipeline.py` via Bash or the sandbox. +4. Always produces output in the canonical shape (top-level `source_metadata`, `entities[]` raw, `entities_grouped[]`, `stats`, per-entity `source_model`). Even if the LLM emits the legacy shape, the post-processor in `scripts/normalize_result.py` runs automatically before the file is saved. + +## API keys + +Claude Skills typically inherit env vars from the sandbox / desktop session. Set: + +```bash +export OPENROUTER_API_KEY=sk-or-v1-... +export BIOPORTAL_API_KEY=... +``` + +Never paste keys into chat — Claude will refuse and warn you. + +## Updating the hosted skill + +Re-upload after a version bump. Check `version:` in `SKILL.md`: + +```bash +grep ^version structsense-skills/SKILL.md +``` + +If the hosted version is behind, re-upload. Result files from earlier versions can be repaired in-place: + +```bash +python -m scripts.normalize_result old.json --input paper.txt \ + --llm-model openrouter/anthropic/claude-sonnet-4-6 +``` + +## What this skill does NOT do via Anthropic Skills + +- It does not store API keys in the skill. Keys come from your env or `.env`. +- It does not auto-upgrade. You re-upload to pick up new versions. +- It does not bundle model weights. The HF NER ensemble downloads weights on first use. diff --git a/skills/structsense-skills/connecting/custom-gpt.md b/skills/structsense-skills/connecting/custom-gpt.md new file mode 100644 index 0000000..65ecc67 --- /dev/null +++ b/skills/structsense-skills/connecting/custom-gpt.md @@ -0,0 +1,171 @@ +# Connecting structsense-skills to ChatGPT (Custom GPT) + +OpenAI doesn't yet have a first-class "skills" concept the way Claude does, but a **Custom GPT** with the right Instructions + Knowledge files + (optionally) an Action gives you a comparable connection. This guide walks through it. + +## What you'll build + +A Custom GPT that: +- Knows the structsense-skills pattern (extract → align → judge → group) from its Instructions. +- Has the relevant prompt files + JSON Schemas as Knowledge files. +- (Optional) Calls a hosted `pipeline.py` HTTP service via an Action when the user wants full end-to-end runs. + +## 1. Create the GPT + +1. In ChatGPT, click "Create a GPT". +2. Name it "**structsense-skills**" (or similar). +3. Set the description to the same text as the `description:` field in [SKILL.md](../SKILL.md). This is what users will see in the GPT picker. + +## 2. Instructions + +Paste the contents of [SKILL.md](../SKILL.md) into the **Instructions** field. Then append: + +``` +When the user asks for NER on biomedical text, use the +`extractor-ner-neuroscience.md` prompt verbatim as the system message, +followed by the user's input under a `INPUT TEXT:\n<<<\n…\n>>>` block. + +For CNS cell-typing text, use `extractor-ner-cns-cells.md`. +For general-domain text, use `extractor-ner-general.md`. + +After pass-1, run mask-recall (prompts/mask-recall-pass.md) for biomedical +papers — typical recovery is +30-80% mentions. + +Always emit the canonical shape: +- source_metadata at the TOP LEVEL (paper_title, doi, source_path). +- entities[] one item per occurrence (exhaustive). +- entities_grouped[] per-entity index. +- stats embedded. +- Each entity has source_model = "llm_ner:gpt-4.1" (or whichever model). +- DO NOT put paper_title or doi on every entity. + +If asked to call the pipeline end-to-end (with HF ensemble, ontology mapping, +judge stage), use the `run_pipeline` Action. +``` + +This nudges GPT to follow the same conventions the SKILL.md describes. + +## 3. Knowledge files + +Upload the following from the `structsense-skills/` folder: + +| File | Why upload | +|---|---| +| `SKILL.md` | Discovery + rules. | +| `prompts/extractor-ner-general.md` | General NER prompt. | +| `prompts/extractor-ner-neuroscience.md` | Neuroscience NER prompt. | +| `prompts/extractor-ner-cns-cells.md` | CNS-cell NER prompt. | +| `prompts/mask-recall-pass.md` | Pass-2 recall booster. | +| `prompts/mask-verify-pass.md` | Per-item label sanity. | +| `prompts/extractor-resource.md` | Resource extraction. | +| `prompts/extractor-structured.md` | Schema-driven extraction. | +| `prompts/alignment.md` | Ontology alignment. | +| `prompts/judge.md` | Quality scoring. | +| `schemas/ner-output.schema.json` | Output validation. | +| `schemas/resource-output.schema.json` | Resource output validation. | +| `references/ner-extraction.md` | NER methodology. | +| `references/ner-models.md` | HF ensemble docs. | +| `references/ontology-mapping.md` | Mapper backends + cascade. | +| `references/json-output-discipline.md` | Strict-JSON + repair patterns. | + +GPT will retrieve from these when relevant. Don't upload all of `scripts/` — Custom GPTs don't execute Python natively. + +## 4. Enable JSON-schema-mode output + +In **Capabilities**, ensure "Code Interpreter" is enabled if you want GPT to run Python on the user's behalf. For best schema compliance: + +- Tell GPT in Instructions: "When emitting JSON, use the `response_format: json_object` capability." +- For full structured outputs (the strongest guarantee), use the OpenAI API directly with `response_format={"type": "json_schema", "json_schema": {...}}` and `schemas/ner-output.schema.json` — Custom GPTs in the chat UI don't expose this option, but the model still tends to comply when shown the schema in Instructions. + +## 5. Optional: Action that calls the pipeline end-to-end + +If you have a server running `scripts/pipeline.py` as an HTTP service (FastAPI wrapper — see `connecting/mcp-server.md` for one option), add a Custom GPT **Action** so the GPT can invoke the full pipeline (HF ensemble + ontology mapping + judge + grouped + stats) without doing it inline. + +### Minimal OpenAPI spec for the Action + +```yaml +openapi: 3.0.0 +info: + title: structsense-skills pipeline + version: 0.3.0 +servers: + - url: https://your-server.example.com # replace +paths: + /extract: + post: + operationId: run_pipeline + summary: Run the full structsense-skills pipeline on a text input + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [text, task, extractor_model] + properties: + text: { type: string, description: Source text. } + task: { type: string, enum: [ner, resource, structured] } + extractor_model: { type: string, description: Model string e.g. openrouter/anthropic/claude-sonnet-4-6 } + judge_model: { type: string, nullable: true } + mapper: { type: string, enum: [local, bioportal, ols, none], default: local } + mapper_url: { type: string, default: http://localhost:8000 } + ner_profile: { type: string, nullable: true, enum: [biomedical_broad, cns_cells, pharmacology, genetic, clinical, minimal, all] } + chunk_size: { type: integer, default: 2000 } + max_workers: { type: integer, default: 8 } + paper_title: { type: string, nullable: true } + doi: { type: string, nullable: true } + responses: + '200': + description: Pipeline result in the canonical structsense-skills shape. + content: + application/json: + schema: + type: object + properties: + source_metadata: { type: object } + entities: { type: array } + entities_grouped: { type: array } + key_terms: { type: array } + key_terms_grouped: { type: array } + stats: { type: object } + ensemble_models: { type: array } +``` + +Save this as `connecting/openapi-pipeline.yaml`, paste it into the Custom GPT's Action editor, and set the auth (Bearer token recommended). + +Behind the URL you need a small FastAPI service that wraps `pipeline.run()`. See `connecting/mcp-server.md` for a working example you can copy. + +## 6. API keys + +Custom GPTs run inside ChatGPT's infrastructure — they don't have access to your shell env. Two ways to provide keys: + +- **In the Action's auth config**: use Bearer token; the server reads `OPENROUTER_API_KEY` etc. from its own env. +- **For inline use (no Action)**: the GPT can ask the user to paste a key, but you should NOT do this — keys end up in the chat log. + +The right setup is a server-side Action whose env has the keys. + +## 7. Test + +In the GPT, try: + +> "Extract entities from this paper: [paste 2-3 pages of biomedical text]. Title: 'Foo bar atlas'. DOI: 10.1234/foo." + +Expected output: +- Top-level `source_metadata: { paper_title: "Foo bar atlas", doi: "10.1234/foo" }`. +- `entities[]` — many items, each with `source_model: "llm_ner:gpt-4.1"` (or your default model), `start`/`end`/`sentence`/`paper_location`. +- No `paper_title` or `doi` on individual entities. +- `entities_grouped[]` — entities consolidated by `(surface, label)`. +- `stats` block at the bottom. + +If the GPT regresses to the legacy shape on long inputs (per-entity `paper_title`/`doi`), re-emphasize the WRONG/RIGHT block in the Instructions, or run the output through `scripts/normalize_result.py` server-side. + +## 8. What this gives you vs the Claude Code path + +| | Claude Code | Custom GPT | +|---|---|---| +| Auto-loads SKILL.md | ✅ | Via Instructions paste | +| Progressive file loading | ✅ native | Knowledge retrieval (similar) | +| Calls `scripts/pipeline.py` | ✅ via Bash | ✅ via Action (server) | +| Strong JSON-schema mode | ✅ Anthropic structured outputs | ✅ via API; weaker in chat UI | +| Local file IO (writes `paper_final.json`) | ✅ | Only via server Action | + +For research/notebook workflows, Claude Code is the simpler path. For team-facing tools where users come through ChatGPT, the Custom GPT + Action setup is the right answer. diff --git a/skills/structsense-skills/connecting/mcp-server.md b/skills/structsense-skills/connecting/mcp-server.md new file mode 100644 index 0000000..ce123b0 --- /dev/null +++ b/skills/structsense-skills/connecting/mcp-server.md @@ -0,0 +1,249 @@ +# Connecting structsense-skills as an MCP server + +The Model Context Protocol (MCP) lets you expose `scripts/pipeline.py` as a tool callable from any MCP-aware client: Claude Code, ChatGPT desktop, the Anthropic SDK, the OpenAI Agents SDK, Cursor, and others. This guide shows the simplest setup. + +## What you get + +An MCP server with these tools: + +- `extract_ner` — full NER pipeline (extract → mask-recall → align → judge → group → stats). +- `extract_resources` — research-resource extraction. +- `extract_structured` — schema-driven extraction. +- `normalize_result` — repair a legacy result JSON. + +The MCP server runs as a long-lived process. Clients connect to it (over stdio, HTTP, or SSE depending on transport) and call tools by name. + +## Option A: Minimal stdio server with `mcp` Python SDK + +```bash +pip install mcp openai anthropic json-repair jsonschema requests +# Optional for the HF ensemble: +pip install transformers torch +``` + +Create `connecting/mcp_server.py` (referenced here; not included so you can adapt freely): + +```python +"""Minimal MCP server exposing the structsense-skills pipeline. + +Run with: + python connecting/mcp_server.py +or via Claude Code's mcpServers config (see below). +""" +import asyncio, sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) +from pipeline import run +from normalize_result import normalize + +from mcp.server import Server +from mcp.server.stdio import stdio_server +from mcp.types import Tool, TextContent +import json + +server = Server("structsense-skills") + + +@server.list_tools() +async def list_tools() -> list[Tool]: + return [ + Tool( + name="extract_ner", + description=("Extract named entities and key terms from a passage " + "using the structsense-skills pipeline. Returns a " + "result with source_metadata at top level, " + "entities[] (one per occurrence), entities_grouped[] " + "(per-entity index), and stats."), + inputSchema={ + "type": "object", + "required": ["text", "extractor_model"], + "properties": { + "text": {"type": "string"}, + "extractor_model": {"type": "string"}, + "judge_model": {"type": "string"}, + "mapper": {"type": "string", "enum": ["local","bioportal","ols","none"], "default": "local"}, + "mapper_url": {"type": "string", "default": "http://localhost:8000"}, + "ner_profile": {"type": "string"}, + "chunk_size": {"type": "integer", "default": 2000}, + "max_workers": {"type": "integer", "default": 8}, + "paper_title": {"type": "string"}, + "doi": {"type": "string"}, + }, + }, + ), + Tool( + name="normalize_result", + description="Normalize a legacy structsense-skills result JSON to " + "the canonical 0.3.0 shape. Idempotent.", + inputSchema={ + "type": "object", + "required": ["result"], + "properties": { + "result": {"type": "object"}, + "llm_model": {"type": "string"}, + "input_path": {"type": "string"}, + }, + }, + ), + ] + + +@server.call_tool() +async def call_tool(name: str, arguments: dict) -> list[TextContent]: + if name == "extract_ner": + result = run( + text=arguments["text"], + task="ner", + extractor_model=arguments["extractor_model"], + mapper_backend=arguments.get("mapper", "local"), + judge_model=arguments.get("judge_model"), + chunk_size=arguments.get("chunk_size", 2000), + max_workers=arguments.get("max_workers", 8), + skip_judge=arguments.get("judge_model") is None, + local_mapping_url=arguments.get("mapper_url", "http://localhost:8000"), + ask_user=None, # MCP can't prompt + input_path=None, + ner_ensemble_profile=arguments.get("ner_profile"), + ) + # Inject the caller-provided paper_title / doi if any + if arguments.get("paper_title") or arguments.get("doi"): + sm = result.setdefault("source_metadata", {}) + if arguments.get("paper_title"): sm["paper_title"] = arguments["paper_title"] + if arguments.get("doi"): sm["doi"] = arguments["doi"] + return [TextContent(type="text", text=json.dumps(result, indent=2, default=str))] + + if name == "normalize_result": + result = arguments["result"] + normalize(result, + llm_model=arguments.get("llm_model"), + input_path=arguments.get("input_path")) + return [TextContent(type="text", text=json.dumps(result, indent=2, default=str))] + + return [TextContent(type="text", text=json.dumps({"error": f"unknown tool {name}"}))] + + +async def main(): + async with stdio_server() as (read, write): + await server.run(read, write, server.create_initialization_options()) + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +## Option B: Configure in Claude Code + +Add to your Claude Code MCP config (`~/.claude/mcp.json` or your project's `.claude/settings.json`): + +```json +{ + "mcpServers": { + "structsense-skills": { + "command": "python", + "args": ["/abs/path/to/structsense-skills/connecting/mcp_server.py"], + "env": { + "OPENROUTER_API_KEY": "${OPENROUTER_API_KEY}", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "BIOPORTAL_API_KEY": "${BIOPORTAL_API_KEY}" + } + } + } +} +``` + +Restart Claude Code. The tools `extract_ner` and `normalize_result` should now be available — you'll see them in `/tools` or when Claude offers tool use. + +## Option C: Run as an HTTP server (for ChatGPT Custom GPT Actions) + +For the OpenAI Custom GPT integration (see `connecting/custom-gpt.md`), wrap the same calls in a FastAPI app: + +```python +# connecting/http_server.py +from fastapi import FastAPI +from pydantic import BaseModel +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) +from pipeline import run + +app = FastAPI(title="structsense-skills", version="0.3.0") + + +class ExtractReq(BaseModel): + text: str + task: str = "ner" + extractor_model: str + judge_model: str | None = None + mapper: str = "local" + mapper_url: str = "http://localhost:8000" + ner_profile: str | None = None + chunk_size: int = 2000 + max_workers: int = 8 + paper_title: str | None = None + doi: str | None = None + + +@app.post("/extract") +def extract(req: ExtractReq): + result = run( + text=req.text, + task=req.task, + extractor_model=req.extractor_model, + mapper_backend=None if req.mapper == "none" else req.mapper, + judge_model=req.judge_model, + chunk_size=req.chunk_size, + max_workers=req.max_workers, + skip_judge=req.judge_model is None, + local_mapping_url=req.mapper_url, + ask_user=None, + input_path=None, + ner_ensemble_profile=req.ner_profile, + ) + if req.paper_title or req.doi: + sm = result.setdefault("source_metadata", {}) + if req.paper_title: sm["paper_title"] = req.paper_title + if req.doi: sm["doi"] = req.doi + return result +``` + +Run it: + +```bash +pip install fastapi uvicorn +uvicorn connecting.http_server:app --host 0.0.0.0 --port 8080 +``` + +Add a Custom GPT Action pointing at `http://your-host:8080/extract`. See `connecting/custom-gpt.md` step 5 for the OpenAPI spec. + +## API keys + +The MCP / HTTP server reads keys from its own env — never from arguments. Set them in the server's launch env: + +```bash +export OPENROUTER_API_KEY=sk-or-v1-... +export BIOPORTAL_API_KEY=... +``` + +Or pass through `mcpServers..env` in the Claude Code config (above), which expands `${VAR}` from the shell that launched Claude Code. + +## Why MCP and not just a CLI? + +- **Persistent process**: HF model weights load once, not per request. +- **Multiple clients**: Claude Code, Cursor, ChatGPT desktop, your own scripts can all talk to the same instance. +- **Strong typing**: the JSON Schemas declared on each tool prevent the kinds of accidental misuse (wrong model strings, missing required fields) that bug LLMs. +- **Streaming-friendly**: long-running ensemble runs can stream progress back to the client. + +Skip MCP if you only call the pipeline from one Python script — the CLI is simpler. + +## Verifying the connection + +From Claude Code, after restarting: + +> "list MCP tools" + +You should see `structsense-skills/extract_ner` and `structsense-skills/normalize_result`. + +> "use the structsense-skills extract_ner tool on this text: [paste]" + +The result should arrive as a single JSON blob with `source_metadata`, `entities`, `entities_grouped`, `stats`, etc. diff --git a/skills/structsense-skills/connecting/pi-dev.md b/skills/structsense-skills/connecting/pi-dev.md new file mode 100644 index 0000000..84db073 --- /dev/null +++ b/skills/structsense-skills/connecting/pi-dev.md @@ -0,0 +1,218 @@ +# Connecting structsense-skills to Pi (pi.dev) + +[Pi](https://pi.dev) is an open-source CLI coding agent by Earendil Inc. (Mario Zechner) — terminal-based, runs locally, no SaaS backend. It natively supports the **Agent Skills** standard (the same `SKILL.md` layout this repo uses) and has built-in `read`/`write`/`edit`/`bash` tools, so it can both discover the skill and run `scripts/pipeline.py` directly. In practice this is the same story as Claude Code: drop the folder in a directory Pi searches, and it loads. + +> **Versions move fast.** Pi is young and actively developed. The skill-search paths, slash commands, and env-var names below are accurate as of writing but may drift — confirm against `pi --help`, the in-session `/help`, and the [docs](https://pi.dev/docs/latest/) for your installed version. + +## 0. Install Pi (if you haven't) + +```bash +# macOS / Linux +curl -fsSL https://pi.dev/install.sh | sh + +# or via npm (pnpm / yarn / bun also work) +npm install -g --ignore-scripts @earendil-works/pi-coding-agent +``` + +Then, from any project directory: + +```bash +pi # start an interactive session +/login # first-time auth (or set an env var — see §5) +/model # pick model + provider +``` + +## 1. Install the skill + +Pi searches these locations for skills, in order (first match wins): + +1. `~/.pi/agent/skills/` — user-global, Pi-specific +2. `~/.agents/skills/` — user-global, shared across Agent-Skills-compatible tools +3. `.pi/skills/` — per-project, Pi-specific +4. `.agents/skills/` — per-project, shared (cwd walking upward) +5. bundled Pi packages + +Pick a scope: + +### User-global (every project on this machine) + +```bash +mkdir -p ~/.pi/agent/skills +cp -r /path/to/structsense-skills ~/.pi/agent/skills/ +``` + +Or, if you want the skill available to **both Pi and other Agent-Skills tools** from one copy, use the shared location instead: + +```bash +mkdir -p ~/.agents/skills +cp -r /path/to/structsense-skills ~/.agents/skills/ +``` + +### Per-project (versioned with the repo) + +```bash +cd /path/to/your/project +mkdir -p .pi/skills +cp -r /path/to/structsense-skills .pi/skills/ +git add .pi/skills/structsense-skills && git commit -m "Add structsense-skills" +``` + +Project-scoped skills take precedence over user-global ones with the same name. + +> **If Pi is already running**, it won't see a skill you just copied in — Pi scans the search paths at startup and doesn't auto-detect new ones mid-session. Run `/reload` (reloads skills, extensions, keybindings, prompts, and context files in place) or fully restart Pi to pick it up. + +## 2. Verify it loaded + +In a Pi session, list available skills (the exact command may vary by version — try `/help` if `/skills` isn't recognized): + +``` +/skills +``` + +You should see `structsense-skills` with its description. If you don't: + +- The folder must be `structsense-skills/` with `SKILL.md` at its root (not nested in a subfolder). +- `SKILL.md` frontmatter MUST have both `name:` and `description:` (three dashes, key/value lines, three dashes). +- Make sure it's in one of the search paths in §1 — a typo like `~/.pi/skills/` (missing `agent/`) won't be found. +- If you added the skill while this Pi session was already open, run `/reload` (or restart Pi) — Pi only scans the search paths at startup, so newly copied skills aren't picked up until you reload. + +## 3. Invoke + +Pi lets you invoke a skill explicitly by name: + +``` +/skill:structsense-skills +``` + +…or implicitly, by describing the task and letting Pi pick it up: + +> "Extract all the gene mentions from this paper and map them to ontology IDs." + +> "Pull the tools and datasets out of these three READMEs as a structured table." + +You can also point Pi at a file directly when launching: + +```bash +pi @paper.txt "Use structsense-skills to extract entities and write the result as paper_final.json." +``` + +## 4. What Pi will do + +When invoked, Pi reads `SKILL.md`, then loads only the reference / prompt files relevant to the task (progressive disclosure). For NER on a biomedical paper, that's typically: + +- `references/ner-extraction.md` +- `prompts/extractor-ner-neuroscience.md` (or `extractor-ner-cns-cells.md`) +- `prompts/mask-recall-pass.md` +- `prompts/judge.md` +- `references/ontology-mapping.md` +- `references/ner-models.md` + +For the Python pipeline driver, Pi calls `scripts/pipeline.py` via its built-in `bash` tool. Make sure your project's working directory has Python + dependencies available (`pip install requests openai anthropic json-repair jsonschema` — install the providers you actually use). For the optional HF NER ensemble, also `pip install transformers torch`. + +## 5. API keys + +Pi reads provider keys from your shell environment, or you can authenticate in-session with `/login` (which also supports Claude Pro/Max, ChatGPT, and Copilot subscription auth and stores credentials at `~/.pi/agent/auth.json`). + +Set keys in the shell **before** launching Pi, or in a `.env` you source: + +```bash +export OPENROUTER_API_KEY=sk-or-v1-... +export ANTHROPIC_API_KEY=sk-ant-... +export OPENAI_API_KEY=sk-... +export GEMINI_API_KEY=... +export BIOPORTAL_API_KEY=... +``` + +Pi inherits the env from the shell that launched it. **Do not paste API keys into chat.** + +Note: the pipeline itself reads `OPENROUTER_API_KEY` / `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` / `BIOPORTAL_API_KEY` from the environment when it shells out — Pi's own `/login` auth (in `auth.json`) covers Pi's conversation model, but `scripts/pipeline.py` still needs the keys in the env to call its extractor/judge models. Set both if you want Pi-the-agent and the pipeline to use different providers. + +## 6. Picking models inside Pi + +When Pi calls `scripts/pipeline.py`, it can pass model strings just like any other CLI invocation: + +```bash +python -m scripts.pipeline \ + --task ner --input paper.txt \ + --extractor openrouter/anthropic/claude-sonnet-4-6 \ + --judge openrouter/openai/gpt-4o-mini \ + --mapper local --mapper-url http://localhost:8000 \ + --ner-profile biomedical_broad +``` + +Tell Pi in plain English — *"use claude-sonnet-4-6 for extraction and gpt-4o-mini for the judge, with the cns_cells NER profile"* — and it translates that into the right flags. The pipeline is model-agnostic, so Pi (the conversation model) and the pipeline's extractor/judge can be entirely different providers. + +## 7. Working with the local concept-mapping service + +Because Pi runs `bash` on your machine (no cloud sandbox in between), the skill's local hybrid mapper at `http://localhost:8000` is reachable directly — same as Claude Code, unlike the Claude Desktop / claude.ai sandbox case (see [`claude-desktop.md`](claude-desktop.md)). + +Verify the [search_hybrid](https://github.com/sensein/search_hybrid) service is up before invoking the skill: + +```bash +curl -s http://localhost:8000/docs > /dev/null && echo OK +``` + +If it's unreachable, the pipeline cascades local → BioPortal → asks you for an alternate URL → hard-stops (it will not silently skip alignment). + +## 8. Project-level instructions (optional) + +If you want Pi to *always* follow certain conventions when this skill runs (e.g. preferred default models, or "always write the canonical shape"), add them to an `AGENTS.md` at your project root — Pi reads `AGENTS.md` (and `CLAUDE.md`) walking from cwd up through parent dirs, plus a global `~/.pi/agent/AGENTS.md`. Example: + +```markdown +# Project conventions + +When using the structsense-skills skill: +- Default extractor: openrouter/anthropic/claude-sonnet-4-6; default judge: gpt-4o-mini. +- Always emit the canonical shape (top-level source_metadata, entities[] one per + occurrence, entities_grouped[], stats). Never put paper_title/doi on each entity. +- For biomedical papers, run mask-recall after pass-1. +``` + +This is optional — the skill's own `SKILL.md` already encodes these rules. Use `AGENTS.md` only for per-project overrides. + +## 9. MCP (optional, not built in) + +Pi ships with **no native MCP support** — by design ("build CLI tools with READMEs, or an extension that adds MCP"). For this skill that doesn't matter: the Agent Skills path above is the first-class integration and is all you need. + +If you specifically want to expose the pipeline as an MCP tool (e.g. to share one running instance across Pi, Cursor, and Claude Code), you can: + +- Build the MCP server from [`mcp-server.md`](mcp-server.md), then +- Add MCP support to Pi via the community **`pi-mcp-adapter`** extension (third-party, by nicobailon — install separately; config paths and options are version-dependent, so check its README), and configure the server in the adapter's `mcp.json`. + +Treat this as an ecosystem add-on, not core Pi behavior. For a single-machine workflow, the Skills path (§1) is simpler and recommended. + +## 10. Updating the skill + +When you pull a new version: + +```bash +cd ~/.pi/agent/skills/structsense-skills # or .pi/skills/structsense-skills, or ~/.agents/skills/... +git pull # if it's a checkout +# OR +rm -rf structsense-skills/ && cp -r /path/to/new/structsense-skills . +``` + +Check the version: + +```bash +head -5 ~/.pi/agent/skills/structsense-skills/SKILL.md # look for version: 0.4.0 or higher +``` + +If you have output JSON from an older version, bring it up to the canonical shape (idempotent): + +```bash +python -m scripts.normalize_result old_output.json --input paper.txt \ + --llm-model openrouter/anthropic/claude-sonnet-4-6 +``` + +## 11. Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| Skill not listed | Wrong folder location or missing frontmatter | Confirm it's under a §1 search path; check `name:` + `description:` in `SKILL.md`. | +| `/skills` not recognized | Command name differs by version | Run `/help` to find the skill-listing / `/skill:` command for your build. | +| Pi ignores the skill in conversation | No trigger detected | Invoke explicitly with `/skill:structsense-skills`, or name the task ("extract entities", "map to ontologies"). | +| Pipeline can't find the mapping service | `search_hybrid` not running on the expected port | Start it, verify with `curl http://localhost:8000/docs`, then re-run. | +| HF NER models fail ("transformers not installed") | `transformers` missing | `pip install transformers torch`, or omit `--ner-profile` to skip the ensemble. | +| Pipeline auth fails even after `/login` | `/login` auths Pi's model, not the pipeline subprocess | Export `OPENROUTER_API_KEY` / `ANTHROPIC_API_KEY` etc. in the shell that launched Pi (§5). | +| Output has `paper_title`/`doi` on every entity | Legacy shape | `python -m scripts.normalize_result --input --llm-model ` — idempotent. | diff --git a/skills/structsense-skills/examples/ner-example.md b/skills/structsense-skills/examples/ner-example.md new file mode 100644 index 0000000..3b72b5f --- /dev/null +++ b/skills/structsense-skills/examples/ner-example.md @@ -0,0 +1,204 @@ +# Worked example — neuroscience NER end-to-end + +Walks through extracting, aligning, and judging a short neuroscience passage. Uses the **neuroscience** NER variant (broad coverage) and OLS for ontology mapping (no API key). + +## Source text + +``` +Title: Parvalbumin interneurons in the medial prefrontal cortex shape working memory in mice +DOI: 10.1234/example.fake + +We recorded from fast-spiking parvalbumin (Pvalb)-expressing interneurons in the mouse +medial prefrontal cortex (mPFC) during a delayed alternation task. Pvalb interneurons +exhibited elevated firing rates during the delay period (mean = 28 Hz, p < 0.001). +Optogenetic silencing of Pvalb interneurons impaired working memory performance, while +silencing Sst interneurons had no effect. +``` + +## Stage 1 — extractor + +System prompt: `prompts/extractor-ner-neuroscience.md`. + +```python +from scripts.pipeline import extract +extraction = extract( + text=source, + model="openrouter/anthropic/claude-sonnet-4-6", + task="ner", + metadata={"paper_title": "Parvalbumin interneurons …", "doi": "10.1234/example.fake"}, + chunk_size=2000, max_workers=1, +) +``` + +Expected output (abbreviated). Note: `paper_title` and `doi` appear ONCE at the top under `source_metadata`, not on every entity. + +```jsonc +{ + "source_metadata": { + "paper_title": "Parvalbumin interneurons in the medial prefrontal cortex shape working memory in mice", + "doi": "10.1234/example.fake", + "source_path": "paper.txt" + }, + "entities": [ + {"entity": "parvalbumin", "label": "Protein", + "sentence": "We recorded from fast-spiking parvalbumin (Pvalb)-expressing interneurons in the mouse medial prefrontal cortex (mPFC) during a delayed alternation task.", + "start": 24, "end": 35, "paper_location": null}, + + {"entity": "Pvalb", "label": "Gene", + "sentence": "...", "start": 37, "end": 42, "...": "..."}, + + {"entity": "mouse", "label": "Species", + "sentence": "...", "start": 75, "end": 80, "...": "..."}, + + {"entity": "medial prefrontal cortex", "label": "BrainRegion", + "sentence": "...", "start": 81, "end": 105, "...": "..."}, + + {"entity": "mPFC", "label": "BrainRegion", + "sentence": "...", "start": 107, "end": 111, "...": "..."}, + + {"entity": "delayed alternation task", "label": "BehavioralAssay", + "sentence": "...", "start": 120, "end": 144, "...": "..."}, + + {"entity": "fast-spiking", "label": "Phenomenon", + "sentence": "...", "start": 11, "end": 23, "...": "..."}, + + {"entity": "Pvalb interneurons", "label": "CellType", + "sentence": "Pvalb interneurons exhibited elevated firing rates during the delay period (mean = 28 Hz, p < 0.001).", + "start": 146, "end": 164, "...": "..."}, + + {"entity": "firing rate", "label": "Measurement", + "sentence": "...", "start": 184, "end": 195, "...": "..."}, + + {"entity": "Optogenetic silencing", "label": "Method", + "sentence": "...", "start": 246, "end": 267, "...": "..."}, + + {"entity": "working memory", "label": "Phenomenon", + "sentence": "...", "start": 297, "end": 311, "...": "..."}, + + {"entity": "Sst interneurons", "label": "CellType", + "sentence": "...", "start": 340, "end": 356, "...": "..."} + ], + "key_terms": [ + {"term": "delay period", "sentence": "...", "start": 218, "end": 230} + ], + "task_type": "ner" +} +``` + +The extractor emits both the long form ("parvalbumin") and the symbol ("Pvalb"), per rule 6 in the neuroscience prompt. Same for "medial prefrontal cortex" / "mPFC". Notice the disambiguation: "parvalbumin" the protein is `Protein`; "Pvalb" used as a gene-symbol-style marker is `Gene`; "Pvalb interneurons" used as a cell descriptor is `CellType`. + +## Stage 2 — alignment (direct OLS tool call) + +```python +from scripts.pipeline import align_direct + +aligned = align_direct(extraction, mapper_backend="ols", + ontologies_for_label=lambda lbl: { + "Gene": ["hgnc", "ncbigene"], + "Protein": ["pr", "uniprot"], + "Species": ["ncbitaxon"], + "BrainRegion": ["uberon"], + "CellType": ["cl"], + "BehavioralAssay": ["obi", "efo"], + "Method": ["obi"], + "Phenomenon": ["nifstd", "go"], + "Measurement": ["obi"], + }.get(lbl, [])) +``` + +Sample entity after alignment: + +```jsonc +{ + "entity": "medial prefrontal cortex", "label": "BrainRegion", + "start": 81, "end": 105, "sentence": "...", + "ontology_id": "http://purl.obolibrary.org/obo/UBERON_0009834", + "ontology_label": "medial prefrontal cortex", + "ontology": "UBERON", + "concept_mapping_provenance": "tool", + "alignment_method": "direct_tool_call" +} +``` + +And the gene example: + +```jsonc +{ + "entity": "Pvalb", "label": "Gene", + "ontology_id": "http://identifiers.org/hgnc/9704", + "ontology_label": "PVALB", + "ontology": "HGNC", + "concept_mapping_provenance": "tool", + "alignment_method": "direct_tool_call" +} +``` + +## Stage 3 — judge + +```python +from scripts.pipeline import judge +judged = judge(aligned, source_text=source, + model="openrouter/openai/gpt-4o-mini", + max_workers=4) +``` + +Sample entity after judging: + +```jsonc +{ + "entity": "Pvalb", "label": "Gene", + "ontology_id": "http://identifiers.org/hgnc/9704", + "ontology_label": "PVALB", + "ontology": "HGNC", + "concept_mapping_provenance": "tool", + "judge_score": 0.95, + "remarks": "PVALB is the canonical gene symbol; surface form matches.", + "judge_method": "llm" +} +``` + +A lower-confidence case might look like: + +```jsonc +{ + "entity": "fast-spiking", "label": "Phenomenon", + "ontology_id": null, "ontology_label": null, "ontology": null, + "concept_mapping_provenance": "unmapped", + "judge_score": 0.40, + "remarks": "Term refers to firing pattern; NIFSTD has no exact match, OBI rejected. Consider label=EphysProperty.", + "judge_method": "llm" +} +``` + +This is the kind of signal you act on: re-run with the **CNS-cells** variant and "fast-spiking" becomes an `EphysProperty` with a proper alignment target. + +## Running it end-to-end + +```bash +# from the repo containing structsense-skills/ +export OPENROUTER_API_KEY=sk-or-v1-... +python -m structsense-skills.scripts.pipeline \ + --task ner --input paper.txt \ + --extractor openrouter/anthropic/claude-sonnet-4-6 \ + --judge openrouter/openai/gpt-4o-mini \ + --mapper ols \ + --chunk-size 2000 --max-workers 8 \ + --out result.json +``` + +## What to inspect + +- **Provenance**: every item carries `concept_mapping_provenance` (`tool` / `llm_knowledge` / `unmapped` / `skipped`) and `alignment_method` (`direct_tool_call` / `llm_agent` / `skipped`). +- **Span integrity**: `source[start:end] == entity` for every item. Drop or repair items that fail. +- **Score distribution**: a healthy run has spread (not all 0.5). All-1.0 means the judge is mis-calibrated; all-low means the alignment is upstream-wrong. +- **Unmapped rate**: > 20% suggests wrong ontology routing for some label. + +## Variants of this example + +Same source text, different extractor variant: + +| Variant | What changes | +|---|---| +| `extractor-ner-general.md` | Most labels collapse to `Other`; misses Gene/Protein distinction. Useful as a baseline. | +| `extractor-ner-neuroscience.md` | The version run above. Broad domain coverage. | +| `extractor-ner-cns-cells.md` | "Pvalb interneurons" → `CellType` with `cell_context.lineage_markers: ["Pvalb"]`. "fast-spiking" → `EphysProperty`. "mPFC" → `BrainRegion`. Atlas / profiling-method fields populated when present. | diff --git a/skills/structsense-skills/examples/reproschema-example.md b/skills/structsense-skills/examples/reproschema-example.md new file mode 100644 index 0000000..8eac9db --- /dev/null +++ b/skills/structsense-skills/examples/reproschema-example.md @@ -0,0 +1,223 @@ +# Worked example — PDF questionnaire → ReproSchema + +Converts a paper-based questionnaire PDF into structured JSON matching a simplified ReproSchema-style activity + items schema. Uses the **structured extraction** variant. + +## The target schema (locked up front) + +```json +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["activity", "items"], + "properties": { + "activity": { + "type": "object", + "required": ["id", "prefLabel", "description"], + "properties": { + "id": { "type": "string", "pattern": "^[A-Z][A-Za-z0-9_-]*$" }, + "prefLabel": { "type": "string" }, + "description": { "type": ["string", "null"] }, + "preamble": { "type": ["string", "null"] }, + "citation": { "type": ["string", "null"] } + } + }, + "items": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "question", "inputType", "valueRequired"], + "properties": { + "id": { "type": "string", "pattern": "^Q[0-9]+$" }, + "question": { "type": "string" }, + "prefLabel": { "type": ["string", "null"] }, + "inputType": { "type": "string", + "enum": ["radio", "text", "number", "slider"] }, + "valueRequired": { "type": "boolean" }, + "responseOptions": { + "type": "array", + "items": { + "type": "object", + "required": ["value", "name"], + "properties": { + "value": { "type": "integer" }, + "name": { "type": "string" } + } + } + }, + "scoring": { + "type": ["object", "null"], + "properties": { + "type": { "type": "string", "enum": ["sum", "mean", "custom"] }, + "weights": { "type": "array", "items": { "type": "number" } } + } + } + } + } + } + } +} +``` + +Save it as `schemas/reproschema-questionnaire.schema.json`. + +## Source (excerpt from a 12-page Mood and Feelings Questionnaire PDF) + +``` +MOOD AND FEELINGS QUESTIONNAIRE: Long Version + +Below is a list of feelings or behaviors that may apply to you. +For each one, please mark how often it was true for you in the last two +weeks: 0 = not true, 1 = sometimes true, 2 = true. + + 1. I felt miserable or unhappy. [0] [1] [2] + 2. I didn't enjoy anything at all. [0] [1] [2] + 3. I felt so tired I just sat around. [0] [1] [2] + ... +``` + +## Stage 1 — extractor (schema-driven) + +System prompt: `prompts/extractor-structured.md`, with the schema inline. + +```python +import json +from pathlib import Path +from scripts.llm_client import call +from scripts.json_repair import parse_or_repair + +schema_text = Path("schemas/reproschema-questionnaire.schema.json").read_text() +system = Path("prompts/extractor-structured.md").read_text() # the file's ## System block +# (or use scripts.pipeline._load_prompt("extractor-structured")) +system = system.format(json_schema=schema_text, field_hints=""" +- activity.id: short identifier built from title initials, ignoring stopwords + ('AND', 'THE'). e.g. 'MOOD AND FEELINGS QUESTIONNAIRE' -> 'MFQ-long'. +- items[].inputType: one of [radio | text | number | slider]. +- items[].responseOptions[].value: integer score, lowest first. +- preamble: the instruction paragraph before the first question. +""") + +raw = call( + model="openrouter/anthropic/claude-sonnet-4-6", + system=system, + user=f"SOURCE DOCUMENT:\n<<<\n{pdf_text}\n>>>\n\nADDITIONAL CONTEXT:\n{{}}", + json_mode=True, + temperature=0, +) +data = parse_or_repair(raw) +``` + +Expected output: + +```jsonc +{ + "activity": { + "id": "MFQ-long", + "prefLabel": "Mood and Feelings Questionnaire — Long Version", + "description": null, + "preamble": "Below is a list of feelings or behaviors that may apply to you. For each one, please mark how often it was true for you in the last two weeks: 0 = not true, 1 = sometimes true, 2 = true.", + "citation": null + }, + "items": [ + { + "id": "Q1", + "question": "I felt miserable or unhappy.", + "prefLabel": "miserable_unhappy", + "inputType": "radio", + "valueRequired": true, + "responseOptions": [ + { "value": 0, "name": "not true" }, + { "value": 1, "name": "sometimes true" }, + { "value": 2, "name": "true" } + ], + "scoring": null + }, + { + "id": "Q2", + "question": "I didn't enjoy anything at all.", + "prefLabel": "no_enjoyment", + "inputType": "radio", + "valueRequired": true, + "responseOptions": [ + { "value": 0, "name": "not true" }, + { "value": 1, "name": "sometimes true" }, + { "value": 2, "name": "true" } + ], + "scoring": null + } + // ... 31 more items + ] +} +``` + +## Stage 2 — validate + repair + +```python +import jsonschema +from scripts.json_repair import repair_to_schema + +schema = json.loads(schema_text) +fixed, errors = repair_to_schema( + data, schema, + llm_call=lambda prompt: call( + model="openrouter/openai/gpt-4o-mini", + system="You output strict JSON only.", + user=prompt, json_mode=True, temperature=0, + ), + max_attempts=2, +) +if errors: + print("Repair history:", errors) +jsonschema.validate(fixed, schema) +``` + +Typical repair errors caught here: + +- `items[2].responseOptions[1].value: expected integer, got string "1"` → `value: 1` (int). +- `items[5].inputType: 'Radio' is not one of ['radio','text','number','slider']` → `inputType: "radio"`. +- `activity.id: 'MFQ_long' does not match '^[A-Z][A-Za-z0-9_-]*$'` (underscore not in pattern? this one **does** match — included as a reminder to always re-test). + +## Stage 3 — judge (faithfulness + schema validity) + +Two judgments: + +1. `schema_valid`: did the final JSON validate? (deterministic) +2. `field_judgments`: per-field LLM faithfulness check. + +```jsonc +{ + "data": { "...": "..." }, + "schema_valid": true, + "field_judgments": { + "activity.prefLabel": {"score": 1.0, "remarks": "Title taken verbatim from source."}, + "activity.preamble": {"score": 0.85, "remarks": "Slightly compressed whitespace; semantics preserved."}, + "items": {"score": 1.0, "remarks": "Counted 33 items matching the questionnaire structure."}, + "items[*].responseOptions": {"score": 1.0, "remarks": "All items share the 0/1/2 scale shown in the preamble."} + } +} +``` + +If any `score < 0.6`, surface that field in the human-feedback stage for review. + +## Running it end-to-end + +```python +# Pseudocode — wire your own driver around scripts/pipeline.py and the repair helpers +from scripts import pipeline +result = pipeline.run( + text=pdf_text, + task="structured", # uses extractor-structured.md + extractor_model="openrouter/anthropic/claude-sonnet-4-6", + mapper_backend=None, # no ontology mapping for this schema + judge_model="openrouter/openai/gpt-4o-mini", + chunk_size=8000, # questionnaires fit in larger chunks + max_workers=4, + skip_judge=False, +) +``` + +## Common gotchas + +- **Tables in PDFs.** A questionnaire's repeated response columns get rendered weirdly by PDF text extractors. Preprocess: collapse "[0] [1] [2]" patterns into the structured shape before running the extractor, or include an example of the malformed PDF table in the field hints. +- **Numbering schemes.** Some questionnaires number items "Q1, Q2…", others "1., 2.…", others by domain ("D1.1, D1.2…"). Pin the format in `field_hints`. +- **Subscales.** A questionnaire often has computed subscales (e.g. items 1, 4, 7 → "anhedonia"). The schema above doesn't model that; add a `computedScores` field if you need it. +- **Conditional logic.** Some items only appear if a previous answer matched a condition. Add `conditionalLogic` to the schema and have the extractor capture the rule as a parseable expression. diff --git a/skills/structsense-skills/examples/resource-example.md b/skills/structsense-skills/examples/resource-example.md new file mode 100644 index 0000000..bb3f03e --- /dev/null +++ b/skills/structsense-skills/examples/resource-example.md @@ -0,0 +1,142 @@ +# Worked example — research resource extraction end-to-end + +Extracts the one primary resource described in a paper, with other tools/datasets/models as `mentions`. + +## Source text + +``` +Title: SuperAnimal-Quadruped: a foundation model for quadruped pose estimation + +We introduce SuperAnimal-Quadruped, a pre-trained pose-estimation model +released through the DeepLabCut Model Zoo +(https://deeplabcut.github.io/DeepLabCut/docs/ModelZoo.html). The model is +trained on the Quadruped-80K dataset, which we curated from AnimalPose, +AP-10K, and additional in-house labelling. SuperAnimal-Quadruped builds on +the DeepLabCut framework and is evaluated against the AnimalPose and AP-10K +benchmarks. Target species include mice, rats, dogs, and horses. +``` + +## Stage 1 — extractor + +System prompt: `prompts/extractor-resource.md`. + +Expected output: + +```jsonc +{ + "extracted_resources": { + "1": [{ + "name": "SuperAnimal-Quadruped", + "description": "Pre-trained pose-estimation foundation model for quadrupeds, released through the DeepLabCut Model Zoo.", + "type": "Model", + "category": "Pose Estimation", + "target": "Animal", + "specific_target": "Mice, Rats, Dogs, Horses", + "url": "https://deeplabcut.github.io/DeepLabCut/docs/ModelZoo.html", + "mentions": { + "datasets": ["Quadruped-80K", "AnimalPose", "AP-10K"], + "benchmarks": ["AnimalPose", "AP-10K"], + "models": ["DeepLabCut"], + "papers": [] + } + }] + }, + "task_type": "resource" +} +``` + +Key observations: + +- The **primary** resource is the new model SuperAnimal-Quadruped, **not** the DeepLabCut framework (which is a mention). +- The dataset Quadruped-80K, although introduced in this paper too, is also placed under `mentions.datasets`. If you wanted it as a sibling top-level resource, prompt-side hint: "Emit multiple top-level resources only when the source introduces multiple primary artifacts; for this paper, the model is the primary resource." +- `specific_target` keeps the order from the source ("mice, rats, dogs, horses") for the alignment stage to split on commas. + +## Stage 2 — alignment (resource variant) + +Use the resource-specific alignment variant (see `prompts/alignment.md` → "Resource-specific variation"). Alignment does **not** add `ontology_id` to the resource itself — it adds `mapped_target_concept` and `mapped_specific_target_concept`. + +```jsonc +{ + "name": "SuperAnimal-Quadruped", + "type": "Model", + "target": "Animal", + "mapped_target_concept": [ + { "id": "http://purl.obolibrary.org/obo/BTO_0000042", + "label": "animal", "ontology": "BTO" } + ], + "specific_target": "Mice, Rats, Dogs, Horses", + "mapped_specific_target_concept": [ + { "specific_target": "Mice", + "mapped_target_concept": { + "id": "NCBITaxon:10090", "label": "Mus musculus", "ontology": "NCBITaxon" + } + }, + { "specific_target": "Rats", + "mapped_target_concept": { + "id": "NCBITaxon:10116", "label": "Rattus norvegicus", "ontology": "NCBITaxon" + } + }, + { "specific_target": "Dogs", + "mapped_target_concept": { + "id": "NCBITaxon:9615", "label": "Canis lupus familiaris", "ontology": "NCBITaxon" + } + }, + { "specific_target": "Horses", + "mapped_target_concept": { + "id": "NCBITaxon:9796", "label": "Equus caballus", "ontology": "NCBITaxon" + } + } + ], + "url": "https://deeplabcut.github.io/DeepLabCut/docs/ModelZoo.html", + "mentions": { "...": "..." }, + "concept_mapping_provenance": "tool", + "alignment_method": "direct_tool_call" +} +``` + +`name`, `description`, `type`, `category`, `url`, and `mentions` are preserved verbatim. Only the two mapping fields are added. + +## Stage 3 — judge + +```jsonc +{ + "judge_resource": { + "1": [{ + "name": "SuperAnimal-Quadruped", + "type": "Model", + "judge_score": 0.93, + "remarks": "Type=Model is correct; mapped species IDs are canonical NCBITaxon. Description faithful to source.", + "judge_method": "llm" + }] + } +} +``` + +The judge scores **only the primary resource** under `"1"`, not the items inside `mentions`. + +## Running it end-to-end + +```bash +python -m structsense-skills.scripts.pipeline \ + --task resource --input paper.txt \ + --extractor openrouter/anthropic/claude-sonnet-4-6 \ + --judge openrouter/openai/gpt-4o-mini \ + --mapper ols \ + --out result.json +``` + +## Failure mode walkthrough + +A common bug: the model dumps every cited dataset and tool as a sibling top-level resource, leaving `mentions` empty. Symptom in the output: + +```jsonc +{ + "extracted_resources": { + "1": [{ "name": "SuperAnimal-Quadruped", "mentions": {} }], + "2": [{ "name": "DeepLabCut", "mentions": {} }], + "3": [{ "name": "AnimalPose", "mentions": {} }] + } +} +``` + +Fix: strengthen the extractor prompt's "primary resource" rule (see `prompts/extractor-resource.md` → "Common failure modes"). Reject and reprompt when `len(extracted_resources) > 1` and `mentions` is empty across all of them — that's the heuristic signature of this bug. diff --git a/skills/structsense-skills/prompts/alignment-via-http.md b/skills/structsense-skills/prompts/alignment-via-http.md new file mode 100644 index 0000000..a3dc9ee --- /dev/null +++ b/skills/structsense-skills/prompts/alignment-via-http.md @@ -0,0 +1,232 @@ +# Alignment via direct HTTP (no Python client needed) + +Use this when: +- You have a list of extracted entities/key_terms ready to align. +- The local hybrid mapping service is running and reachable from your runtime + (verify with the **probe step** below). +- You either don't have the Python `scripts/local_hybrid_map.py` available, or + you want the lowest-overhead path: just curl. + +## Step 0 — runtime reachability probe (DO THIS FIRST) + +Before declaring the mapper "unavailable", you MUST attempt at least one probe. +Do not skip this step. Many "no mapper available" messages are wrong because +the model never tried. + +```bash +# Adjust the URL if needed: ports 8000/8001/8080/9000 are all common. +# Set MAPPER_URL once and reuse it. +MAPPER_URL=${MAPPER_URL:-http://localhost:8000} + +# Probe /docs (every FastAPI service exposes it). HTTP 200 = service is up. +curl -s -o /dev/null -w '%{http_code}\n' "${MAPPER_URL}/docs" +``` + +| Result | Meaning | Action | +|---|---|---| +| `200` | Service is up; you can proceed. | Continue to Step 1. | +| Any 4xx / 5xx | Service is reachable but `/docs` isn't where expected. | Try `${MAPPER_URL}/health`, or open `${MAPPER_URL}/openapi.json` to inspect routes. | +| Connection refused / timeout | Either the service isn't running OR your runtime can't reach `localhost`. | See **Runtime reachability** below. | + +### Runtime reachability — when "localhost" doesn't mean the user's machine + +| Runtime | Can it reach the user's localhost? | What to do if not | +|---|---|---| +| **Claude Code (CLI)** | ✅ Yes — Bash runs on the user's machine. | — | +| **Cursor / Codeium with local agents** | ✅ Usually yes. | — | +| **Claude.ai web app** | ❌ No — Claude runs in Anthropic's cloud; cannot dial the user's machine. | Ask the user to run the **MCP bridge** described in `connecting/mcp-server.md`, OR run the pipeline locally and paste the result back. | +| **ChatGPT (custom GPT in web)** | ❌ No. | Same — needs an HTTP Action pointing at a public URL, see `connecting/custom-gpt.md`. | +| **Anthropic Skills hosted runtime** | ❌ No (sandboxed; no outbound network to localhost). | Same — MCP bridge or public URL. | + +If the user is in the web app and the mapper is "running on my machine", the +only way to use it is to set up a tunnel (`ngrok`, `cloudflared`, Tailscale) +that exposes `localhost:8000` to a URL the cloud LLM can reach, OR an MCP +bridge. + +## Step 1 — call `/map/batch` + +The API expects **`text: [{text, context?}]`** — NOT `terms: [...]`. The +`context` field is optional but dramatically improves disambiguation for +acronyms and ambiguous surface forms. + +```bash +curl -s -X POST "${MAPPER_URL}/map/batch" \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "max_results": 5, + "text": [ + {"text": "kidney disease", "context": "progressive decline in GFR"}, + {"text": "T2DM", "context": "type 2 diabetes with insulin resistance"}, + {"text": "astrocyte"} + ] + }' +``` + +**Always provide `context`** when you have it. For NER, the natural choice is +the entity's containing sentence (which the extractor already emits as +`sentence`). For resources, pass `description`. + +### Batching + +One POST may carry hundreds to thousands of items. Don't loop one-at-a-time. + +Practical batch size: 500 items per request is a safe default. If a request +exceeds 30s (`LOCAL_CONCEPT_MAPPING_TIMEOUT`), split in half and retry. + +## Step 2 — parse the response + +The service returns (verified against the reference deployment): + +```jsonc +{ + "query": "...", + "type": "batch", + "results": { + "kidney disease": [ + { + "rank": 1, + "ontology_id": "http://purl.obolibrary.org/obo/HP_0012622", + "ontology_label": "Chronic kidney disease", + "ontology": "CIDO", + "original_score": 0.249, + "llm_score": 0.0, + "late_interaction_score": 1.0, + "final_score": 1.0 + }, + // ... up to max_results candidates + ], + "T2DM": [ ... ], + "astrocyte": [ ... ] + } +} +``` + +Important: `results` is a **dict keyed by the input `text`**, not a list. Look +up each input term by name. Use `final_score` (not `score`). + +## Step 3 — pick the top candidate per term + +The candidates are pre-ranked. The first one (`rank: 1`) is usually correct. +For high-confidence pipelines, also keep candidates 2–3 in case `judge` later +needs to reconsider. + +## Step 4 — merge into the extraction + +For each entity in the extraction's `entities[]` (and `key_terms[]`), set: + +```jsonc +{ + // existing fields preserved + "entity": "kidney disease", + "label": "Disease", + "sentence": "...", + "start": 42, "end": 56, + + // ADD these four + "ontology_id": "http://purl.obolibrary.org/obo/HP_0012622", + "ontology_label": "Chronic kidney disease", + "ontology": "CIDO", + "concept_mapping_provenance": "tool", + "alignment_method": "direct_tool_call" +} +``` + +Do not modify any pre-existing fields. Do not drop items that came back +unmapped — instead set: + +```jsonc +{ + "ontology_id": null, "ontology_label": null, "ontology": null, + "concept_mapping_provenance": "unmapped", + "alignment_method": "direct_tool_call" +} +``` + +## Step 5 — preserve provenance throughout + +`concept_mapping_provenance: "tool"` is the signal downstream consumers use +to trust an ontology mapping. The only legal values are: + +- `"tool"` — tool returned a match for this term +- `"unmapped"` — tool returned no candidates +- `"skipped"` — alignment stage was explicitly disabled + +**Never `"llm_knowledge"`.** The skill's policy is tool-backed mappings +only. The post-processor (`scripts/normalize_result.py` → +`scripts/iri_validation.py`) demotes any item with `llm_knowledge` +provenance to `unmapped` and labels it with `alignment_method: +"validation_failed"` so the audit trail is clear. + +Do not paper over a missing/unreachable mapper by inventing IRIs from +prior knowledge. If the mapper is unreachable, the right action is to +SURFACE that fact to the user (see "When the service is unreachable" below) +and let them decide between (a) bringing it up, (b) configuring BioPortal, +or (c) explicitly opting out of alignment. + +## When the service is unreachable + +Decide between three fallbacks in this order: + +1. **Ask the user for an alternative URL.** Ports often differ from the default. + ``` + I tried http://localhost:8000/docs — connection refused. What URL is + your local mapping service running on? (e.g. http://localhost:8001, + https://internal.example.com/concept-map/, or skip alignment.) + ``` +2. **Try BioPortal as a fallback** (requires `BIOPORTAL_API_KEY`). See + `prompts/alignment.md` and `scripts/bioportal_map.py`. +3. **Skip alignment** with `concept_mapping_provenance: "skipped"` on every + item. The pipeline still completes; only ontology fields are unpopulated. + +Do not silently invent IRIs. Hallucinated `http://purl.obolibrary.org/obo/...` +strings have done real damage to downstream datasets. The post-processor +will catch them via `scripts/iri_validation.py` and demote them, but you +should not produce them in the first place. + +## Worked example — full alignment pass from a bash session + +```bash +set -e +MAPPER_URL=${MAPPER_URL:-http://localhost:8000} + +# Probe +test "$(curl -s -o /dev/null -w '%{http_code}' "${MAPPER_URL}/docs")" = "200" + +# Build the payload from extraction.json (entities + their sentence as context) +jq '{ + max_results: 5, + text: [.entities[] | {text: .entity, context: .sentence}] +}' extraction.json > /tmp/batch.json + +# Call /map/batch +curl -s -X POST "${MAPPER_URL}/map/batch" \ + -H 'Content-Type: application/json' \ + -d @/tmp/batch.json > /tmp/mappings.json + +# Merge mappings back into the extraction +jq ' + .entities |= map( + . as $e + | (input.results[$e.entity] // [])[0] as $top + | if $top + then . + { + ontology_id: $top.ontology_id, + ontology_label: $top.ontology_label, + ontology: $top.ontology, + concept_mapping_provenance: "tool", + alignment_method: "direct_tool_call" + } + else . + { + ontology_id: null, ontology_label: null, ontology: null, + concept_mapping_provenance: "unmapped", + alignment_method: "direct_tool_call" + } + end + ) +' extraction.json /tmp/mappings.json > aligned.json +``` + +This whole flow takes ~1–3 seconds on a well-provisioned service for ~1000 +entities. Compare to ~30+ minutes if you run alignment through an LLM agent +loop. diff --git a/skills/structsense-skills/prompts/alignment.md b/skills/structsense-skills/prompts/alignment.md new file mode 100644 index 0000000..d203608 --- /dev/null +++ b/skills/structsense-skills/prompts/alignment.md @@ -0,0 +1,123 @@ +# Alignment prompt — ontology mapping + +Use this prompt only when running alignment **via an LLM with a tool**. If you have a batch concept-mapping endpoint, call it directly instead and skip the LLM (see `references/ontology-mapping.md` → "Backend 3" and the direct-tool-call section). + +## System + +``` +You add ontology mappings to a JSON document of extracted entities/terms. + +INPUT +A JSON object with one or more lists of items (entities, key_terms, resources). +Each item has at minimum a surface form (entity / term / name). + +YOUR JOB +For each item: +1. Call the Concept Mapping Tool with the surface form to look up the + best-matching ontology concept. +2. **If the tool returns no result, set the mapping to unmapped.** + Do NOT use your own knowledge to fabricate an IRI. Hallucinated IRIs + are the single most damaging downstream defect this pipeline can + produce; the policy is **tool-backed mappings only**. +3. Add these four fields to the item: + - ontology_id: OR null if unmapped + - ontology_label: OR null if unmapped + - ontology: OR null if unmapped + - concept_mapping_provenance: + "tool" — tool returned a match (use this) + "unmapped" — tool returned nothing (use this when (1) above had no hits) + "skipped" — alignment stage was explicitly disabled by the operator + NEVER "llm_knowledge". The post-processor will reject any item with + this value and treat it as unmapped. + +PRESERVE every existing field. Do not remove, rename, or rewrite anything +already in the input. ADD ONLY the four fields above. + +OUTPUT +Strict JSON with the same top-level structure as the input. +No prose, no markdown fences. + +If a particular item cannot be mapped at all, set + ontology_id: null, ontology_label: null, ontology: null, + concept_mapping_provenance: "unmapped". + +Reminder: **never invent ontology IRIs.** It is better to leave 1000 +items as "unmapped" than to ship 10 hallucinated `purl.obolibrary.org/obo/…` +strings. The post-processor validates every IRI structurally and rejects +fabrications. + +If you cannot comply, output {"error": ""}. +``` + +## User + +``` +EXTRACTED JSON TO ALIGN: +{extracted_structured_information} + +ONTOLOGY HINTS (optional — restrict the search to these when applicable) +- Species / organism → NCBITaxon +- Anatomy / brain region → UBERON +- Cell type → CL +- Disease → MONDO +- Chemical / drug → CHEBI +- Tissue / cell line → BTO +- Method / assay → OBI, EFO +``` + +## Tool schema (what the model sees as the tool) + +When wiring this prompt to a tool-using model (Claude / GPT / etc.), expose: + +```json +{ + "name": "concept_mapping_tool", + "description": "Look up an ontology concept by free-text term. Returns the best matching concept's IRI, preferred label, and ontology shortname. Accepts a single term or a list of terms.", + "parameters": { + "type": "object", + "properties": { + "terms": { + "type": "array", + "items": {"type": "string"}, + "description": "One or more free-text terms to map." + }, + "ontologies": { + "type": "array", + "items": {"type": "string"}, + "description": "Optional ontology shortnames to restrict the search (e.g. ['UBERON', 'CL'])." + }, + "max_results": { + "type": "integer", + "default": 1, + "description": "Number of candidates per term (1-20)." + } + }, + "required": ["terms"] + } +} +``` + +**Encourage batching.** Add a system instruction: "Call the tool ONCE with all terms in a single `terms` array — do not call the tool per item." + +## Resource-specific variation + +For resource extraction, alignment doesn't add ontology fields to the resource itself. It adds nested `mapped_target_concept` and `mapped_specific_target_concept`. Use this variant: + +``` +For each resource item, additionally: +- Look up `target` and store result as `mapped_target_concept`: [{id, label, ontology}]. +- Split `specific_target` on commas, look up each, and store as + `mapped_specific_target_concept`: [{specific_target, mapped_target_concept: {id, label, ontology}}]. +- Do NOT add ontology_id/ontology_label to the resource itself. +- Do NOT rewrite name, type, category, url, mentions. +``` + +## Common failure modes + +| Symptom | Fix | +|---|---| +| Tool called one item at a time (slow) | Strengthen: "Batch ALL terms into a single tool call." | +| Invented IRIs (no tool call) | Validate every IRI against a regex of known prefixes; force `provenance: "llm_knowledge"` if no tool call was logged. | +| Existing fields rewritten | Strengthen: "ADD ONLY the four fields below. NEVER modify existing fields." Validate diff in parser. | +| Output wraps original in a new key | Strengthen: "Top-level structure of the output equals the top-level structure of the input." | +| Mapping picks wrong ontology (e.g. SNOMED disease for cell type) | Pass `ontologies=` hint based on item `label`. | diff --git a/skills/structsense-skills/prompts/extractor-ner-cns-cells.md b/skills/structsense-skills/prompts/extractor-ner-cns-cells.md new file mode 100644 index 0000000..858c94f --- /dev/null +++ b/skills/structsense-skills/prompts/extractor-ner-cns-cells.md @@ -0,0 +1,252 @@ +# Extractor prompt — CNS cell-focused NER + +Specialized for **central nervous system (CNS) cell information**: cell types (with subclass/subtype granularity), their morphologies, electrophysiological properties, molecular markers, layer/region location, and connectivity. Use this on single-cell atlases, cell-type taxonomy papers, patch-seq studies, scRNA-seq/snRNA-seq descriptions, BICCN-style cell census reports, or any text whose central subject is "what cells are in this part of the CNS." + +For broader neuroscience text, use `extractor-ner-neuroscience.md`. +For domain-agnostic text, use `extractor-ner-general.md`. + +## System + +``` +You are a CNS cell NER extractor. Your focus is cells of the central nervous +system (brain + spinal cord) — their types, subtypes, molecular markers, +morphologies, electrophysiology, location, and connectivity. + +You extract EXHAUSTIVELY. Recall matters more than precision. + +EXHAUSTIVENESS — READ CAREFULLY +- Extract EVERY occurrence. If "Pvalb interneuron" appears 25 times, emit + 25 entity items, each with its own distinct start/end pair. +- Do NOT deduplicate. Do NOT collapse repeat mentions. Do NOT emit "one + row per unique cell type." The post-processor handles dedup. +- Mentions in different sentences ARE different mentions — emit all. +- Mentions in the same sentence are different mentions — emit all. +- When a cell is described together with its markers, layer, morphology, + and ephys (e.g. "fast-spiking Pvalb basket cells in L2/3 of mPFC"), + emit a SEPARATE entity for EACH labelled element: + - "fast-spiking" → EphysProperty + - "Pvalb" → LineageMarker + - "Pvalb basket cells" → CellType (with cell_context populated) + - "basket cells" → MorphologyClass (if mentioned bare elsewhere) + - "L2/3" → CorticalLayer + - "mPFC" → BrainRegion + And repeat all of them every time the same construct recurs. +- Cluster names from atlases (e.g. "L5 IT MET-type", "Sst-Chodl", + "Pvalb-Vipr2") are CellSubtype mentions and must each be emitted. +- The expected count is HIGH. A single cell-atlas paper paragraph yields + 20–50 cell-relevant mentions; a full paper easily yields 500–2000+. + If your output feels short, you are missing mentions — re-scan. + +LABEL TAXONOMY (use these exactly; do NOT invent others) + +== Cell identity == +- CellClass High-level class: neuron, astrocyte, microglia, + oligodendrocyte, OPC, ependymal cell, vascular cell. +- CellType Recognized type: pyramidal neuron, basket cell, + Purkinje cell, layer 5 IT, parvalbumin interneuron. +- CellSubtype Finer subclass / cluster: L2/3 IT MET-type, Sst-Chodl, + Pvalb-Vipr2, "Glut-1 cluster 17". Use this for atlas + cluster names (Allen, BICCN, MapMyCells). +- LineageMarker A gene/protein used to define or label the cell: + Pvalb, Sst, Vip, Gad1, Slc17a7, Cux2, Rbp4, Pdgfra, + GFAP, Iba1, Olig2. + +== Morphology / structure == +- MorphologyClass Named morphological category: bipolar, basket, chandelier, + pyramidal, granule, fusiform, Martinotti. +- CellularStructure Subcellular: apical dendrite, basal dendrite, axon + initial segment, dendritic spine, soma, axon collateral. +- Synapse Synapse types involving this cell: glutamatergic synapse, + GABAergic synapse, axosomatic synapse. + +== Electrophysiology / function == +- EphysProperty Intrinsic electrophysiological property name: + fast-spiking, regular-spiking, burst-firing, + low-threshold spiking, accommodating, non-accommodating, + input resistance, sag ratio, AP half-width. +- FiringPattern Named patterns: tonic firing, phasic firing, bursting, + irregular spiking. + +== Location & circuit == +- BrainRegion Where the cell sits: hippocampus, mPFC, ventral striatum, + dentate gyrus. +- CorticalLayer L1–L6 designations or named layers. +- Projection Named projection / target region pair: + "L5 IT projecting to contralateral cortex", + "MSDB → CA1". +- ConnectivityMotif Recurrent connectivity descriptors: feedforward inhibition, + disynaptic inhibition, mutual inhibition. + +== Atlas, taxonomy & method context == +- CellOntologyTerm Explicit reference to a CL/PCL/NIFSTD identifier or + canonical term used as a label. +- AtlasReference Named atlas / taxonomy version: Allen Mouse Brain Atlas, + ABC atlas, BICCN human MTG, Tasic 2018 taxonomy. +- ProfilingMethod Method that produced the cell-type call: + patch-seq, scRNA-seq, snRNA-seq, MERFISH, smFISH, + STARmap, MAPseq, retrograde tracing. + +== Species / model == +- Species Mus musculus, mouse, rat, human, macaque. +- DevelopmentalStage Postnatal day 21, E14.5, adult, juvenile. + +== Misc == +- Other Clearly relevant to CNS cell description but no label fits. + +OUTPUT +Strict JSON. No prose. No markdown fences. No comments inside JSON. + +The source's paper_title / doi / source_path live ONCE at the top level +under `source_metadata`. Do NOT repeat them on every entity. `paper_location` +(section / page) varies and stays per-entity. + +❌ WRONG — DO NOT EMIT (output that looks like this is INVALID): +{ + "entities": [ + {"entity": "Pvalb interneurons", "label": "CellType", "sentence": "...", + "start": 200, "end": 218, + "paper_title": "...", "doi": "..."} ← WRONG: per-entity dupes + ] +} + +✅ RIGHT — emit paper_title/doi ONCE at top level: +{ + "source_metadata": { ← ONCE per run + "paper_title": "...", "doi": "..." + }, + "entities": [ + {"entity": "Pvalb interneurons", "label": "CellType", + "sentence": "...", "start": 200, "end": 218, + "cell_context": {"lineage_markers": ["Pvalb"], "region": "mPFC", + "layer": "L2/3", "species": "mouse", + "ephys_summary": "fast-spiking"}, + "paper_location": "Results, Fig 3"} + ] +} + +Schema: +{ + "source_metadata": { + "paper_title": "", + "doi": "<doi if provided in METADATA, else null>", + "source_path": "<file path / url if provided, else null>" + }, + "entities": [ + { + "entity": "<surface form, EXACTLY as in text>", + "label": "<one of the labels above>", + "sentence": "<full sentence containing the entity>", + "start": <int char offset in input>, + "end": <int char offset (exclusive)>, + + "cell_context": { + "lineage_markers": ["<gene/protein name>", ...], + "region": "<BrainRegion if explicit, else null>", + "layer": "<CorticalLayer if explicit, else null>", + "species": "<Species if explicit, else null>", + "ephys_summary": "<short string if mentioned, else null>" + }, + + "paper_location": "<section/page/paragraph if inferable from text, else null>" + } + ], + "key_terms": [ + { + "term": "<surface form>", + "sentence": "<containing sentence>", + "start": <int>, + "end": <int>, + "paper_location": "<section/page if inferable, else null>" + } + ] +} + +RULES +1. start/end are character offsets into the INPUT text — NOT the sentence. +2. text[start:end] MUST equal entity (or term). Verify before emitting. +3. Sentence MUST be a substring of the input text. +4. The same (entity, start, end) triple must not appear twice. Different + start/end values for the same surface form ARE different mentions — + emit them all (see "Exhaustiveness" above). +5. Do NOT include the SAME SPAN in both entities and key_terms. (A different + occurrence of the same string in a different position is fine.) +6. Do NOT hallucinate. Include genuine in-text mentions even at ~50% label + confidence — pick the most likely label; the judge handles uncertainty. +6.5 CELL HIERARCHY: when both a class and a finer subtype are mentioned + ("L5 pyramidal neuron, specifically L5 IT MET-type"), emit BOTH: + - "L5 pyramidal neuron" as CellType + - "L5 IT MET-type" as CellSubtype + Tie them together by populating each item's `cell_context` consistently. +7. MARKERS vs CELL TYPES: "Pvalb interneuron" is a CellType (or CellSubtype + if more granular). The bare "Pvalb" mentioned as a marker is a + LineageMarker. The same surface form can be both in different sentences; + judge from context. +8. EPHYS PHRASES: phrases like "fast-spiking parvalbumin interneuron" emit: + - "fast-spiking" as EphysProperty + - "parvalbumin interneuron" as CellType (lineage_markers: ["parvalbumin"]) +9. NON-CNS CELLS: cells from PNS, immune system outside microglia, or + non-neural tissue are OUT OF SCOPE — do NOT emit them. +10. If input has no CNS-cell content, return {"entities": [], "key_terms": []}. + +If you cannot comply, output exactly: {"error": "<one-line reason>"} +``` + +## User + +``` +INPUT TEXT: +<<< +{input_text} +>>> + +METADATA (paper_title / doi / source_path) — populate `source_metadata` from this; +do NOT repeat on every entity: +{metadata_json} +``` + +## Mask-mode passes (run after this prompt to improve extraction) + +This prompt is **pass-1**. CNS cell-typing papers describe each cell with many co-mentioned attributes (marker + morphology + ephys + layer + region), so pass-1 typically misses some of the secondary attributes when focused on the primary cell type. Mask-recall fixes this. + +| Pass | Prompt | What it does | +|---|---|---| +| **Mask-recall** (strongly recommended for CNS cell typing) | `prompts/mask-recall-pass.md` | Re-runs with pass-1 spans replaced by `[E<i>]`. Surfaces missed lineage markers, morphology descriptors, ephys phrases, atlas cluster names, and layer/region tags. Typical recovery: **+30–80% mentions** (markers and ephys properties are the biggest gainers). | +| **Mask-verify** (recommended for taxonomy work) | `prompts/mask-verify-pass.md` | For each extracted item, replaces just that span with `[MASK]` and predicts the label from context. Especially valuable here to catch CellType ↔ CellSubtype confusion and LineageMarker ↔ CellType ("Pvalb" vs "Pvalb interneuron") errors. | + +Use `scripts/mask_pass.py` to build the masked text and translate offsets back to the original. Pass the SAME `LABEL TAXONOMY` block above into the mask-mode prompts so labels stay consistent — in particular, the mask-mode prompts should know about `CellClass / CellType / CellSubtype / LineageMarker / MorphologyClass / EphysProperty / FiringPattern` because those are the categories pass-1 most often confuses. + +## Suggested ontology routing for alignment + +| Label | Primary ontology | Secondary | +|---|---|---| +| `CellClass`, `CellType`, `CellSubtype` | CL (Cell Ontology) | PCL (Provisional CL), NIFSTD | +| `LineageMarker` | NCBIGene, UniProt | HGNC, MGI | +| `MorphologyClass` | NIFSTD | CL | +| `CellularStructure` | GO cellular component | UBERON | +| `Synapse` | NIFSTD | CL | +| `EphysProperty`, `FiringPattern` | NIFSTD (NEMO) | OBI | +| `BrainRegion`, `CorticalLayer` | UBERON, ABA | NIFSTD | +| `Projection`, `ConnectivityMotif` | NIFSTD | UBERON | +| `CellOntologyTerm` | CL / PCL (direct lookup) | — | +| `AtlasReference` | (none — store as literal) | — | +| `ProfilingMethod` | OBI | EFO | +| `Species` | NCBITaxon | — | +| `DevelopmentalStage` | UBERON dev stages / FBdv / MmusDv | — | + +## Tuning knobs + +- **Single species:** add `Restrict extraction to <species>. Skip any cell described in a different species.` +- **Specific region:** add `Only emit cells located in <region>. Skip cells in other regions.` +- **Taxonomy version pinning:** add `Cluster names must match the <Tasic 2018 | BICCN 2023 | Allen ABC> taxonomy. If a cluster name doesn't match that taxonomy, emit it as Other and add a remark.` +- **Drop the `cell_context` block** if your downstream consumer prefers flat items only. Make sure to keep `lineage_markers` somewhere — they're the most useful field for downstream alignment. + +## Common failure modes + +| Symptom | Fix | +|---|---| +| Atlas cluster names (e.g. "L2/3 IT MET-type") emitted as CellType, not CellSubtype | Strengthen CellSubtype with two cluster-name examples; add: "Atlas cluster names ALWAYS go to CellSubtype." | +| Markers ("Pvalb", "Sst") mis-tagged as CellType | Add: "A bare gene/protein symbol is a LineageMarker. A phrase like 'X interneuron' or 'X+ neuron' is a CellType." | +| Cells outside CNS (e.g. retinal cells in the eye) wrongly included | Strengthen rule 9; reject hits whose `cell_context.region` is non-CNS in the parser. | +| Subcellular structures missed | Lower confidence threshold for `CellularStructure`; many are short tokens (spine, soma, AIS). | +| Layer references emitted without their region ("L5" without "mPFC") | Add: "When a layer appears with a region in the surrounding context, populate `cell_context.region` and `cell_context.layer` together." | +| Profiling methods conflated with cell types ("patch-seq cells") | Add: "Methods name how cells were measured, not what they are. Emit 'patch-seq' as ProfilingMethod and the cells separately as CellType/CellSubtype." | diff --git a/skills/structsense-skills/prompts/extractor-ner-general.md b/skills/structsense-skills/prompts/extractor-ner-general.md new file mode 100644 index 0000000..a96718a --- /dev/null +++ b/skills/structsense-skills/prompts/extractor-ner-general.md @@ -0,0 +1,179 @@ +# Extractor prompt — general-domain NER + +Domain-agnostic NER. Use this when the source text is not from a specialized scientific subdomain (news, finance, biographies, generic web pages, customer support tickets, …) or when you don't yet know the domain and want a coarse first pass. + +For biomedical/neuroscience text, prefer `extractor-ner-neuroscience.md`. +For CNS cell-focused text, prefer `extractor-ner-cns-cells.md`. + +## System + +``` +You are a general-domain named-entity recognition (NER) extractor. +You extract EXHAUSTIVELY. Recall matters more than precision. + +TASK +Given a passage of text, identify EVERY mention of: +- entities: typed, real-world referents. +- key_terms: salient phrases that aren't full entities but matter for retrieval. + +EXHAUSTIVENESS — READ CAREFULLY +- Extract EVERY occurrence. If "Apple" appears 12 times, emit 12 separate + entity items, each with its own distinct start/end pair. +- Do NOT deduplicate. Do NOT collapse repeat mentions. Do NOT emit "one row + per unique surface form." The post-processor handles dedup; you do not. +- Mentions in different sentences ARE different mentions. Emit them all. +- Mentions in the same sentence (e.g. "Apple announced … Apple confirmed …") + are different mentions. Emit them all. +- Acronyms and their expansions ("World Health Organization (WHO)") are TWO + mentions sharing a label. Emit BOTH. +- Possessives and inflections ("Apple's", "Apples") are mentions of the + same entity — emit each with its own span and exact surface form. +- The expected count is HIGH. A typical paragraph yields dozens of entity + mentions; a multi-page paper yields hundreds to thousands. If your output + list feels short, you are missing mentions — go back and re-scan. + +LABEL TAXONOMY (use these exactly; do NOT invent others) +- Person A named human (real or fictional). +- Organization A named organization (company, NGO, agency, university). +- Location A named place (city, country, region, address, landmark). +- Product A named product, brand, service, or work of art. +- Event A named event (conference, war, election, sport tournament). +- Date A specific date or date range. +- Time A specific time of day or interval. +- Money A monetary amount with currency. +- Percent A percentage value. +- Quantity A measured quantity with unit (e.g. "5 km", "200 mg"). +- Law A named statute, treaty, or legal instrument. +- Language A natural language name. +- Other Anything that is clearly an entity but doesn't fit above. + +OUTPUT +Strict JSON. No prose. No markdown fences. No comments inside JSON. + +The source's paper_title / doi / source_path live ONCE at the top level +under `source_metadata`. Do NOT repeat them on every entity — that +duplicates the same value across hundreds or thousands of items. +`paper_location` (section / page / paragraph) DOES vary per mention and +stays per-entity. + +❌ WRONG — DO NOT EMIT (output that looks like this is INVALID): +{ + "entities": [ + {"entity": "Apple", "label": "Organization", "sentence": "...", + "start": 100, "end": 105, + "paper_title": "...", "doi": "..."} ← WRONG: per-entity dupes + ] +} + +✅ RIGHT — emit paper_title/doi ONCE at top level: +{ + "source_metadata": { ← ONCE per run + "paper_title": "...", "doi": "..." + }, + "entities": [ + {"entity": "Apple", "label": "Organization", "sentence": "...", + "start": 100, "end": 105, "paper_location": "page 3"} + ] +} + +Schema: +{ + "source_metadata": { + "paper_title": "<title if provided by user metadata, else null>", + "doi": "<doi if provided by user metadata, else null>", + "source_path": "<file path / url if provided, else null>" + }, + "entities": [ + { + "entity": "<surface form, EXACTLY as in text>", + "label": "<one of the labels above>", + "sentence": "<full sentence containing the entity>", + "start": <int char offset in input>, + "end": <int char offset (exclusive)>, + "paper_location": "<section/page/paragraph if inferable from text, else null>" + } + ], + "key_terms": [ + { + "term": "<surface form>", + "sentence": "<containing sentence>", + "start": <int>, + "end": <int>, + "paper_location": "<section/page if inferable, else null>" + } + ] +} + +RULES +1. start/end are character offsets into the INPUT text below — NOT the sentence. +2. text[start:end] MUST equal entity (or term). Verify before emitting. +3. Sentence MUST be a substring of the input text. +4. The SAME (entity, start, end) triple must not appear twice. Different + start/end values for the same surface form ARE different mentions — + emit them all (see "Exhaustiveness" above). +5. Do NOT include the SAME SPAN in both entities and key_terms. (A different + occurrence of the same string in a different position is fine.) +6. Do NOT hallucinate (do not emit a span that isn't in the text). But DO + include any genuine in-text mention even if you are only ~50% confident + of its label — pick the most likely label and lower judge_score later. +7. If the input is empty or has no entities, return {"entities": [], "key_terms": []}. + +If you cannot comply for any reason, output exactly: +{"error": "<one-line reason>"} +``` + +## User + +``` +INPUT TEXT: +<<< +{input_text} +>>> + +METADATA (paper_title / doi / source_path) — populate `source_metadata` from this; +do NOT repeat on every entity: +{metadata_json} +``` + +## Mask-mode passes (run after this prompt to improve extraction) + +This prompt is **pass-1**. To get closer to exhaustive coverage and catch label errors, chain one or both mask-mode passes: + +| Pass | Prompt | What it does | +|---|---|---| +| **Mask-recall** (recommended) | `prompts/mask-recall-pass.md` | Re-runs over the same text with pass-1 spans replaced by `[E<i>]` placeholders. The model surfaces mentions pass-1 missed (acronyms, plurals, lower-confidence forms). Typical recovery: 20–60% more mentions. | +| **Mask-verify** (optional) | `prompts/mask-verify-pass.md` | For each extracted item, replaces just that span with `[MASK]` and predicts the label from context. Disagreement signals candidate label errors; feeds into `judge_score`. | + +Use `scripts/mask_pass.py` to build the masked text and translate offsets back to the original. Pass the SAME label taxonomy block to the mask-mode prompts so labels align. + +## Tuning knobs + +- **Drop `Other`** to enforce strict label discipline. +- **Add domain-specific labels** (e.g. add `Cryptocurrency`, `Hashtag`) by appending to the taxonomy block — keep the list closed. +- **Recall boost:** add `When in doubt, INCLUDE the term; the next stage filters.` +- **Precision boost:** add `When in doubt, OMIT the term; only include high-confidence entities.` + +## Suggested ontology routing for alignment + +When this output is passed to the alignment stage, route lookups by label: + +| Label | Ontology / resource | +|---|---| +| `Person` | Wikidata, DBpedia | +| `Organization` | Wikidata, ROR (research orgs), GRID | +| `Location` | GeoNames, Wikidata | +| `Product` | Wikidata | +| `Event` | Wikidata | +| `Law` | Wikidata, LegislationGovUk | +| `Language` | Glottolog, Wikidata | + +(See `references/ontology-mapping.md` → "Choosing the right ontology".) + +## Common failure modes + +| Symptom | Fix | +|---|---| +| Output wrapped in ```json fences | Strip fences in parser; lower temperature. | +| `Person` over-applied to titles ("Dr.", "Mr.") | Add: "Do NOT emit honorifics or titles alone as Person entities." | +| Dates mis-typed as Quantity | Add: "ISO dates / month-day-year forms are `Date`, never `Quantity`." | +| Locations also tagged as Organization (e.g. "Microsoft" the building) | Add: "If a name is ambiguous between Org and Location, prefer Organization unless the sentence is clearly about the physical place." | diff --git a/skills/structsense-skills/prompts/extractor-ner-neuroscience.md b/skills/structsense-skills/prompts/extractor-ner-neuroscience.md new file mode 100644 index 0000000..a5e95fd --- /dev/null +++ b/skills/structsense-skills/prompts/extractor-ner-neuroscience.md @@ -0,0 +1,260 @@ +# Extractor prompt — neuroscience-wide NER + +NER for neuroscience text broadly: behavior + systems + cellular + molecular + computational neuroscience. Covers a wide label set spanning whole-brain to molecular scale. + +For CNS cell-specific extraction (cell types, morphologies, electrophysiological subclasses), use `extractor-ner-cns-cells.md`. +For domain-agnostic text, use `extractor-ner-general.md`. + +## System + +``` +You are a neuroscience-domain named-entity recognition (NER) extractor. +You extract EXHAUSTIVELY. Recall matters more than precision. + +TASK +Given neuroscience text (paper, abstract, methods section, review), +identify EVERY mention of: +- entities: typed neuroscience referents (proteins, regions, methods, …) +- key_terms: salient phrases that aren't single entities but matter for + retrieval (paradigms, technique families, behavioral assays). + +EXHAUSTIVENESS — READ CAREFULLY +- Extract EVERY occurrence. If "BDNF" appears 30 times, emit 30 entity + items, each with its own distinct start/end pair. +- Do NOT deduplicate. Do NOT collapse repeat mentions. Do NOT emit "one + row per unique surface form." The post-processor handles dedup. +- Mentions in different sentences ARE different mentions — emit all. +- Mentions in the same sentence are different mentions — emit all. +- Acronyms AND their expansions (e.g. "long-term potentiation (LTP)") are + TWO mentions sharing a label. Emit BOTH every time. +- Symbol/full-name pairs ("Pvalb (parvalbumin)") are TWO mentions every + time they appear — typically Gene + Protein labels respectively. +- Plurals, possessives, inflections ("neurons", "neuron's") are mentions + of the same entity — emit each with its own span and exact surface form. +- Methods sections in particular have very high mention density (reagents, + catalog numbers, instruments, protocols, statistics). Be thorough. +- The expected count is HIGH. A typical neuroscience paper paragraph yields + 20–60 entity mentions; a full methods section yields 200–500; a full paper + yields 800–2000+. If your output feels short, you are missing mentions + — go back and re-scan. + +LABEL TAXONOMY (SUGGESTED labels — prefer these, but not a closed list) +These labels cover the common neuroscience entity types and should be your +FIRST choice: if a mention fits one of them, use it verbatim so labels stay +consistent across the corpus. They are guidance, NOT an exhaustive whitelist. +When a mention is clearly an entity but none of these labels fits well, assign +the MOST APPROPRIATE label you can — coin a concise, descriptive PascalCase +label (e.g. `ImagingModality`, `AnatomicalAxis`) rather than forcing a poor fit +or falling back to `Other`. Reserve `Other` for entities you genuinely cannot +characterize. Reuse any new label consistently within a document. The judge and +post-processor reconcile labels downstream, so a well-chosen new label is far +more useful than a wrong one from the list. +== Anatomy & function == +- BrainRegion Macroscopic structures: hippocampus, mPFC, CA1, layer 5. +- NeuralCircuit Named pathways/loops: mesolimbic pathway, default mode network. +- CorticalLayer L1–L6 or named layers. +- NervousSystemPart PNS components: dorsal root ganglion, sciatic nerve. + +== Cells & subcellular == +- CellType Neuron / glia subtypes: CA1 pyramidal neuron, microglia, + parvalbumin interneuron, astrocyte. +- CellularStructure Subcellular components: dendritic spine, axon initial + segment, postsynaptic density, mitochondrion. +- Synapse Synapse types or named synapses: excitatory synapse, + CA3–CA1 synapse. + +== Molecules == +- Gene Gene symbols or names: BDNF, MECP2, Fos. +- Protein Proteins / receptors / channels: NMDAR, tau, GluA1, + Nav1.6, c-Fos. +- Chemical Small molecules: dopamine, glutamate, kainate, TTX. +- Drug Pharmacological agents with action: ketamine, propofol, + muscimol. +- Neuropeptide Bombesin, oxytocin, NPY. +- IonChannel Specific channels: Kv1.2, HCN1, NaV1.6 (overrides Protein + when channel-typing matters). +- Neurotransmitter GABA, glutamate, dopamine, serotonin. + +== Species & models == +- Species Mus musculus, mouse, rat, zebrafish, C. elegans. +- Strain C57BL/6J, Sprague-Dawley, Long-Evans. +- TransgenicLine Pvalb-Cre, Thy1-GCaMP6f, App/PS1. + +== Methods & assays == +- Method Techniques: patch clamp, two-photon calcium imaging, + scRNA-seq, optogenetics, fMRI. +- BehavioralAssay Named tasks: Morris water maze, novel object recognition, + fear conditioning. +- Stimulus Sensory or experimental stimuli: 1 kHz tone, blue light + (470 nm), foot shock. + +== Measurements & phenomena == +- Measurement Quantifiable variables: firing rate, EPSC amplitude, + calcium transient, BOLD signal. +- Phenomenon Named effects/states: long-term potentiation (LTP), + theta rhythm, sharp-wave ripple. +- Disease Disorders: Alzheimer's disease, autism spectrum disorder, + epilepsy, schizophrenia. +- Phenotype Observed traits: hyperactivity, memory deficit, anxiety- + like behavior. + +== Misc == +- Software Named software/toolkits used as analytic methods. +- Other Clearly an entity but no label above fits. + +OUTPUT +Strict JSON. No prose. No markdown fences. No comments inside JSON. + +The source's paper_title / doi / source_path live ONCE at the top level +under `source_metadata`. Do NOT repeat them on every entity. With hundreds +of mentions per paper, repeating these would inflate the JSON size 5–10x +for zero information gain. `paper_location` (section / page) is +per-entity because it varies. + +❌ WRONG — DO NOT EMIT (this is a hard rejection signal; output that + looks like this will be rejected as INVALID): +{ + "entities": [ + { + "entity": "basal ganglia", "label": "BrainRegion", + "sentence": "...", + "start": 1143, "end": 1156, + "paper_title": "Multiscale Spatial Transcriptomic Atlas", ← WRONG + "doi": "10.64898/2025.12.02.691876" ← WRONG + }, + { "...repeated 1000 more times..." } ← WRONG + ] +} + +✅ RIGHT — emit paper_title/doi ONCE at the top, never per-entity: +{ + "source_metadata": { ← ONCE + "paper_title": "Multiscale Spatial Transcriptomic Atlas", + "doi": "10.64898/2025.12.02.691876" + }, + "entities": [ + { + "entity": "basal ganglia", "label": "BrainRegion", + "sentence": "...", "start": 1143, "end": 1156, + "paper_location": "Introduction" ← paper_location IS per-entity + }, + { "...more entities — none with paper_title or doi..." } + ] +} + +Schema: +{ + "source_metadata": { + "paper_title": "<title if provided in METADATA, else null>", + "doi": "<doi if provided in METADATA, else null>", + "source_path": "<file path / url if provided, else null>" + }, + "entities": [ + { + "entity": "<surface form, EXACTLY as in text>", + "label": "<a label from the taxonomy above, or a coined PascalCase label if none fits>", + "sentence": "<full sentence containing the entity>", + "start": <int char offset in input>, + "end": <int char offset (exclusive)>, + "paper_location": "<section/page/paragraph if inferable from text, else null>" + } + ], + "key_terms": [ + { + "term": "<surface form>", + "sentence": "<containing sentence>", + "start": <int>, + "end": <int>, + "paper_location": "<section/page if inferable, else null>" + } + ] +} + +RULES +1. start/end are character offsets into the INPUT text — NOT the sentence. +2. text[start:end] MUST equal entity (or term). Verify before emitting. +3. Sentence MUST be a substring of the input text. +4. The same (entity, start, end) triple must not appear twice. Different + start/end values for the same surface form ARE different mentions — + emit them all (see "Exhaustiveness" above). +5. Do NOT include the SAME SPAN in both entities and key_terms. (A different + occurrence of the same string in a different position is fine.) +6. Do NOT hallucinate (do not emit a span that isn't in the text). But DO + include genuine in-text mentions even at ~50% label confidence — pick + the most likely label (preferring the suggested taxonomy, otherwise the\n most appropriate PascalCase label you can coin); the judge handles uncertain labels later. +7. ACRONYM HANDLING: if both expansion and acronym are in the source + ("hippocampus (HP)"), emit BOTH as separate entities sharing a `label`. + Repeat this every time the pair recurs. +8. NEGATED MENTIONS: still emit ("no significant change in BDNF" → BDNF as Gene). +9. LABEL DISAMBIGUATION: + - `Drug` overrides `Chemical` when the source describes therapeutic/ + pharmacological use. + - `IonChannel` overrides `Protein` for channel proteins when the source + emphasizes channel function. + - `Phenomenon` is for named effects, not single measurements (firing rate + is a Measurement; LTP is a Phenomenon). +10. If input has no entities, return {"entities": [], "key_terms": []}. + +If you cannot comply, output exactly: {"error": "<one-line reason>"} +``` + +## User + +``` +INPUT TEXT: +<<< +{input_text} +>>> + +METADATA (paper_title / doi / source_path) — populate `source_metadata` from this; +do NOT repeat on every entity: +{metadata_json} +``` + +## Mask-mode passes (run after this prompt to improve extraction) + +This prompt is **pass-1**. Neuroscience papers are mention-dense (a single methods section easily yields hundreds of entity mentions across reagents, instruments, protocols, regions, species, drug doses, statistical tests, etc.). To get closer to exhaustive coverage, **always run the mask-recall pass on top of pass-1 for neuroscience text**. + +| Pass | Prompt | What it does | +|---|---|---| +| **Mask-recall** (strongly recommended for neuroscience) | `prompts/mask-recall-pass.md` | Re-runs over the same text with pass-1 spans replaced by `[E<i>]` placeholders. The model surfaces the mentions pass-1 missed — typically catalog numbers, low-frequency reagents, plurals, acronyms in tables. Typical recovery on a neuroscience paper: **+30–80% mentions**. | +| **Mask-verify** (optional) | `prompts/mask-verify-pass.md` | For each extracted item, replaces just that span with `[MASK]` and predicts the label from context. Disagreement is a strong signal of label confusion (e.g. Gene vs Protein, Drug vs Chemical, Phenomenon vs Measurement). Use to calibrate `judge_score`. | + +Use `scripts/mask_pass.py` to build the masked text and translate offsets back to the original. Pass the SAME `LABEL TAXONOMY` block above into the mask-mode prompts so labels stay consistent. + +## Suggested ontology routing for alignment + +| Label | Primary ontology | Secondary | +|---|---|---| +| `BrainRegion`, `NeuralCircuit`, `CorticalLayer`, `NervousSystemPart` | UBERON | NIFSTD, ABA | +| `CellType` | CL (Cell Ontology) | NIFSTD | +| `CellularStructure` | GO (cellular component) | UBERON | +| `Synapse` | NIFSTD | CL | +| `Gene` | NCBIGene | HGNC, MGI | +| `Protein` | UniProt / PR (Protein Ontology) | — | +| `Chemical`, `Drug`, `Neurotransmitter`, `Neuropeptide` | CHEBI | DrON | +| `IonChannel` | PR | UniProt, IUPHAR | +| `Species` | NCBITaxon | — | +| `Strain` / `TransgenicLine` | MGI (mouse), RGD (rat), MMRRC | — | +| `Method`, `BehavioralAssay`, `Stimulus` | OBI | EFO, NIFSTD | +| `Measurement` | OBI, NIFSTD | — | +| `Phenomenon` | NIFSTD | GO biological process | +| `Disease` | MONDO | DOID | +| `Phenotype` | HP (human), MP (mouse) | — | + +## Tuning knobs + +- **Sub-focus to molecular only:** drop Anatomy/Cells labels; keep Gene/Protein/Chemical/Drug/IonChannel/Neurotransmitter/Neuropeptide. +- **Sub-focus to behavioral neuroscience:** keep BrainRegion/CellType/BehavioralAssay/Stimulus/Phenomenon/Phenotype; drop molecular labels. +- **Strict mode (no `Other`):** drop the `Other` label entirely. +- **Closed taxonomy:** to restrict output to the suggested labels only (no coined labels), replace the taxonomy header with "use these exactly; do NOT invent others" and revert rule 6 / the schema `label` field accordingly. +- **Acronym strictness:** by default both forms are emitted; if your downstream consumer wants only one, add: "Emit only the long form when both appear in the same sentence." + +## Common failure modes + +| Symptom | Fix | +|---|---| +| Receptor names labeled `Gene` (NMDAR is the receptor, GRIN2B the gene) | Add: "Receptors and channel complexes are `Protein`; gene symbols are `Gene`." | +| Brain regions emitted with stripped descriptors ("CA1" but not "dorsal CA1") | Add: "Preserve qualifiers like 'dorsal', 'lateral' — they're part of the region name." | +| Behavioral assays missed when paraphrased | Reduce confidence threshold; behavior assays often vary in wording. | +| Drug names tagged Chemical instead of Drug | Strengthen the Drug/Chemical disambiguation rule with examples. | diff --git a/skills/structsense-skills/prompts/extractor-resource.md b/skills/structsense-skills/prompts/extractor-resource.md new file mode 100644 index 0000000..dba2947 --- /dev/null +++ b/skills/structsense-skills/prompts/extractor-resource.md @@ -0,0 +1,78 @@ +# Extractor prompt — research resource + +## System + +``` +You extract metadata for ONE primary research resource from a source document +(paper, README, model card). Other tools/datasets/models that are merely +referenced go under `mentions`, NEVER as a sibling top-level resource. + +WHAT COUNTS AS A RESOURCE +- Model, Dataset, Tool, Benchmark, Leaderboard, Paper. + +WHAT IS THE PRIMARY RESOURCE +- For a paper: the artifact named in the title and "we introduce/propose/release" sentence. +- For a README: the repository's own name. +- For a model card: the model that page documents. + +OUTPUT +Strict JSON. No prose. No markdown fences. +Schema: +{ + "extracted_resources": { + "1": [{ + "name": "<canonical name>", + "description": "<1–3 sentences, factual, from the source>", + "type": "Model | Dataset | Tool | Benchmark | Leaderboard | Paper", + "category": "<task category, e.g. 'Pose Estimation'>", + "target": "<primary target domain, e.g. 'Animal', 'Human', 'Multimodal'>", + "specific_target": "<comma-separated specifics, e.g. 'Mouse, Rat'>", + "url": "<canonical URL or null>", + "mentions": { + "datasets": ["<name>", ...], + "benchmarks": ["<name>", ...], + "models": ["<name>", ...], + "papers": ["<name>", ...] + } + }] + } +} + +RULES +1. `type` MUST be one of the listed enum values. Normalize: + "Library"/"Framework" → "Tool", "Corpus" → "Dataset". +2. Emit EXACTLY ONE primary resource under key "1" unless the source + genuinely introduces multiple primary resources (e.g. "we release a + dataset and a model"). Then use "1", "2", ... — never merge. +3. Every other referenced artifact goes under `mentions`, never as a sibling. +4. If a URL is not present in the source, set it to null. Do NOT invent URLs. +5. `description` is factual and copied/paraphrased from the source. + Do not infer capabilities the source doesn't claim. + +If you cannot comply, output {"error": "<one-line reason>"}. +``` + +## User + +``` +INPUT TEXT: +<<< +{input_text} +>>> +``` + +## Tuning knobs + +- **Restrict to specific resource types:** "Only emit a resource if its type is `Model` or `Dataset`." +- **Add domain constraint:** "Only emit resources relevant to brain-behavior research." +- **Add canonical URL requirement:** "If `url` is unknown, return `{}` for this resource — do not emit it without a URL." (Stricter; reduces silent hallucinations.) + +## Common failure modes + +| Symptom | Fix | +|---|---| +| Model lists every cited tool as a top-level resource | Strengthen "primary resource" rule; add: "If unsure whether something is primary, put it in `mentions`." | +| Invented URLs (e.g. `https://example-tool.github.io`) | Add: "URLs MUST appear literally in the source text or be null." Validate URLs in the parser. | +| `type` set to `Library` or other off-vocabulary | Hard-reject in parser; reprompt. | +| Overly long descriptions | Add char limit: "`description` ≤ 400 characters." | +| `mentions` missing real references | Add: "List EVERY other named tool/dataset/model under `mentions`, even if mentioned only in passing." | diff --git a/skills/structsense-skills/prompts/extractor-structured.md b/skills/structsense-skills/prompts/extractor-structured.md new file mode 100644 index 0000000..b7e30af --- /dev/null +++ b/skills/structsense-skills/prompts/extractor-structured.md @@ -0,0 +1,86 @@ +# Extractor prompt — schema-driven structured extraction + +Use this prompt template when the user has a target JSON schema and wants a document mapped into it (e.g. PDF → ReproSchema, paper → Croissant dataset card, form → custom schema). + +## System + +``` +You extract structured metadata from a source document and emit a JSON +document that MUST validate against the JSON Schema below. + +OUTPUT +- Strict JSON. No prose. No markdown fences. No comments. +- The JSON MUST validate against the provided JSON Schema. +- For every required field that is not present in the source, set its value + to null. Do NOT invent values. +- Preserve original strings; never reword without need. + +JSON SCHEMA (the contract): +{json_schema} + +FIELD HINTS +{field_hints} + +RULES +1. Output exactly one top-level JSON object matching the root schema. +2. Enum fields MUST use one of the listed enum values verbatim. +3. Date fields use ISO 8601: "YYYY-MM-DD" or "YYYY-MM-DDTHH:MM:SSZ". +4. URLs must appear literally in the source or be null. +5. If a list field has no items in the source, return [] (not null). +6. If a string field is not present in the source, return null. +7. Preserve the original phrasing as <field>_raw alongside any normalized + value where the schema allows it. + +If you cannot comply, output {"error": "<one-line reason>"}. +``` + +## User + +``` +SOURCE DOCUMENT: +<<< +{input_text} +>>> + +ADDITIONAL CONTEXT (optional metadata, hints, page references): +{metadata_json} +``` + +## How to fill the placeholders + +- `{json_schema}`: paste the entire JSON Schema document. Yes, the whole thing. Models do better with the explicit contract than with a paraphrase. +- `{field_hints}`: a short per-field note where it helps. Example: + ``` + - activity.id: short identifier built from title initials, ignoring stopwords ("and", "the"). + e.g. "Mood and Feelings Questionnaire" → "MFQ". + - items[].inputType: one of [radio | text | number | slider]. + - items[].responseOptions[].value: integer score; lowest first. + ``` + +## Repair prompt (used after schema-validation failure) + +``` +The JSON below is INVALID for the schema. Fix it. Output ONLY the +corrected JSON — no prose, no markdown fences. + +SCHEMA: +{json_schema} + +ERROR: +{validator_error} (path: {validator_path}) + +INVALID JSON: +{invalid_json} +``` + +Cap repair attempts at 2. If still invalid, return the closest-valid output plus `errors[]` populated with the validator errors. + +## Common failure modes + +| Symptom | Fix | +|---|---| +| Model omits required fields | Add: "Every required field is mandatory. Use null only if explicitly allowed." | +| Numeric fields emitted as strings (`"3"`) | Add explicit type hint in `{field_hints}`. Validate types. | +| Enum mis-spelt (e.g. `"Radio"` not `"radio"`) | Hard-reject in parser; reprompt with the enum list. | +| Hallucinated URLs / citations | Add: "Do NOT include URLs that don't appear literally in the source." | +| List flattened to a comma-separated string | Add: "List fields are JSON arrays, never comma-separated strings." | diff --git a/skills/structsense-skills/prompts/humanfeedback.md b/skills/structsense-skills/prompts/humanfeedback.md new file mode 100644 index 0000000..3b5cf7e --- /dev/null +++ b/skills/structsense-skills/prompts/humanfeedback.md @@ -0,0 +1,87 @@ +# Human feedback prompt — apply reviewer edits + +## System + +``` +You revise a judged extraction by applying human reviewer feedback. + +INPUT +- A judged JSON document (entities/resources/items with judge_score, remarks). +- A free-text feedback string from a human reviewer. +- An optional structured modification_context. + +YOUR JOB +Apply the reviewer's feedback to the JSON. Common feedback patterns: + - "Item N should be label=X, not Y." → locate item N (by index or content), update. + - "All mentions of 'X' should map to <IRI>." → find matching items, update mapping. + - "Drop the items in the References section." → filter by paper_location. + - "Item N is wrong, remove it." → drop that item. + +PRESERVE STRUCTURE +- Same top-level keys. +- Same item shape (you may drop items, but never rename fields). +- If feedback is ambiguous or cannot be safely applied, leave items + unchanged and append a note to errors[] instead of guessing. + +OUTPUT +Strict JSON with: +- The revised top-level structure (entities / resources / etc). +- A new field `human_feedback_log` (list) appended with one entry describing + what you changed (action, items_changed, brief summary). +- `errors[]` populated for any feedback you couldn't safely apply. + +No prose, no markdown fences. + +If you cannot comply at all, output {"error": "<one-line reason>"}. +``` + +## User + +``` +JUDGED JSON: +{judged_structured_information} + +REVIEWER FEEDBACK: +{user_feedback_text} + +MODIFICATION CONTEXT (optional): +{modification_context} +``` + +## What the agent should refuse + +Be polite-but-firm about refusing unsafe edits. Add to system prompt: + +``` +REFUSE if the feedback would: +- Add new items not present in the input (the reviewer should re-run extraction). +- Change field names or types (the schema is fixed). +- Apply edits the model cannot verify (e.g. "verify these against the original PDF" without source text). + +In each case, leave items unchanged and add an entry to errors[] like: + {"code": "unsafe_edit", "feedback": "<the part you refused>", "reason": "<why>"} +``` + +## human_feedback_log entry + +```jsonc +{ + "human_feedback_log": [ + { + "timestamp": "<ISO 8601>", + "action": "edited", // "edited" | "removed" | "approved" | "refused" + "items_changed": [3, 7, 12], // indices or stable IDs + "summary": "Relabeled items 3,7,12 from Phenotype to Disease per reviewer." + } + ] +} +``` + +## Common failure modes + +| Symptom | Fix | +|---|---| +| Agent rewrites unrelated items | Strengthen "only change items matching the feedback." | +| Agent invents new items based on feedback | Strengthen "NEVER add new items; the reviewer must re-run extraction for additions." | +| Agent silently drops items | Add: "If you remove an item, log it in human_feedback_log with action=removed." | +| Agent ignores ambiguous feedback | Encourage `errors[]` entries for refusals rather than silent passes. | diff --git a/skills/structsense-skills/prompts/judge.md b/skills/structsense-skills/prompts/judge.md new file mode 100644 index 0000000..ac0530f --- /dev/null +++ b/skills/structsense-skills/prompts/judge.md @@ -0,0 +1,112 @@ +# Judge prompt — per-item quality scoring + +## System + +``` +You evaluate the quality of an extraction + ontology alignment. + +INPUT +A JSON document of aligned items. Each item has fields: +- a surface form (entity / term / name) +- a label / type +- ontology_id, ontology_label, ontology, concept_mapping_provenance +- a containing sentence (source context) + +YOUR JOB +For each item, add: +- judge_score: float in [0, 1] — your confidence that the extraction + + alignment is correct. +- remarks: one short sentence explaining the score. + +Calibration: + 1.00 unambiguous match, ontology label matches surface form exactly + 0.85 strong match; small surface/label variation but clearly correct + 0.65 plausible; some ambiguity (homonym, low-context) + 0.40 uncertain; ontology fits the type but may be wrong concept + 0.20 likely wrong; ontology mismatch or sense-disambiguation error + 0.00 obviously wrong / hallucinated + +PRESERVE every existing field. Add only judge_score and remarks. +Do not modify ontology_id, ontology_label, label, or any other field. + +OUTPUT +Strict JSON with the same top-level structure as the input. +No prose, no markdown fences. + +If you cannot comply, output {"error": "<one-line reason>"}. +``` + +## User + +``` +ALIGNED JSON TO JUDGE: +{aligned_structured_information} + +ORIGINAL SOURCE TEXT (use for context when judging): +{source_text} +``` + +## Calibration anchors + +Help the model calibrate by including 2–4 example items + scores in the system prompt for the specific task. For NER: + +``` +EXAMPLES +- {"entity":"hippocampus","label":"BrainRegion","ontology_id":"UBERON_0002421", + "ontology_label":"hippocampal formation","sentence":"... in the hippocampus."} + → judge_score: 0.95, remarks: "Standard anatomy mapping; concept matches surface form." + +- {"entity":"BDNF","label":"Gene","ontology_id":"HGNC:1033", + "ontology_label":"BDNF","sentence":"... BDNF mRNA levels ..."} + → judge_score: 1.0, remarks: "Exact gene symbol; HGNC ID is canonical." + +- {"entity":"mouse","label":"Species","ontology_id":"NCBITaxon:10116", + "ontology_label":"Rattus norvegicus","sentence":"... in mouse ..."} + → judge_score: 0.05, remarks: "Mapped to RAT instead of MOUSE; ontology mismatch." +``` + +These calibration examples are worth ~5× the words in raw instructions. + +## Direct-API batched judging (faster + cheaper) + +For large inputs, don't run the judge through an agent loop. Make per-item batched API calls in parallel: + +```python +async def judge_batch(items, source_text, model, concurrency=8): + sem = asyncio.Semaphore(concurrency) + async def one(item): + async with sem: + return await call_llm_async( + model=model, + system=PROMPT_JUDGE, + user={"item": item, "source": source_text}, + response_format={"type": "json_object"}, + temperature=0, + ) + return await asyncio.gather(*[one(i) for i in items]) +``` + +This is ~5× cheaper than an agent loop and has predictable latency. + +## Auto-approve mode (skip the judge entirely) + +When you trust alignment and don't need scoring: + +```python +for item in flatten(aligned): + item["judge_score"] = 1.0 + item["remarks"] = "auto-approved" + item["judge_method"] = "auto_approved" +``` + +Record `judge_method: "auto_approved"` so consumers know there was no real scoring. + +## Common failure modes + +| Symptom | Fix | +|---|---| +| Every score is 0.5 (no signal) | Add stronger calibration examples; lower temperature. | +| Scores cluster at 1.0 | Add at least one "obviously wrong" calibration example with score 0.0. | +| Judge modifies ontology IDs | Strengthen: "PRESERVE every existing field. ADD only judge_score and remarks." Validate diff in parser. | +| Remarks are verbose paragraphs | Add char limit: "`remarks` ≤ 140 characters." | +| Judge invents new items | Add: "Output exactly the same number of items as the input." Validate count in parser. | diff --git a/skills/structsense-skills/prompts/mask-recall-pass.md b/skills/structsense-skills/prompts/mask-recall-pass.md new file mode 100644 index 0000000..71886a5 --- /dev/null +++ b/skills/structsense-skills/prompts/mask-recall-pass.md @@ -0,0 +1,114 @@ +# Mask-recall pass — find missed mentions (boosts recall) + +A second-pass prompt that runs **after** any of the three extractor prompts (`extractor-ner-general.md`, `extractor-ner-neuroscience.md`, `extractor-ner-cns-cells.md`). Its only job: find mentions the first pass missed. + +## How it works + +1. Run any extractor prompt → pass-1 entities `E1`. +2. **Mask** every span in `E1` in the source text: replace `text[start:end]` with a placeholder like `[E0]`, `[E1]`, …, preserving character offsets via padding. +3. Feed the masked text back to the model with this prompt. +4. The model returns NEW mentions (anything still visible that should have been extracted). +5. Append the new mentions to `E1` and run span validation. + +This is the cheapest, most reliable recall booster for NER. Use the same model as pass-1 or a smaller cheap model — even Haiku/gpt-4o-mini handles this well. + +## System + +``` +You are running a SECOND-PASS recall check on a named-entity extraction. + +CONTEXT +A first pass already extracted some entities. Those mentions have been +REPLACED in the input text with placeholder tokens of the form [E0], +[E1], [E2], … — one per mention previously extracted. Treat every [En] +token as already-extracted; do NOT re-emit it. + +YOUR JOB +Identify entity mentions that the first pass MISSED — anything still +visible as natural text in the input that should have been extracted +under the same label taxonomy as the first pass. + +LABEL TAXONOMY +{label_taxonomy_block} + +OUTPUT +Strict JSON. No prose. No markdown fences. Schema: +{ + "missed_entities": [ + { + "entity": "<surface form, EXACTLY as in the input>", + "label": "<one of the labels above>", + "sentence": "<containing sentence as it appears in the input, + with [En] tokens preserved if any are inside>", + "start": <int char offset in the MASKED input>, + "end": <int char offset (exclusive) in the MASKED input>, + "missed_reason": "<short string: e.g. 'unfamiliar acronym', + 'plural form', 'in a table', 'low-frequency surface form'>" + } + ], + "missed_key_terms": [ + { + "term": "<surface form>", + "sentence": "<containing sentence>", + "start": <int>, "end": <int>, + "missed_reason": "<short>" + } + ] +} + +RULES +1. EXHAUSTIVE: emit EVERY missed occurrence, not one per unique surface form. + If the same missed term appears 10 times in the masked text, emit 10 items + with 10 distinct start/end pairs. +2. start/end are offsets into the MASKED text (the input you see, with [En] + placeholders). The post-processor will translate back to original offsets. +3. NEVER emit a [En] placeholder as an entity. NEVER produce an item whose + `entity` field starts with '[E' and ends with ']'. +4. Use the SAME label taxonomy as pass-1. +5. If nothing was missed, return {"missed_entities": [], "missed_key_terms": []}. + +If you cannot comply, output {"error": "<one-line reason>"}. +``` + +## User + +``` +MASKED INPUT TEXT: +<<< +{masked_text} +>>> + +WHAT THE PLACEHOLDERS REPLACED (for context only — do not re-emit these): +{placeholder_map_json} + +LABEL TAXONOMY REMINDER: +{label_taxonomy_block} +``` + +## How to fill the placeholders + +- `{label_taxonomy_block}`: paste the LABEL TAXONOMY block from the pass-1 prompt that was used. Keep it identical so pass-2 uses the same labels. +- `{masked_text}`: source text with each pass-1 entity span replaced by `[E<i>]`. Use `scripts/mask_pass.py` to build this. +- `{placeholder_map_json}`: `[{"placeholder": "[E0]", "entity": "BDNF", "label": "Gene"}, …]` so the model knows what it's not supposed to re-extract. + +## Merge back into pass-1 results + +After receiving the response: + +1. Translate masked-text offsets → original-text offsets (the helper in `scripts/mask_pass.py` does this). +2. Run span validation (`scripts/span_validator.py`) on each missed entity. +3. Append survivors to the pass-1 list. **Do not deduplicate by surface form** — the whole point is to keep every occurrence. +4. Re-run the alignment / judge stages on the merged result (or only on the appended subset, then merge mappings back). + +## When to use it + +| Situation | Verdict | +|---|---| +| Pass-1 yield feels too low (e.g. 230 entities where you expected ~1000) | **Always run.** Typical recovery: 20–60% more mentions. | +| You're on a tight budget | Skip; pass-1 alone is fine for prototypes. | +| Pass-1 already extracted ≥ 80% of expected mentions | Optional; expect <10% additional recovery. | +| You changed the label taxonomy mid-run | **Re-extract from scratch.** Mask-recall assumes the same taxonomy. | + +## When NOT to chain a third pass + +Stop at pass-2. Three passes give diminishing returns and start surfacing genuinely junk mentions. If recall is still insufficient after pass-2, the right fix is a better extractor model or smaller chunks, not more mask passes. diff --git a/skills/structsense-skills/prompts/mask-verify-pass.md b/skills/structsense-skills/prompts/mask-verify-pass.md new file mode 100644 index 0000000..895e05e --- /dev/null +++ b/skills/structsense-skills/prompts/mask-verify-pass.md @@ -0,0 +1,106 @@ +# Mask-verify pass — label sanity check from context (boosts precision) + +Cloze-style verification of an already-extracted entity. For each candidate item, replace its span with `[MASK]` and ask the model to predict the label and surface form from context alone. Disagreement with the original label signals a candidate error. + +Use this **after** the extractor and (optionally) the alignment stage; before or alongside the judge. It produces a per-item `mask_label_agreement` boolean + a `mask_predicted_label` field that the judge can weight into its score. + +## How it works + +For each entity `e` in the extracted list: + +1. Replace `text[e.start:e.end]` with `[MASK]` (preserving offsets via padding). +2. Send the resulting masked-context sentence (or paragraph) to the model. +3. Model returns: a predicted label + the most likely surface form. +4. Compare: + - `mask_label_agreement` = (predicted_label == e.label). + - `mask_surface_agreement` = (predicted_surface in {e.entity, …common variants})`. + +Run this in parallel over all items (it's per-item batched API calls). + +## System + +``` +You are running a verification check on a single named-entity mention. + +CONTEXT +You are shown a sentence with ONE span replaced by [MASK]. The original +text at that position was extracted as an entity. Your job is to predict, +PURELY FROM CONTEXT, what should fill the [MASK]: +- the most likely entity label (one of the labels listed below) +- the most likely surface form (a short string) +- your confidence that any entity belongs at that position at all + +LABEL TAXONOMY +{label_taxonomy_block} + +OUTPUT +Strict JSON. No prose. No markdown fences. +{ + "predicted_label": "<one of the labels above, or 'NotAnEntity'>", + "predicted_surface": "<most likely surface form, or null>", + "is_entity_position": <true|false>, + "context_confidence": <float 0-1>, + "remarks": "<one short sentence>" +} + +RULES +1. Use ONLY the visible context to decide. Do not invent unrelated content. +2. If the context is too generic to identify any entity ("the [MASK] was + observed"), set predicted_label="NotAnEntity", + is_entity_position=false, context_confidence near 0. +3. Use the SAME label taxonomy as the extractor. +4. predicted_surface should be short (1-5 tokens), guessing what word(s) + the [MASK] replaced. + +If you cannot comply, output {"error": "<one-line reason>"}. +``` + +## User + +``` +MASKED CONTEXT (one sentence; the [MASK] is the position under verification): +<<< +{masked_sentence} +>>> + +(Optional surrounding context, up to 1–2 paragraphs:) +{surrounding_context} + +LABEL TAXONOMY REMINDER: +{label_taxonomy_block} +``` + +## Wiring it into the pipeline + +```python +# Pseudocode +for item in extracted_entities: + masked = mask_one(item, source_text) # replace just this span + pred = call_llm(model, MASK_VERIFY_PROMPT, + user={"masked_sentence": masked.sentence, + "surrounding_context": masked.context}) + item["mask_predicted_label"] = pred["predicted_label"] + item["mask_predicted_surface"] = pred["predicted_surface"] + item["mask_label_agreement"] = (pred["predicted_label"] == item["label"]) + item["mask_surface_agreement"] = surface_match(pred["predicted_surface"], item["entity"]) + item["mask_context_confidence"] = pred["context_confidence"] +``` + +The judge can now weight `judge_score`: + +- Big down-weight if `mask_label_agreement=false` AND `mask_context_confidence > 0.7`. +- Slight down-weight if `mask_surface_agreement=false` AND the surface forms aren't paraphrases. +- Up-weight if both agree AND context confidence is high. + +## When to use it + +| Situation | Verdict | +|---|---| +| Cost-sensitive run | **Skip.** It's a per-item LLM call; expensive on large extractions. | +| Quality-first run on high-stakes outputs | **Run.** Catches mis-labelled entities the judge often misses. | +| Calibrating a new extractor model | **Run on a sample.** Lets you measure label-confusion patterns. | +| Verifying alignment IRIs are sensible | Use `judge.md` instead — it sees the IRI too. | + +## Cost notes + +For 1000 entities and a cheap model (`gpt-4o-mini`, `claude-haiku-4-5`), one verify pass is roughly the cost of 1000 short completions. With parallel concurrency=8 it finishes in 1–3 minutes. Skip on prototypes; consider running on a 5–10% sample to surface systematic label errors before paying for a full pass. diff --git a/skills/structsense-skills/references/chunking-strategy.md b/skills/structsense-skills/references/chunking-strategy.md new file mode 100644 index 0000000..a6c5b28 --- /dev/null +++ b/skills/structsense-skills/references/chunking-strategy.md @@ -0,0 +1,138 @@ +# Chunking strategy + +When the input is larger than the model can swallow in one prompt (typically anything over ~25,000 characters for a 128k-token model once you account for the system prompt + JSON schema + output budget), you must chunk. + +## Core principles + +1. **Chunk on sentence boundaries**, never mid-sentence. NER spans that cross a chunk boundary are lost. +2. **Track each chunk's `start` offset** in the original text. When you merge results, add this offset back to every `start`/`end` to get global offsets. +3. **Parallelize the extractor**, not the alignment or judge (which depend on the merged result). +4. **Merge by stable identifier**, not by order. Don't assume chunk 1's output came back before chunk 2's. + +## Sentence-aligned chunking + +Use spaCy or NLTK to find sentence boundaries; greedily accumulate sentences until the next one would push you over `max_chars`. See `scripts/chunking.py` for a ready-to-use implementation. + +Pseudocode: + +```python +def chunk_by_sentences(text: str, max_chars: int = 2000) -> list[dict]: + sents = sentence_split(text) # list[(start, end, sentence_text)] + chunks = [] + cur_start, cur_end = None, None + for s_start, s_end, _ in sents: + if cur_start is None: + cur_start, cur_end = s_start, s_end + continue + if s_end - cur_start <= max_chars: + cur_end = s_end + else: + chunks.append({"text": text[cur_start:cur_end], "start": cur_start}) + cur_start, cur_end = s_start, s_end + if cur_start is not None: + chunks.append({"text": text[cur_start:cur_end], "start": cur_start}) + return chunks +``` + +The result is a list of `{text, start}`. Every chunk overlaps no other chunk, and total reconstructs the original text minus inter-sentence whitespace. + +## Picking `max_chars` + +Rule of thumb: `max_chars ≈ model_context_chars × 0.20`. + +Why 20%, not 90%? + +- The system prompt + JSON schema + instructions eat ~5–15k tokens. +- The model needs room to **emit** the output JSON, which for dense text is roughly the size of the input again (more entities = more output). +- Internal model overhead (chain-of-thought tokens, retries) consume another chunk. + +For a 128k-token model (≈500k chars after tokenization), aim for `max_chars = 25000`. +For a 32k model, aim for `max_chars = 6000`. +For an 8k model (rare now), aim for `max_chars = 1500`. + +Smaller chunks = higher recall but slower and pricier. Common values in practice: **1500–3000 chars** for NER, **5000–8000** for resource extraction (which needs more context), **2000** for structured extraction. + +## Parallel extraction + +Run all chunks concurrently. The exact concurrency depends on the provider: + +| Provider | Safe concurrent requests | +|---|---| +| OpenRouter | 8–16 (varies by underlying model and your account tier) | +| OpenAI direct | 16+ | +| Anthropic direct | 5–10 | +| Ollama local | 1–2 (single GPU) | + +```python +import asyncio + +async def extract_all(chunks): + sem = asyncio.Semaphore(8) + async def one(c): + async with sem: + return await extract_async(c["text"], chunk_start=c["start"]) + return await asyncio.gather(*[one(c) for c in chunks]) +``` + +## Re-mapping offsets after extraction + +Every chunk returns `start`/`end` **relative to the chunk text** (because that's all the model saw). Add `chunk.start` back to make them global: + +```python +def reanchor(chunk_start: int, items: list[dict]) -> list[dict]: + for it in items: + it["start"] = it["start"] + chunk_start + it["end"] = it["end"] + chunk_start + return items +``` + +Validate after reanchoring: `assert original_text[item["start"]:item["end"]] == item["entity"]` for every item. Drop or repair items that fail. + +## Merging chunk outputs + +After reanchoring, concatenate the lists and deduplicate: + +```python +def merge(chunk_results: list[dict]) -> dict: + entities, key_terms = [], [] + for r in chunk_results: + entities.extend(r.get("entities", [])) + key_terms.extend(r.get("key_terms", [])) + return { + "entities": dedupe(entities, key=lambda e: (e["entity"], e["start"], e["end"])), + "key_terms": dedupe(key_terms, key=lambda k: (k["term"], k["start"], k["end"])), + } +``` + +## Downstream chunking (alignment / judge) + +The merged extractor output may itself be too large for a single alignment or judge call. Chunk **by item count**: + +```python +def chunk_items(items: list, chunk_size: int) -> list[list]: + return [items[i:i+chunk_size] for i in range(0, len(items), chunk_size)] +``` + +Default `chunk_size`: `ceil(len(items) / max_workers)`. With 800 items and 8 workers → 100 items per chunk → 8 parallel jobs. + +After each chunk runs through alignment/judge, **merge by stable key** (e.g. `(entity, start, end)`). Never assume positional alignment. + +## Edge cases + +- **Tables in PDFs** often render with `\n` between every cell. Sentence-splitters explode this into one-cell "sentences." Preprocess: collapse runs of single-line cells into a synthetic paragraph before chunking. +- **Code blocks and equations** can defeat sentence splitters. Treat any line starting with `$$` or `\(` as its own pseudo-sentence to prevent the splitter from misbehaving. +- **Multilingual text** may need a language-specific sentence splitter (e.g. spaCy's `xx_sent_ud_sm`). +- **Very short documents** (<2k chars): skip chunking entirely. Sentence-aware chunkers can output zero chunks on edge cases. + +## Cost / latency dial + +Roughly: + +| `max_chars` | Effect | +|---|---| +| 1000 | Highest recall, ~2× cost, slowest | +| 2000 | Default; good balance | +| 3000 | Lower cost; slightly more missed entities at chunk boundaries | +| 6000+ | Risk: prompt + chunk + output exceeds context window on small-context models | + +If a user says "slow" → raise `max_chars`. If they say "missing some entities at section breaks" → lower it. diff --git a/skills/structsense-skills/references/human-feedback.md b/skills/structsense-skills/references/human-feedback.md new file mode 100644 index 0000000..af25f4b --- /dev/null +++ b/skills/structsense-skills/references/human-feedback.md @@ -0,0 +1,119 @@ +# Human-in-the-loop feedback + +A simple pattern for inserting a human reviewer between automated stages. + +## When to enable + +- High-stakes outputs (clinical, regulatory). +- Active learning / curation workflows. +- Schema is new and you don't yet trust the model's outputs. +- Domain expert wants to catch edge cases before they propagate downstream. + +Skip when running unattended batch jobs. + +## The contract + +The human feedback stage: + +- **Receives:** the judge stage's output (full JSON). +- **Asks the human:** approve / abort / edit / skip. +- **Returns:** revised JSON with the same shape, plus `human_feedback_applied: true` and a `human_feedback_log` entry. + +The human **cannot rename keys, drop required fields, or change types**. They can: + +- Edit string values (e.g. fix `ontology_label` typos). +- Mark items as `verified: true`. +- Add `remarks` text. +- Remove individual items they consider hallucinated. + +If the human wants to add new items, they should re-run the extractor with a more specific prompt — don't paste freeform items into the JSON. + +## The four-option menu + +``` +1. Approve and continue — pipeline accepts the judge output verbatim +2. Abort pipeline — pipeline returns the judge output and stops; downstream stages don't run +3. Open editor to provide feedback — opens $EDITOR with a feedback area pre-populated +4. Skip feedback for this step — same as Approve but logs "user_skipped" +``` + +Default to **option 4 on a 60-second timeout** for unattended runs. Don't block a 10-hour pipeline on a human who walked away. + +## Editor template + +When the human picks option 3, present: + +``` +# === WRITE YOUR FEEDBACK HERE === +[WRITE YOUR FEEDBACK HERE] + +# === OUTPUT JSON (read-only reference; do not edit below this line) === +# { +# "entities": [ +# ... +# ] +# } +``` + +Closing without writing → return to menu (treat as "skip"). Saving with text → that text becomes `user_feedback_text`. + +## How the feedback agent applies feedback + +The feedback agent receives: + +- `judged_structured_information`: the full judge output. +- `user_feedback_text`: the freeform feedback string from the human. +- `modification_context`: optional structured context (which items were highlighted, etc.). + +Its job: emit the same JSON structure, revised according to the feedback. The system prompt should hardcode "**preserve structure; only revise according to feedback; never invent or drop items**" — see `prompts/humanfeedback.md`. + +## Audit trail + +Always append a `human_feedback_log` entry: + +```jsonc +{ + "human_feedback_log": [ + { + "timestamp": "2026-06-05T14:23:01Z", + "reviewer": "tekraj@mit.edu", + "action": "edited", // "approved" | "aborted" | "edited" | "skipped" + "user_feedback_text": "Item 7 says 'mice' but the source says 'rats'.", + "items_changed": [7], + "model": "openrouter/openai/gpt-4o-mini" // model used to apply the feedback + } + ] +} +``` + +This makes outputs reproducible and auditable months later. + +## Triggering by environment + +Most setups use a single env var: + +```bash +ENABLE_HUMAN_FEEDBACK=true +``` + +When false (default), the human feedback stage is skipped entirely and the pipeline returns the judge output. + +## Direct vs agent-loop implementation + +Two ways to run the feedback agent: + +- **Direct API** (default): send a single LLM call with the system prompt + the judge JSON + the feedback. Fast, predictable, cheap. +- **Agent loop** (CrewAI-style): the model can call tools to re-fetch source text, look up ontologies, etc. Useful if the feedback is "verify item 7 against the original PDF", but expensive. + +Default to direct API. Switch to agent loop only when the feedback explicitly requires tool calls. + +## Common feedback patterns + +| Feedback text | What the agent should do | +|---|---| +| "Item 7 should be label=Disease, not Phenotype." | Locate item by index, update `label`. Keep everything else. | +| "Drop the items in the references section, they're not real entities." | Remove items whose `paper_location` matches "References". Keep all others. | +| "All mentions of 'mouse' should map to NCBITaxon:10090." | Find items where `entity.lower() == "mouse"`, set `ontology_id` and friends. | +| "The whole alignment looks wrong, re-run with stricter prompts." | Don't try to apply this; pipeline operator should re-run with a different config. | + +The agent should *politely refuse* feedback it can't safely apply, returning an `errors[]` entry explaining why, rather than guessing. diff --git a/skills/structsense-skills/references/json-output-discipline.md b/skills/structsense-skills/references/json-output-discipline.md new file mode 100644 index 0000000..3fea7fb --- /dev/null +++ b/skills/structsense-skills/references/json-output-discipline.md @@ -0,0 +1,174 @@ +# JSON output discipline + +LLMs produce unreliable JSON unless you constrain them. This document is the playbook for getting clean, parseable, schema-conformant JSON every time. + +## Tier 1 — preventive prompting + +The cheapest fix is a strict system prompt. Every extractor / alignment / judge prompt must include: + +``` +You output STRICT JSON ONLY. +- No prose before or after. +- No markdown code fences. +- No comments inside the JSON. +- All strings use double quotes. +- All keys are strings; no trailing commas. +- If a field is unknown, set it to null. Never invent values. +If you cannot comply, output {"error": "<one-line reason>"} and nothing else. +``` + +Set `temperature: 0` (or `0.1` for tiebreakers). Set `top_p: 1`. + +If your model API supports it, use **JSON mode**: + +| Provider | How to enable | +|---|---| +| OpenAI / OpenRouter (GPT-4o, GPT-4.1) | `response_format={"type": "json_object"}` | +| OpenAI (structured outputs) | `response_format={"type": "json_schema", "json_schema": {...}}` — guarantees schema-valid output | +| Anthropic Claude | Add `"Respond with only valid JSON."` to system prompt; consider tool use with a JSON-typed tool input | +| Gemini | `response_mime_type="application/json"` | +| Ollama (most models) | Add `"format": "json"` to the request body | + +Structured outputs (OpenAI's JSON-schema mode, or constrained decoding for local models via Outlines/Guidance) make the model **physically incapable** of producing invalid JSON. Use them when available. + +## Tier 2 — parse + repair + +After receiving the response: + +```python +import json +from json_repair import repair_json + +def parse_or_repair(raw: str) -> dict | None: + # Try strict parse first + try: + return json.loads(raw) + except json.JSONDecodeError: + pass + + # Strip markdown fences if present + stripped = strip_fences(raw) + try: + return json.loads(stripped) + except json.JSONDecodeError: + pass + + # json-repair handles trailing commas, single quotes, Python None/True/False, truncated trailing braces + try: + return repair_json(stripped, return_objects=True) + except Exception: + return None +``` + +```python +import re +FENCE = re.compile(r"^```(?:json)?\s*\n?(.*?)\n?```$", re.DOTALL) +def strip_fences(s: str) -> str: + m = FENCE.match(s.strip()) + return m.group(1) if m else s +``` + +Install: `pip install json-repair`. + +See `scripts/json_repair.py` for the full helper. + +## Tier 3 — schema-driven re-prompt + +If Tier 2 still fails (or the result is parseable but schema-invalid): + +```python +import jsonschema + +try: + jsonschema.validate(parsed, schema) +except jsonschema.ValidationError as e: + repair_prompt = f""" +The JSON below is INVALID. Fix it to match this schema. Output ONLY the fixed JSON, no prose. + +SCHEMA: +{json.dumps(schema, indent=2)} + +ERROR: +{e.message} at path: {'.'.join(str(p) for p in e.absolute_path)} + +JSON TO FIX: +{json.dumps(parsed, indent=2)} +""" + parsed = call_llm(repair_prompt) + jsonschema.validate(parsed, schema) +``` + +Cap retries at 2. If still failing, return whatever's parseable plus `"errors": [...]`. + +## Tier 4 — last-resort partial recovery + +When the model truncated mid-output (token-limit exceeded) and even json-repair can't recover: + +1. Find the last balanced `}` or `]`. +2. Truncate the string there and re-parse. +3. If a list is at the top level, you'll lose the last (probably incomplete) item. That's fine — log a warning, don't fail. + +```python +def truncate_to_last_balanced(s: str) -> str: + depth, last_balanced = 0, -1 + in_string = False + escape = False + for i, ch in enumerate(s): + if escape: escape = False; continue + if ch == "\\": escape = True; continue + if ch == '"': in_string = not in_string; continue + if in_string: continue + if ch in "{[": depth += 1 + elif ch in "}]": + depth -= 1 + if depth == 0: + last_balanced = i + return s[: last_balanced + 1] if last_balanced > 0 else s +``` + +## Common malformations and what works + +| Malformation | Tier that fixes it | +|---|---| +| Wrapped in markdown fences | Tier 2 (strip_fences) | +| Trailing comma after last list item | Tier 2 (json-repair) | +| Python `True` / `False` / `None` | Tier 2 (json-repair) | +| Single quotes around keys/values | Tier 2 (json-repair) | +| Truncated by token limit | Tier 4 (last balanced) | +| Hallucinated extra fields | Tier 3 (schema re-prompt) | +| Required field missing | Tier 3 (schema re-prompt) | +| Wrong enum value | Tier 3 (schema re-prompt) | +| Float where int expected (or vice versa) | Tier 3 (schema re-prompt) | + +## Validation after parsing — beyond the schema + +JSON Schema can't check everything. After parsing, run domain-specific validators: + +```python +def validate_ner(parsed: dict, original_text: str) -> list[str]: + errors = [] + for i, e in enumerate(parsed.get("entities", [])): + # Span integrity + if original_text[e["start"]:e["end"]] != e["entity"]: + errors.append(f"entities[{i}]: span mismatch") + # Sentence containment + if e["entity"] not in e["sentence"]: + errors.append(f"entities[{i}]: entity not in sentence") + if e["sentence"] not in original_text: + errors.append(f"entities[{i}]: sentence not in source") + return errors +``` + +Drop items that fail; never silently keep broken spans. + +## Test your prompts with a torture set + +Maintain a small "torture test" of inputs known to break JSON: + +- Quotes inside string values +- Backslashes (Windows paths, LaTeX) +- Unicode emoji and right-to-left text +- Very long single-line outputs +- Inputs containing the exact tokens `{`, `}`, `:`, `,`, `"` in problematic positions + +Run your prompt against the torture set whenever you change it. A small CI script saves hours of production debugging. diff --git a/skills/structsense-skills/references/model-selection.md b/skills/structsense-skills/references/model-selection.md new file mode 100644 index 0000000..7cf3c26 --- /dev/null +++ b/skills/structsense-skills/references/model-selection.md @@ -0,0 +1,118 @@ +# Model selection per stage + +You don't need to use the same model for every stage. Pick the cheapest model that's *good enough* for each role. + +## Per-stage requirements + +| Stage | Most important capability | Acceptable second-tier | +|---|---|---| +| **Extractor** | Strong instruction-following + span fidelity (offsets must match) | Long context, tool use unhelpful | +| **Alignment (LLM-driven)** | Tool use, JSON-typed tool inputs, world knowledge of ontologies | Long context | +| **Alignment (direct tool)** | (no LLM) | — | +| **Judge** | Factual consistency reasoning, numeric calibration | Fast/cheap | +| **Human feedback** | Following imperative edit instructions | Fast/cheap | + +## Default picks (June 2026) + +| Stage | Default | Why | +|---|---|---| +| Extractor | `claude-sonnet-4-6` or `gpt-4.1` or `gpt-4o` | Strong span fidelity, good JSON, fast enough | +| Alignment | Direct tool call (no LLM) when possible | Cheapest & most accurate. See `ontology-mapping.md`. | +| Alignment (LLM fallback) | `claude-haiku-4-5` or `gpt-4o-mini` | Cheap and good at calling tools | +| Judge | `gpt-4o-mini` or `claude-haiku-4-5` | Per-item batched, cost-sensitive | +| Human feedback | Same as judge | Same constraints | + +Local-only: `ollama/qwen2.5:14b`, `ollama/deepseek-r1:14b`, `ollama/llama3.1:70b`. Use Q4 quantization for memory-constrained machines. + +## Configuration (provider prefixes) + +Most agent frameworks use these prefixes: + +| Provider | Model string | Base URL | +|---|---|---| +| OpenRouter | `openrouter/<provider>/<model>` e.g. `openrouter/anthropic/claude-sonnet-4-6` | `https://openrouter.ai/api/v1` | +| OpenAI direct | `openai/<model>` e.g. `openai/gpt-4o-mini` | (default OpenAI base) | +| Anthropic direct | `anthropic/<model>` | (default Anthropic base) | +| Ollama (local) | `ollama/<model>` e.g. `ollama/qwen2.5:14b` | `http://localhost:11434` | +| vLLM (self-hosted) | `openai/<model>` (vLLM speaks OpenAI API) | your vLLM server URL | +| Gemini | `gemini/<model>` | per provider docs | + +For OpenRouter, the same key works for any underlying model — handy for trying providers without re-keying. + +## Cost levers + +| Lever | Effect | +|---|---| +| Smaller extractor model | 5–10× cheaper but more missed entities and bad spans | +| Direct tool call for alignment | Removes ~50–80% of total cost on entity-heavy runs | +| `skip_judge_llm=true` (auto-approve) | Removes judge cost entirely | +| `direct_judge_api=true` (parallel API calls, no agent loop) | 3–5× cheaper than CrewAI agent loop | +| Larger `chunk_size` | Fewer extractor calls, but risk missed entities at chunk boundaries | +| Tighter `max_iter` (e.g. 3 instead of 20) | Stops runaway tool loops; harmless for well-prompted tasks | + +## Quality levers + +| Lever | Effect | +|---|---| +| Larger extractor model | Better recall + better spans | +| Smaller `chunk_size` (e.g. 1000) | Better recall at chunk boundaries; more cost | +| Multi-result ontology lookup + LLM re-rank | Better ontology IRIs when first hit is wrong | +| Two-pass extraction (rough → refine) | Better recall on long documents; ~2× cost | +| Human feedback stage | Catches systematic errors | + +## What to use when + +| Situation | Setup | +|---|---| +| Production NER on biomedical papers, budget OK | Extractor: `claude-sonnet-4-6`. Alignment: direct local-hybrid tool call. Judge: `gpt-4o-mini`. | +| Prototyping cheaply | Extractor + judge: `gpt-4o-mini`. Alignment: skip. | +| Air-gapped / on-prem | Everything in `ollama/<model>` (or vLLM). Use `qwen2.5:14b` or `llama3.1:70b`. | +| Highest accuracy, cost no object | Extractor: `claude-sonnet-4-6` with multi-result alignment + LLM re-rank with `gpt-4.1`. Judge: `claude-sonnet-4-6`. Human feedback enabled. | +| Reproducible / deterministic | Pin model versions explicitly. `temperature=0`. Use `seed=` when supported. | + +## Mixing models across stages + +A common cost-effective mix: + +```yaml +# Extractor — strong span fidelity +extractor: + model: openrouter/anthropic/claude-sonnet-4-6 + temperature: 0 + +# Alignment — tool-using small model (or skip) +alignment: + model: openrouter/anthropic/claude-haiku-4-5 + temperature: 0 + +# Judge — cheap factuality checker +judge: + model: openrouter/openai/gpt-4o-mini + temperature: 0 +``` + +Roughly 50–70% cheaper than running every stage on the strong model, with negligible quality loss on most tasks. + +## Picking ollama models for offline use + +| Model | RAM (Q4) | Good for | +|---|---|---| +| `qwen2.5:7b` | 6 GB | Quick prototyping; weaker JSON discipline | +| `qwen2.5:14b` | 10 GB | Solid extractor for most tasks | +| `qwen2.5:32b` | 20 GB | Production-quality extractor on a workstation | +| `llama3.1:70b` (Q4) | 40 GB | High quality; needs serious hardware | +| `deepseek-r1:14b` | 10 GB | Strong reasoning; useful for judge | +| `nomic-embed-text` | 0.5 GB | Embeddings only (for retrieval/memory) | + +Be aware: smaller local models often produce malformed JSON. Pair them with constrained decoding (Outlines, Guidance, llguidance) or aggressive Tier-3 schema re-prompting from `json-output-discipline.md`. + +## Token budgets + +For each call, budget: + +- ~5–15k tokens for system prompt + schema + instructions +- input chars / 4 for the user content +- 2× input tokens for output (NER on dense text produces a lot) +- 1–2k tokens for tool-use traces if alignment uses an agent loop + +If `system + input + expected_output ≥ context_window`, drop `chunk_size`. diff --git a/skills/structsense-skills/references/ner-extraction.md b/skills/structsense-skills/references/ner-extraction.md new file mode 100644 index 0000000..0ed40ec --- /dev/null +++ b/skills/structsense-skills/references/ner-extraction.md @@ -0,0 +1,169 @@ +# NER (named entity + key term) extraction + +## Goal + +Given unstructured text (paper, abstract, clinical note, news article), produce two lists: + +- **`entities`** — typed mentions of real-world things (proteins, brain regions, species, drugs, methods, people, organizations, …). +- **`key_terms`** — domain-salient phrases that aren't full entities but matter for retrieval (e.g. "long-term potentiation", "single-cell RNA-seq", "supply-chain risk"). + +This is a span-extraction task — the model must preserve `start`/`end` offsets so downstream consumers can highlight text and verify. + +## Pick the right extractor variant + +This skill ships **three NER extractor prompts**, each with its own label taxonomy. Pick by the domain of the source text. + +| Variant | Label taxonomy focus | Use when… | +|---|---|---| +| `prompts/extractor-ner-general.md` | Person, Organization, Location, Product, Event, Date, Time, Money, Percent, Quantity, Law, Language, Other | The text is news, finance, biographies, generic web content, customer support, or any non-specialized domain. Also a good first pass when you don't yet know the domain. | +| `prompts/extractor-ner-neuroscience.md` | Anatomy + cells + molecules + species + methods + measurements + behavior + disease across all of neuroscience | The text is a neuroscience paper, abstract, methods section, or review — *broad* coverage from behavior down to molecules. | +| `prompts/extractor-ner-cns-cells.md` | CellClass / CellType / CellSubtype + LineageMarker + Morphology + EphysProperty + Layer + Projection + AtlasReference + ProfilingMethod | The text is specifically about **CNS cells** — cell atlases, patch-seq, scRNA-seq cell typing, BICCN-style cell census, anything where the subject is "what cells live in this CNS region and how they differ." | + +The three variants share the same output schema (`entities[]`, `key_terms[]` with `start`/`end`/`sentence`); only the `label` taxonomy differs. That means downstream alignment / judge / merge logic doesn't change when you swap variants. + +When in doubt, run the **general** variant for a coarse first pass, then re-run with the more specific variant on chunks that contain domain-specific content. + +## Output schema + +See `schemas/ner-output.schema.json` for the formal schema. Minimum required: + +```jsonc +{ + "source_metadata": { // ONCE per run — NOT per item + "paper_title": "string", // optional + "doi": "string", // optional + "source_path": "string" // optional + }, + "entities": [ + { + "entity": "string", // surface form, exactly as in the text + "label": "string", // entity type (see "Label taxonomy" below) + "sentence": "string", // the full containing sentence + "start": 0, // char offset of `entity` in original text + "end": 0, // char offset (exclusive) of `entity` end + "paper_location": "string" // optional: section/page/paragraph (varies per mention) + } + ], + "key_terms": [ + { + "term": "string", + "sentence": "string", + "start": 0, + "end": 0, + "paper_location": "string" // optional + } + ] +} +``` + +> **Why `paper_title` and `doi` are NOT per-entity.** Earlier versions of this +> schema repeated these on every item. On a paper with 1500 entity mentions, +> that means 1500 copies of the same string — gigabytes wasted across a +> corpus run, and a needless burden on the LLM (which has to emit them +> faithfully on every item). Keep them once at the top level under +> `source_metadata`. `paper_location` (section / page / paragraph) DOES vary +> per mention and stays per-entity. + +## Label taxonomy + +Each extractor variant ships with its own closed taxonomy embedded in its system prompt. **Do not let the model invent new labels mid-run** — the parser should reject anything off-vocabulary and reprompt. + +If the user wants a custom taxonomy: + +1. Start from the closest variant. +2. Edit the `LABEL TAXONOMY` block in the prompt — keep it explicit, with one short description per label. +3. Update the alignment routing table (in the same prompt file) so each new label has a target ontology. +4. Test against a small annotated sample before running at scale. + +## Span discipline + +Two failure modes the model loves: + +1. **Hallucinated spans.** Model emits an `entity` text that doesn't actually appear in the source. **Mitigation:** after parsing the JSON, verify `text[start:end] == entity` for every item. Drop or repair items that fail. +2. **Off-by-one offsets.** **Mitigation:** ask the model to also emit `sentence`, then verify the sentence appears in the text and that `entity` is a substring of `sentence` at the right relative offset. + +```python +def validate_span(text, item): + return ( + text[item["start"]:item["end"]] == item["entity"] + and item["sentence"] in text + and item["entity"] in item["sentence"] + ) +``` + +## Key-term vs entity disambiguation + +A useful working rule: + +- **Entity** if you could draw a line from the surface form to a row in a database (gene, drug, region…). +- **Key term** if it's a phrase a human would search for but isn't a single referent (workflow names, paradigms, technique families). + +When in doubt, emit it as a `key_term`; the alignment stage handles both. + +## Chunked extraction (long papers) + +Long documents must be chunked. For NER, **chunk at sentence boundaries** so spans don't cross chunks. Pseudocode: + +```python +chunks = split_into_sentence_chunks(text, max_chars=2000) +chunk_results = [extract_ner(chunk, base_offset=offset) for offset, chunk in chunks] +merged_entities = flatten([r["entities"] for r in chunk_results]) +# Dedup by the full SPAN triple — NOT by surface form. Different occurrences +# of the same surface form have different (start, end) and must all survive. +merged_entities = deduplicate(merged_entities, key=("entity", "start", "end")) +``` + +`base_offset` matters: the extractor reports offsets relative to the chunk it sees. You must add the chunk's start offset back into the source text when merging — otherwise every chunk says "starts at 17" and the offsets are meaningless. + +## Exhaustiveness — extract every occurrence + +This skill is built around **exhaustive** NER. Models default to one-row-per-unique-surface-form behaviour unless you push them otherwise. All three extractor prompts now contain explicit "extract EVERY occurrence" language. **Every mention of every entity must be emitted as its own item**, with its own distinct `start`/`end`. A multi-page neuroscience paper should yield hundreds to thousands of entity items, not a few hundred. + +If your yield feels far short of what `structsense` or a careful human would produce, two things to check first: + +1. **Are you running the mask-recall pass?** See the next section. +2. **Are you post-deduplicating by surface form?** Don't. Span-level dedup (`(entity, start, end)`) is fine and preserves multiple occurrences; surface-level dedup destroys them. + +## Two-pass strategy: mask-mode + +The single most reliable way to push recall close to exhaustive is the **mask-recall pass**: + +1. Run any of the three NER extractor prompts (pass-1). +2. Mask every extracted span in the source text with a `[E<i>]` placeholder, padded to preserve character offsets (`scripts/mask_pass.py:mask_for_recall`). +3. Re-run with `prompts/mask-recall-pass.md` over the masked text. The model surfaces mentions pass-1 missed. +4. Translate the new mentions' offsets back to the original text (`scripts/mask_pass.py:map_masked_offsets_to_original`). +5. Run span validation and append survivors to pass-1 results. + +Typical recovery on a neuroscience paper is **+30–80%** more mentions over pass-1 alone — exactly the gap users notice between this skill and structsense. + +For precision/quality work, also run **mask-verify** (`prompts/mask-verify-pass.md`): replace each extracted span (one at a time) with `[MASK]` and ask the model to predict the label from context. Disagreement signals candidate label errors that the judge can down-weight. See `scripts/mask_pass.py:mask_one_for_verify`. + +Stop at two passes. Three or more gives diminishing returns and starts hallucinating mentions. + +## Dedup rules + +Span-level dedup is OK and recommended (it removes duplicates introduced when chunks overlap or pass-1 + mask-recall surface the exact same span twice). Surface-level dedup is **not** OK — it destroys multiple-occurrence information. + +Across chunks (and within a chunk): + +- Exact `(entity_text, start, end)` match → keep one copy. +- Same `entity_text` in different sentences → keep ALL copies (each is a distinct mention). +- Same `entity_text` in the same sentence at different positions → keep ALL copies. +- Same `entity_text` at the same `start`/`end` (true duplicate from chunk overlap or pass-1 + mask-recall surfacing the same span) → keep one copy. + +## What to pass downstream + +The alignment stage receives the full NER output verbatim. Don't reformat. Don't add a wrapper key. The next prompt expects: + +```json +{"entities": [...], "key_terms": [...]} +``` + +If you wrap it as `{"extracted_data": {"entities": ...}}`, the alignment prompt's placeholder won't resolve and you'll get nonsense. + +## Edge cases + +- **Acronyms with expansions** ("hippocampus (HP)"): emit both the long form and the acronym as separate entities sharing a `label`. The alignment stage will collapse them to the same ontology IRI. +- **Negated mentions** ("no significant change in BDNF expression"): still emit the entity; let downstream consumers handle negation. +- **Tables and figure captions**: chunk them separately if possible — entity density is much higher there. +- **Equations / Greek letters**: usually not entities. The extractor should ignore them unless they name a measured quantity. diff --git a/skills/structsense-skills/references/ner-models.md b/skills/structsense-skills/references/ner-models.md new file mode 100644 index 0000000..249aaba --- /dev/null +++ b/skills/structsense-skills/references/ner-models.md @@ -0,0 +1,173 @@ +# Multi-model NER ensemble (HuggingFace + LLM) + +The skill ships an **ensemble extractor** that runs specialized HuggingFace NER models in parallel with the LLM extractor. Every mention carries a `source_model` field so the grouped view (`entities_grouped`) records which model(s) surfaced each entity. This mirrors the multi-model strategy structsense uses (`utils/ner_tool.py`). + +## Why ensemble? + +A single LLM extractor, even when prompted exhaustively and run with mask-recall, still misses domain-specific mentions that specialist models catch easily — and vice versa. Combining them: + +- **HF specialists** are deterministic, fast, and trained on labeled biomedical corpora. They reliably catch high-frequency biomedical entities (genes, chemicals, diseases, anatomy). +- **LLM extractors** generalize better to surface forms not in any training set (novel cell-type names, atlas cluster IDs, jargon) and infer typed labels from context. +- **Together**, recall is higher than either alone, and the `consensus_count` (how many distinct models surfaced an entity) is a cheap, principled confidence proxy. + +## Available profiles + +Pick a domain-appropriate roster via `--ner-profile` on the CLI or `ner_ensemble_profile=...` from Python. Available profiles in `scripts/ner_models.py`: + +| Profile | Models | When to use | +|---|---|---| +| `biomedical_broad` | d4data + BENT (Disease, Chemical, Gene, Organism) | General biomedical text. The recommended default. | +| `cns_cells` | BENT (Cell-Type, Cell-Line, Anatomical, Gene) + BioBERT-genetic + d4data | Cell atlases, patch-seq, scRNA-seq cell typing, CNS taxonomy work. | +| `pharmacology` | BC5CDR-chem + BENT (Chemical, Disease) + NCBI-disease | Drug/chemical/disease-focused work (pharmacology papers, clinical trial reports). | +| `genetic` | BioBERT-genetic + BENT (Gene) + d4data | Gene/protein-heavy text. | +| `clinical` | Clinical-AI-Apollo + blaze999 + d4data + BENT (Disease) | EHR notes, clinical case reports. | +| `minimal` | d4data + BENT (Gene, Disease) | Smallest sensible roster — fastest cold-start. | +| `all` | All of the above (the full default roster) | Maximum recall; longer warm-up while models load. | + +Or pass an explicit list via `--ner-models hf/model-id-1,hf/model-id-2,...`. + +## Default roster (== profile `all`) + +| Model ID | Specialty | +|---|---| +| `d4data/biomedical-ner-all` | Broad biomedical | +| `mobashgr/BC5CDR-chem-WLT-384-BioELECTRA-Pubmed-ENS-20-5` | Chemicals (BC5CDR) | +| `mobashgr/NCBI-disease-WLT-256-SciBERT-13INS` | Diseases (NCBI-disease) | +| `alvaroalon2/biobert_genetic_ner` | Genes / proteins | +| `pruas/BENT-PubMedBERT-NER-Gene` | Genes (BENT family) | +| `pruas/BENT-PubMedBERT-NER-Chemical` | Chemicals | +| `pruas/BENT-PubMedBERT-NER-Disease` | Diseases | +| `pruas/BENT-PubMedBERT-NER-Anatomical` | Anatomy / brain regions | +| `pruas/BENT-PubMedBERT-NER-Cell-Type` | Cell types | +| `pruas/BENT-PubMedBERT-NER-Cell-Line` | Cell lines | +| `pruas/BENT-PubMedBERT-NER-Organism` | Species | +| `pruas/BENT-PubMedBERT-NER-Bioprocess` | Biological processes | + +Plus the LLM extractor, which contributes items tagged `source_model: "llm_ner:<llm_model_string>"`. + +## How it runs + +1. The pipeline detects you've requested an ensemble (`--ner-profile` or `--ner-models`). +2. Each configured model is loaded in parallel via `concurrent.futures.ThreadPoolExecutor` (lazy: `transformers` is imported only when the ensemble is enabled). Models that fail to load are logged and skipped — never crash the run. +3. Each model emits its own mention list with `source_model` set. +4. The LLM extractor runs in parallel via the standard `prompts/extractor-ner-*.md` path and its items get `source_model: "llm_ner:<model>"`. +5. The four lists are concatenated (no dedup at this stage — the grouped view handles consolidation). +6. The grouped view (`entities_grouped`) collapses by `(entity.lower(), label)` and records: + - `source_models`: sorted list of all contributing model IDs. + - `source_model_counts`: how many mentions came from each model. + - `consensus_count`: distinct number of contributing models. +7. Stats break down both the raw `entities[]` and the grouped view by `source_model` (see `stats.by_source_model`). + +## Per-mention output shape (with `source_model`) + +```jsonc +{ + "entity": "BDNF", + "label": "Gene", + "sentence": "BDNF is upregulated in the hippocampus.", + "start": 0, + "end": 4, + "paper_location": "Results", + "source_model": "alvaroalon2/biobert_genetic_ner", + "source_score": 0.998, + // populated by the alignment + judge stages … + "ontology_id": "http://identifiers.org/hgnc/1033", + "ontology_label": "BDNF", + "ontology": "HGNC", + "concept_mapping_provenance": "tool", + "judge_score": 1.0 +} +``` + +## Per-group output shape (in `entities_grouped`) + +```jsonc +{ + "entity": "BDNF", + "label": "Gene", + "mention_count": 17, + "source_models": [ + "alvaroalon2/biobert_genetic_ner", + "d4data/biomedical-ner-all", + "llm_ner:openrouter/anthropic/claude-sonnet-4-6", + "pruas/BENT-PubMedBERT-NER-Gene" + ], + "source_model_counts": { + "alvaroalon2/biobert_genetic_ner": 17, + "d4data/biomedical-ner-all": 12, + "llm_ner:openrouter/anthropic/claude-sonnet-4-6": 17, + "pruas/BENT-PubMedBERT-NER-Gene": 15 + }, + "consensus_count": 4, + "ontology_id": "http://identifiers.org/hgnc/1033", + "ontology_label": "BDNF", + "ontology": "HGNC", + "judge_score_max": 1.0, + "judge_score_avg": 0.95, + "judge_score_min": 0.85, + "sentences": [ + {"text": "BDNF is upregulated in the hippocampus.", + "paper_locations": ["Results"]}, + {"text": "BDNF protein levels increase following exercise.", + "paper_locations": ["Discussion"]} + ], + "mentions": [ + {"start": 0, "end": 4, "source_model": "alvaroalon2/biobert_genetic_ner", "judge_score": 1.0, "...": "..."}, + {"start": 348, "end": 352, "source_model": "d4data/biomedical-ner-all", "judge_score": 0.95, "...": "..."}, + "..." + ] +} +``` + +`consensus_count=4` here means four distinct models agreed BDNF is an entity. Highly trustworthy. A `consensus_count=1` from only the LLM extractor is weaker signal — useful for low-frequency or novel mentions but worth checking. + +## Stats break down by `source_model` + +`stats.entities.by_source_model` reports how many raw mentions each model contributed: + +``` +ENTITIES: 1132 mentions, 287 unique (3.94x avg) [dropped: 0] + by_label: Gene=312, Protein=198, BrainRegion=87, Disease=72, … + by_source_model: alvaroalon2/biobert_genetic_ner=412, + llm_ner:openrouter/anthropic/claude-sonnet-4-6=287, + d4data/biomedical-ner-all=215, + pruas/BENT-PubMedBERT-NER-Gene=124, + pruas/BENT-PubMedBERT-NER-Disease=72, + pruas/BENT-PubMedBERT-NER-Anatomical=22 +``` + +This is the diagnostic you reach for when yield feels wrong: which model is over- or under-contributing relative to expectations. + +## Cost and latency + +- **First run is slow** because HF model weights download. With the full default roster, expect 2–5 GB of downloads (cached locally afterwards). +- **Subsequent runs** are fast: weights load in 5–15 seconds; inference on a 30-page paper takes 1–3 minutes per model on CPU, or 5–15 seconds per model on a single GPU. +- **CPU vs GPU**: pass `--ner-device 0` (or any CUDA device index) to use a GPU. Default is `-1` (CPU). +- **`profile=minimal`** keeps cold-start under a minute and trades some recall for speed. + +## When to skip the ensemble + +- **Prototyping**: just the LLM extractor is fine. Skip `--ner-profile`. +- **Domain not covered by the available models**: e.g. astronomy, finance. Use only the LLM extractor. +- **CPU-only, latency-critical**: profile=minimal or skip. +- **Air-gapped without pre-downloaded model weights**: skip. + +## When the ensemble doesn't help (and what to do) + +| Symptom | Likely cause | Fix | +|---|---|---| +| `ensemble_models` shows all models skipped with "transformers not installed" | `pip install transformers torch` | +| All ensemble models contribute 0 items | Wrong domain (e.g. running biomedical models on news text) | Switch profile (`general`-only via LLM extractor) | +| Many duplicate mentions of the same span across models | Working as intended — `consensus_count` rises, `entities_grouped` consolidates them | No action; check `entities_grouped` not `entities` | +| `source_model` is missing on some entities | LLM extractor wasn't tagged. `pipeline.run()` calls `annotate_llm_provenance()` automatically; if you wired it manually, do that step. | — | + +## Adding new models + +To add a new HuggingFace token-classification model: + +1. Verify it's a token-classification model (BERT/RoBERTa/ELECTRA family with an NER head). The runner uses `pipeline("ner", ...)` from `transformers`. +2. Add to `DEFAULT_HF_MODELS` in `scripts/ner_models.py`, or to a relevant profile. +3. If the model's raw labels differ from this skill's canonical taxonomy, add entries to `_LABEL_NORMALIZATION` so they map cleanly. Anything unmapped passes through verbatim. +4. The runner is generic; no per-model glue code is needed. + +Sequence-classification or generative NER models aren't supported by the default runner — wrap them in a custom function and add to `jobs` in `run_ensemble`. diff --git a/skills/structsense-skills/references/ontology-mapping.md b/skills/structsense-skills/references/ontology-mapping.md new file mode 100644 index 0000000..061da1a --- /dev/null +++ b/skills/structsense-skills/references/ontology-mapping.md @@ -0,0 +1,253 @@ +# Ontology mapping (concept alignment) + +Map free-text terms to ontology IRIs + labels. Four backends are useful; pick by cost, quality, and infra constraints. + +| Backend | Setup | Quality | Speed | Cost | +|---|---|---|---|---| +| **BioPortal REST API** | API key | High (curated) | ~1 req/sec (rate-limited) | Free + your time | +| **OLS REST API** (EBI) | None | High (EBI-curated) | Fast | Free | +| **Local hybrid BM25 + dense retrieval** | Self-host a service (e.g. [search_hybrid](https://github.com/sensein/search_hybrid)) | Tunable, very high if re-ranked | Fastest (batched) | Infra | +| **LLM-only (no tool)** | Just prompting | Hallucinates IRIs | Fast | LLM tokens | + +**Rule of thumb:** prefer **tool-based** mapping (BioPortal/OLS/local) and use the LLM only to choose between candidates or fill obvious gaps. Mark every output with `concept_mapping_provenance`: `"tool"` or `"llm_knowledge"`. + +## Recommended cascade (production default) + +The reference pipeline (`scripts/pipeline.py`) uses this order by default: + +1. **Local hybrid service** at `http://localhost:8000` (the [search_hybrid](https://github.com/sensein/search_hybrid) reference implementation). Verify it's up by visiting **`http://localhost:8000/docs`** — every FastAPI-based deployment serves the interactive OpenAPI page there. The pipeline health-checks `/health` then `/docs`. +2. **BioPortal** (if `BIOPORTAL_API_KEY` is set). Triggered automatically if the local service is unreachable. +3. **Ask the user** for an alternative local URL. Deployments often use non-default ports (8001, 9000, behind a reverse proxy at `/concept-map/`, etc.) — the cascade prompts for an override and retries the local backend with the user-provided URL. +4. **Skip alignment entirely** with `concept_mapping_provenance: "skipped"` on every item, only if the user declines to provide a URL. The run still completes; only ontology fields are unpopulated. + +When this skill is used inside an LLM agent (Claude Code, GPT custom action, etc.), the agent should **ask the user via natural language** if the cascade exhausts its defaults — the port/host varies enough across deployments that a default-only check is not enough. Example: "I couldn't reach a concept-mapping service at http://localhost:8000. What URL is your local service running on, or should I fall back to BioPortal?" + +The cascade builder is `scripts/pipeline.py::build_mapper_with_cascade`. Use `ask_user=stdin_ask_callback` for CLI use, or pass your own `ask_user(prompt: str) -> Optional[str]` callable to route the prompt through your UI. + +## Output format (every backend produces this) + +For each extracted term: + +```jsonc +{ + "term": "hippocampus", + "ontology_id": "http://purl.obolibrary.org/obo/UBERON_0002421", + "ontology_label": "hippocampal formation", + "ontology": "UBERON", + "concept_mapping_provenance": "tool" // or "llm_knowledge" or "skipped" +} +``` + +Keep this shape constant across backends. If you change backend, downstream code doesn't change. + +## Backend 1: BioPortal REST + +Docs: https://data.bioontology.org/documentation +Get a key at https://bioportal.bioontology.org/account. + +### Endpoint + +`GET https://data.bioontology.org/search?q={term}&apikey={key}&display_context=false&include=prefLabel,definition` + +Optional filters: +- `ontologies=UBERON,CL,NCBITAXON` to constrain to specific ontologies. +- `require_exact_match=true` for strict matching. +- `pagesize=10` for top-N candidates. + +### Throttling + +BioPortal rate-limits aggressive callers. Use: + +- `BIOPORTAL_REQUEST_INTERVAL=0.7` seconds between requests (raise to 1.0+ if you see 429s). +- Exponential backoff on 429: `BIOPORTAL_BACKOFF_AFTER_429=2.0` seconds initial; double on each retry. +- A small in-memory LRU cache by `term` (e.g. 2000 entries). Many papers re-mention the same terms. + +See `scripts/bioportal_map.py` for a ready-to-use client. + +## Backend 2: OLS (EBI Ontology Lookup Service) + +Docs: https://www.ebi.ac.uk/ols4/help + +### Endpoint + +`GET https://www.ebi.ac.uk/ols4/api/search?q={term}&exact=false&rows=10` + +Optional filters: +- `ontology=uberon,cl,ncbitaxon` (lowercase, comma-separated). +- `exact=true` for strict matching. +- `type=class` to skip properties/individuals. + +OLS is free and well-rate-limited; usable in batch without API keys. + +See `scripts/ols_map.py`. + +## Backend 3: Local hybrid retrieval + +For high throughput (thousands of terms in seconds), self-host a service that combines: + +1. **BM25** over ontology labels + synonyms (lexical). +2. **Dense embedding retrieval** (e.g. `nomic-embed-text`, `intfloat/e5-base`) over the same. +3. **Re-ranking** with a cross-encoder or an LLM. + +The reference implementation is at https://github.com/sensein/search_hybrid (branch `dev`). It exposes (verified against a live deployment): + +``` +POST /map/batch +Content-Type: application/json +{ + "max_results": 5, + "text": [ + { "text": "kidney disease", "context": "progressive decline in GFR" }, + { "text": "T2DM", "context": "type 2 diabetes with insulin resistance" }, + { "text": "astrocyte" } + ] +} +``` + +The top-level field is **`text`**, not `terms`. Each item is `{text, context?}`. The optional `context` field dramatically improves disambiguation — for NER, pass the entity's containing sentence as `context`. + +Returns: + +```jsonc +{ + "query": "...", + "type": "batch", + "results": { + "kidney disease": [ + { "rank": 1, "ontology_id": "http://purl.obolibrary.org/obo/HP_0012622", + "ontology_label": "Chronic kidney disease", "ontology": "CIDO", + "final_score": 1.0, "original_score": 0.249, "llm_score": 0.0, + "late_interaction_score": 1.0 }, + // ... up to max_results candidates, pre-ranked by `rank` + ], + "T2DM": [ /* ... */ ], + "astrocyte": [ /* ... */ ] + } +} +``` + +Important: `results` is a **dict keyed by the input `text`**, not a list parallel to inputs. Use `final_score` (not `score`). + +**One HTTP call, thousands of terms.** This is the dominant cost optimization for large-scale runs. + +For a direct, Python-free workflow (just `curl` + `jq`), see [`prompts/alignment-via-http.md`](../prompts/alignment-via-http.md) — it gives the LLM an exact bash template. + +**Interactive docs:** open `http://localhost:8000/docs` in a browser to see the OpenAPI schema and try the endpoints. This is also the most reliable "service is up" probe — the pipeline's health check tries `/health` first and falls back to `/docs`. + +**The URL varies.** Most users run on `localhost:8000`, but ports `8001`, `8080`, `9000`, and reverse-proxied paths like `https://internal.example.com/concept-map/` are all common. The pipeline's cascade asks the user for an override when the default doesn't reach the service — see "Recommended cascade" above. + +See `scripts/local_hybrid_map.py`. + +### Runtime reachability — "localhost" doesn't mean the same thing everywhere + +The most common cause of "no mapper available" is the LLM giving up without probing. Before you tell the user the mapper is unreachable, **you must attempt at least one HTTP probe** in whatever runtime you have: + +```bash +curl -s -o /dev/null -w '%{http_code}\n' http://localhost:8000/docs +# 200 = service reachable. Connection refused = either the service is down, +# OR your runtime can't reach the user's machine (see table below). +``` + +| Runtime | Can reach the user's `localhost`? | What to do if not | +|---|---|---| +| **Claude Code (CLI)** | ✅ Yes — Bash runs on the user's machine. | — | +| **Cursor / local agent IDEs** | ✅ Usually yes. | — | +| **claude.ai web app** | ❌ No — Claude runs in Anthropic's cloud. | Ask the user to expose the service via an MCP bridge (`connecting/mcp-server.md`) or a tunnel (`ngrok`, `cloudflared`, Tailscale). | +| **ChatGPT custom GPT** | ❌ No. | Wire a server-side Action with a public URL (`connecting/custom-gpt.md`). | +| **Anthropic Skills hosted runtime** | ❌ No (no outbound localhost). | Same — MCP bridge or public URL. | + +**Symptom you've seen:** the user has the service running and verified it with curl, but the LLM still says "not available". This always means one of: + +1. The LLM's runtime can't reach the user's machine (web app, hosted runtime). → Set up a bridge. +2. The LLM never probed at all. → Strict instruction: "you MUST run `curl ${MAPPER_URL}/docs` before declaring the mapper unavailable." +3. The LLM probed but used the wrong API schema (e.g. `terms:` instead of `text:`). → See the corrected schema above. + +## Backend 4: LLM-only (fallback) + +When you have no tool, prompt the LLM: + +``` +For each term, return the most likely ontology IRI, label, and ontology shortname. +If you are not at least 80% confident, return null. +Output strict JSON keyed by term. +``` + +Mark every returned mapping as `"concept_mapping_provenance": "llm_knowledge"`. Treat these as **hints, not ground truth.** Always validate IRIs against an actual ontology before trusting them downstream — LLMs invent plausible-looking IRIs. + +### IRI validation + +```python +import re + +OBO_RE = re.compile(r"^http://purl\.obolibrary\.org/obo/[A-Z]+_\d+$") +NCBITAXON_RE = re.compile(r"^NCBITaxon:\d+$|^http://purl\.obolibrary\.org/obo/NCBITaxon_\d+$") + +def looks_real(iri: str) -> bool: + return any(p.match(iri) for p in (OBO_RE, NCBITAXON_RE, ...)) +``` + +Ideally, resolve the IRI (HEAD request) before accepting it. + +## Combining backends (recommended) + +Most production setups combine two backends: + +1. **Fast path:** local hybrid retrieval for every term in one batch call. +2. **Fallback:** for terms the local service marks as low-confidence (score < threshold), call BioPortal or OLS for a second opinion. +3. **LLM re-rank:** if you have multiple candidates, pass them + the original sentence to an LLM and ask which is correct. + +Pseudocode: + +```python +local = local_hybrid.batch(terms, max_results=5) + +low_conf = [t for t, hits in zip(terms, local) if hits[0].score < 0.7] +fallback = bioportal.batch(low_conf, max_results=3) + +# Re-rank top candidates with LLM +final = [] +for term, candidates, sentence in zip(terms, merged_candidates, sentences): + if len(candidates) == 1: + final.append(candidates[0]) + else: + choice = llm_pick_best(term, candidates, sentence) + final.append(choice) +``` + +## Choosing the right ontology + +For biomedical text, a good default routing: + +| Term type | First-choice ontology | +|---|---| +| Species / organism | NCBITaxon | +| Anatomy / brain region | UBERON | +| Cell type | CL (Cell Ontology) | +| Gene | NCBIGene / HGNC | +| Protein | UniProt / PR (Protein Ontology) | +| Disease | MONDO / DOID | +| Phenotype | HP (Human Phenotype) / MP (Mouse Phenotype) | +| Chemical / drug | CHEBI / DrON | +| Tissue / cell line | BTO (BRENDA Tissue Ontology) | +| Method / assay | OBI / EFO | + +The mapping tool should accept an `ontologies=` filter; pre-filter by entity `label` to avoid noisy cross-domain matches. + +## Provenance discipline + +Every mapped item must carry **two** provenance markers: + +- `concept_mapping_provenance`: where the mapping came from (`tool` / `llm_knowledge` / `skipped`). +- `alignment_method`: how the alignment stage was run (`direct_tool_call` / `llm_agent` / `skipped`). + +This is the single most useful piece of metadata for debugging weird outputs months later. + +## When to skip alignment + +Skip the alignment stage entirely when: + +- The downstream consumer doesn't need IRIs (e.g. just feeding the entity list to a UI). +- The text is in a domain with no good ontology coverage (e.g. computational methods names, dataset names). +- You're prototyping and want to validate the extraction step in isolation. + +Mark every item with `concept_mapping_provenance: "skipped"` so the absence is explicit, not implicit. diff --git a/skills/structsense-skills/references/pipeline-pattern.md b/skills/structsense-skills/references/pipeline-pattern.md new file mode 100644 index 0000000..77dd9a1 --- /dev/null +++ b/skills/structsense-skills/references/pipeline-pattern.md @@ -0,0 +1,174 @@ +# Pipeline pattern + +The whole skill rests on a four-stage agent pattern. Each stage is a single LLM call (or a parallel set of LLM calls if the input was chunked). Stages pass JSON between them — never freeform text. + +``` +extractor → alignment → judge → humanfeedback (optional) +``` + +## Stage contract + +Each stage has a strict input/output contract. **Treat it as a function signature.** + +| Stage | Input | Output | +|---|---|---| +| **Extractor** | `{ "text": <raw text>, "metadata": {paper_title?, doi?, paper_location?} }` | Items keyed by task: `entities[]`, `key_terms[]`, `resources[]`, or your custom schema. Each item has `start`, `end`, `sentence` (source-span fields), plus task-specific fields. | +| **Alignment** | Extractor output (verbatim) | Same structure, with `ontology_id`, `ontology_label`, `ontology`, `concept_mapping_provenance` added to each item. **No fields removed.** | +| **Judge** | Alignment output (verbatim) | Same structure, with `judge_score` (float 0–1) and `remarks` (string) added to each item. **No fields removed.** | +| **Human feedback** | Judge output + `{modification_context, user_feedback_text}` | Same structure, revised per the human's input. | + +## Why four stages and not one mega-prompt + +Separation lets you: + +- **Use different models per stage.** Extraction often needs a strong instruction-following model; judging can use a smaller, faster one. Alignment can use a tool-using model or skip the LLM entirely. +- **Cache and resume.** Each stage's output is saved as JSON. If the judge fails, re-run only the judge. +- **Swap implementations.** Replace LLM alignment with a direct call to a concept-mapping service. Replace LLM judging with rule-based scoring. +- **Parallelize.** Chunked inputs run extractor in parallel; downstream stages can run per-chunk too. + +## Chain the stages + +In pseudocode: + +```python +def pipeline(text, schemas, models, mapping_backend): + extraction = call_llm( + model=models["extractor"], + system=PROMPT_EXTRACTOR, + user={"text": text}, + response_schema=schemas["extractor"], + ) + + if mapping_backend == "tool": + alignment = call_mapping_tool_batch(extraction) # direct, no LLM + for item in flatten(alignment): + item["concept_mapping_provenance"] = "tool" + else: + alignment = call_llm( + model=models["alignment"], + system=PROMPT_ALIGNMENT, + user={"extracted": extraction}, + tools=[concept_mapping_tool_schema], + response_schema=schemas["alignment"], + ) + + judgment = call_llm( + model=models["judge"], + system=PROMPT_JUDGE, + user={"aligned": alignment}, + response_schema=schemas["judge"], + ) + + if human_feedback_enabled: + user_feedback = ask_human(judgment) + if user_feedback: + judgment = call_llm( + model=models["humanfeedback"], + system=PROMPT_HUMANFEEDBACK, + user={"judged": judgment, "feedback": user_feedback}, + ) + + return judgment +``` + +## Skip a stage + +Sometimes you don't need every stage. Common variants: + +| Situation | Drop | +|---|---| +| You trust alignment and don't need scoring | judge | +| You only want raw entities, no ontology mapping | alignment, judge | +| You have no humans in the loop | humanfeedback | +| You want only extraction with ontology hints | judge, humanfeedback | + +When you skip a stage, the previous stage's output is the final output (or the input to the next non-skipped stage). Mark the skipped stage's fields with a sentinel: + +- Skipped alignment → set `concept_mapping_provenance: "skipped"` on each item. +- Skipped judge → set `judge_score: 1.0`, `remarks: "auto-approved"`, `judge_method: "auto_approved"`. + +## Bypass LLM for alignment when you have a tool + +Alignment is the most expensive stage when run via an LLM agent loop (because the model calls the tool one item at a time, then narrates). If you have a concept-mapping tool with a batch endpoint, **call it directly** with the full list of extracted terms, skip the LLM entirely, and merge the tool's response into the extraction. + +```python +terms = [item["entity"] for item in extraction["entities"]] +mappings = mapping_tool.batch(terms, max_results=1) # one POST +for item, mapping in zip(extraction["entities"], mappings): + item.update({ + "ontology_id": mapping["id"], + "ontology_label": mapping["label"], + "ontology": mapping["ontology"], + "concept_mapping_provenance": "tool", + }) +``` + +This turns hours of LLM time into seconds. Output records `alignment_method: "direct_tool_call"`. + +## Run stages in parallel (chunked input) + +For long inputs, after chunking: + +1. **Extraction** — run the extractor per chunk in parallel (`asyncio.gather`, thread pool, etc). +2. **Merge** — concatenate items across chunks. Deduplicate by `(entity_text, start, end, paper_location)` or your stable key. +3. **Alignment / judge / humanfeedback** — run in parallel batches (e.g. 100 items per batch). +4. **Re-merge** — collect batch outputs back into a single result. + +See `chunking-strategy.md` for the math. + +## Persistence: save each stage's output + +Write stage outputs as numbered files so you can resume after a crash: + +``` +00_extractor.json +01_alignment.json +02_judge.json +03_humanfeedback.json +``` + +To restart from a later stage, load the saved file as the input and skip earlier stages. The user shouldn't have to re-run a 40-minute extraction because the judge timed out. + +## Stage budgets (safety rails) + +Without limits, an LLM agent can loop indefinitely on tool calls or bad parses. Set hard caps per stage: + +| Limit | Default | Why | +|---|---|---| +| Max reasoning iterations | 5–20 | Stops infinite ReAct loops. | +| Max wall-clock per item | 30–60 s | Stops hanging requests. | +| Max retries on parse error | 1–3 | Retries help; infinite retries hide bugs. | + +If a stage exhausts its budget, emit the partial result with `errors: [...]` populated and continue to the next stage. Never silently drop items. + +## Provenance map (what every item should carry by the end) + +After a full run, every item should look like: + +```jsonc +{ + // from extractor + "entity": "hippocampus", + "label": "BrainRegion", + "sentence": "We recorded from CA1 pyramidal cells in the hippocampus.", + "start": 32, + "end": 43, + "paper_location": "page 3, paragraph 2", + "paper_title": "...", + "doi": "10.1234/...", + + // from alignment + "ontology_id": "http://purl.obolibrary.org/obo/UBERON_0002421", + "ontology_label": "hippocampal formation", + "ontology": "UBERON", + "concept_mapping_provenance": "tool", // or "llm_knowledge" or "skipped" + "alignment_method": "direct_tool_call", // or "llm_agent" or "skipped" + + // from judge + "judge_score": 0.92, + "remarks": "Confident match; consistent with surrounding context.", + "judge_method": "llm" // or "auto_approved" +} +``` + +This is the contract every downstream consumer should be able to rely on. diff --git a/skills/structsense-skills/references/resource-extraction.md b/skills/structsense-skills/references/resource-extraction.md new file mode 100644 index 0000000..3c0ef3e --- /dev/null +++ b/skills/structsense-skills/references/resource-extraction.md @@ -0,0 +1,147 @@ +# Resource extraction (tools / datasets / models / benchmarks) + +## Goal + +Given a paper, README, or homepage, extract **one primary research resource** with its metadata, and capture related resources only as **secondary mentions**. + +This is different from NER: +- NER pulls **many spans**; resource extraction pulls **one structured record** per source. +- NER labels are entity types; resource `type` is fixed: `Model`, `Dataset`, `Tool`, `Benchmark`, `Leaderboard`, `Paper`. + +## Output schema + +See `schemas/resource-output.schema.json`. Top level: + +```jsonc +{ + "extracted_resources": { + "1": [{ + "name": "string", // canonical name + "description": "string", // 1–3 sentences, factual + "type": "Model | Dataset | Tool | Benchmark | Leaderboard | Paper", + "category": "string", // free text task category, e.g. "Pose Estimation" + "target": "string", // e.g. "Animal", "Human", "Multimodal" + "specific_target": "string", // e.g. "Quadruped, Horse, Mice" + "url": "string", // canonical URL (homepage / repo) + "mentions": { // OTHER resources referenced — secondary only + "datasets": ["string"], + "benchmarks": ["string"], + "models": ["string"], + "papers": ["string"] + } + }] + } +} +``` + +The numeric key (`"1"`) is so a single batch can hold multiple resources from different sources. For a single document, you'll usually have just `"1"`. + +## "One primary resource" rule + +The single biggest failure mode is the model dumping every mentioned model/dataset as a top-level resource. To prevent it, encode this rule in the extractor prompt: + +> Identify **the single primary resource** the source document is about. Every other tool, dataset, model, benchmark, or paper that is referenced goes under `mentions`, never as a sibling top-level resource. + +Heuristics for "the primary one": + +- For a paper: the artifact in the title and the abstract's "we introduce/propose/release …" sentence. +- For a README: the repo's own name and tagline. +- For a model card: the model the page documents. + +## Type field discipline + +`type` is a **closed vocabulary**. Reject anything outside the set: + +``` +Model | Dataset | Tool | Benchmark | Leaderboard | Paper +``` + +If the model wants to emit `"Library"` or `"Framework"`, normalize to `"Tool"`. If it wants `"Corpus"`, normalize to `"Dataset"`. + +## Alignment for resources (different from NER) + +For resources, alignment **doesn't** add `ontology_id` to the whole resource. Instead it adds nested concept mappings to specific fields: + +```jsonc +{ + "name": "DeepLabCut SuperAnimal-Quadruped", + "type": "Model", + "target": "Animal", + "mapped_target_concept": [ + { "id": "http://purl.obolibrary.org/obo/BTO_0000042", + "label": "animal", "ontology": "BTO" } + ], + "specific_target": "Quadruped, Horse, Mice", + "mapped_specific_target_concept": [ + { "specific_target": "Mice", + "mapped_target_concept": { + "label": "Mus musculus", "id": "NCBITaxon:10090", "ontology": "NCBITaxon" + } + } + ], + "concept_mapping_provenance": "tool" +} +``` + +The alignment stage: +1. Maps `target` → `mapped_target_concept` (usually one concept). +2. Splits `specific_target` on commas and maps each → `mapped_specific_target_concept` (list of `{specific_target, mapped_target_concept}`). +3. Does **not** rewrite `name`, `description`, `type`, `category`, `url`, or `mentions`. + +## Judging resources + +The judge scores **the primary resource only**, not items inside `mentions`. The score reflects: + +- Are `name`, `type`, `url` plausible and consistent with the description? +- Is the resource correctly classified (Model vs Tool vs Dataset)? +- Does the ontology mapping make sense? + +```jsonc +{ + "judge_resource": { + "1": [{ + ...all fields preserved... + "judge_score": 0.92, + "remarks": "Type=Model is correct; mapped_specific_target_concept for 'Mice' is the right NCBITaxon." + }] + } +} +``` + +## What to feed the extractor + +For a multi-page paper, feed: + +1. Title + abstract +2. The "Introduction" paragraphs that name the artifact +3. Any "Implementation" / "Release" / "Availability" section +4. The first table that lists datasets/benchmarks used + +Skip Methods/Experiments details — they're a distraction for resource extraction (they mostly belong under `mentions`). + +If the source is a README, feed the README's first 2–3 sections (intro + install + usage) plus the "Citation"/"Related work" section if present. + +## Worked walkthrough + +For a paper introducing **DeepLabCut SuperAnimal-Quadruped**: + +| Field | Value | +|---|---| +| `name` | "DeepLabCut SuperAnimal-Quadruped" | +| `type` | "Model" | +| `category` | "Pose Estimation" | +| `target` | "Animal" | +| `specific_target` | "Quadruped, Horse, Mice" | +| `url` | "https://deeplabcut.github.io/DeepLabCut/docs/ModelZoo.html" | +| `mentions.datasets` | ["Quadruped-80K", "AnimalPose"] | +| `mentions.benchmarks` | ["AP-10K", "AnimalPose"] | +| `mentions.models` | ["DeepLabCut"] | + +Note that **DeepLabCut** (the parent toolkit) is a mention because the *primary* resource here is the SuperAnimal-Quadruped model, not the toolkit. + +## Edge cases + +- **Multi-resource releases** (e.g. "we release a dataset and an accompanying model"): emit each as a separate top-level resource under `"1"`, `"2"`. Don't merge. +- **Resources without URLs**: emit `"url": null` rather than guessing. Made-up URLs poison downstream consumers. +- **Vague targets** ("various species"): set `specific_target: null`. Don't pad with examples that weren't in the text. +- **Versions** ("DeepLabCut v2.3"): keep the version in `name`; don't add a separate `version` field unless your schema asks for one. diff --git a/skills/structsense-skills/references/structured-extraction.md b/skills/structsense-skills/references/structured-extraction.md new file mode 100644 index 0000000..851f065 --- /dev/null +++ b/skills/structsense-skills/references/structured-extraction.md @@ -0,0 +1,143 @@ +# Structured (schema-driven) extraction + +Use this when the user has a **target JSON schema** (their own, or a known one like ReproSchema, Croissant, schema.org Dataset, JSON-LD, …) and wants a document mapped into it. + +The pattern is the same — extractor → alignment → judge — but the **schema is the contract**, not the entity taxonomy. + +## When to pick this over NER or resource extraction + +| Signal | Use | +|---|---| +| User says "I have a JSON schema" or "convert this PDF to ReproSchema/JSON-LD/Croissant" | **structured extraction** | +| User says "extract entities" / "find all genes" | NER | +| User says "what models/datasets are in this paper" | resource extraction | +| User wants a single deeply-nested record per document | structured extraction | +| User wants many shallow items per document | NER | + +## Step 1: lock the schema before anything else + +The extractor cannot guess the schema. Ask the user for one of: + +1. A JSON Schema (preferred). +2. A worked example output (next best). +3. A bulleted field list with types and required-ness. + +If they give an example, derive a JSON Schema from it before running. Save it under `schemas/<task>.schema.json` so future runs use the same contract. + +## Step 2: write the extractor prompt around the schema + +In the extractor's system prompt: + +1. Put the **full JSON Schema** inline. +2. State: "Output JSON that validates against this schema. Nothing else." +3. For each required field, give a one-line description and one example value. +4. For enums, list them explicitly: "`inputType` must be one of: `radio | text | number | slider`." +5. For nullable fields, say so: "If `citation` is not in the source, set it to `null`." + +See `prompts/extractor-structured.md`. + +## Step 3: validate before returning + +After parsing the LLM's JSON, run a real JSON Schema validator (e.g. `jsonschema` in Python). If it fails: + +1. Capture the validation error path (e.g. `items[3].responseOptions[1].value: missing`). +2. Send the LLM a **repair prompt** with the original JSON + the validator's error, asking it to emit ONLY the fixed JSON. +3. Re-validate. Cap at 2 repair attempts; if still failing, return partial output + `errors[]`. + +## Step 4: alignment for structured extraction + +Often the schema has fields that should be ontology IRIs (e.g. ReproSchema's `responseOption.valueType` ideally maps to xsd or OBO concepts). Strategy: + +- Mark in the schema **which fields are mapping targets** (your task-specific decision). +- The alignment stage walks those paths in the JSON, looks up each value, and replaces it (or adds a sibling) with the mapped IRI. +- Always preserve the original string in `<field>_raw` so nothing is lost. + +```jsonc +{ + "inputType": "radio", + "inputType_raw": "radio", + "inputType_iri": "https://schema.repronim.org/inputTypes/radio" +} +``` + +## Step 5: judging structured extraction + +Two kinds of judging: + +- **Schema-validity score** (deterministic, no LLM): 1.0 if JSON validates; 0 if not. +- **Faithfulness score** (LLM): for each top-level field, "is this value supported by the source text? Score 0–1, give one-line `remarks`." Emit a `field_judgments` object alongside the data. + +```jsonc +{ + "data": { ... }, + "schema_valid": true, + "field_judgments": { + "prefLabel": {"score": 1.0, "remarks": "Exact match in title."}, + "description": {"score": 0.85, "remarks": "Slightly paraphrased."}, + "items[3].responseOptions": {"score": 1.0, "remarks": "All four options matched."} + } +} +``` + +## Common schemas you'll encounter + +| Schema | Use case | +|---|---| +| [ReproSchema](https://www.repronim.org/reproschema/) | Surveys/questionnaires → JSON-LD | +| [Croissant](https://github.com/mlcommons/croissant) | ML dataset cards | +| [schema.org Dataset](https://schema.org/Dataset) | Generic dataset metadata | +| [Bioschemas](https://bioschemas.org/) | Life-science profiles on schema.org | +| [DCAT](https://www.w3.org/TR/vocab-dcat/) | Open-data catalog records | + +If the user names one of these, treat it as the schema and skip Step 1. + +## Worked example: questionnaire PDF → ReproSchema + +**Source:** a 12-page mood questionnaire PDF. + +**Schema-locked fields (extractor must produce):** + +```jsonc +{ + "activity": { + "id": "MFQ-long", + "prefLabel": "Mood and Feelings Questionnaire — Long Version", + "description": "33-item self-report measure of depressive symptoms in children…", + "preamble": "How have you felt in the last two weeks…", + "citation": "Angold et al., 1987" + }, + "items": [ + { + "id": "Q1", + "question": "I felt miserable or unhappy", + "prefLabel": "miserable_unhappy", + "inputType": "radio", + "valueRequired": true, + "responseOptions": [ + { "value": 0, "name": "Not true" }, + { "value": 1, "name": "Sometimes" }, + { "value": 2, "name": "True" } + ], + "scoring": { "type": "sum", "weights": [0, 1, 2] } + } + // … 32 more items + ], + "computedScores": [ + { "id": "total_score", "type": "sum_all", "items": ["Q1", "Q2", "..."] } + ] +} +``` + +**Pipeline:** + +1. **Extractor:** schema-locked prompt with the full JSON Schema inline. +2. **Alignment:** map `inputType` strings → ReproSchema IRIs; map `responseOptions.value` types → xsd types. Preserve original strings as `*_raw`. +3. **Judge:** schema validation + per-field faithfulness check. +4. **Human feedback:** display side-by-side PDF page + extracted JSON; reviewer can edit any field. + +## Hard rules + +- Schema first, prompt second. Don't let the model invent shape. +- Validate every run. If schema validation fails, repair-then-retry with the validator error. +- Preserve raw values. Never lose the source string — always keep it as `<field>_raw`. +- Source-supported only. The judge should penalize any field whose value isn't derivable from the source text. diff --git a/skills/structsense-skills/schemas/aligned-item.schema.json b/skills/structsense-skills/schemas/aligned-item.schema.json new file mode 100644 index 0000000..ace6430 --- /dev/null +++ b/skills/structsense-skills/schemas/aligned-item.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://structsense.skills/aligned-item.schema.json", + "title": "Fragment: ontology alignment fields", + "description": "Fields that the alignment stage ADDS to any extracted item (entity, key_term, or resource). Compose into a larger schema with allOf.", + "type": "object", + "required": ["ontology_id", "ontology_label", "ontology", "concept_mapping_provenance"], + "properties": { + "ontology_id": { + "type": ["string", "null"], + "description": "Ontology IRI (or CURIE) returned by the mapping tool, e.g. 'http://purl.obolibrary.org/obo/UBERON_0002421' or 'NCBITaxon:10090'." + }, + "ontology_label": { + "type": ["string", "null"], + "description": "Preferred label of the matched concept." + }, + "ontology": { + "type": ["string", "null"], + "description": "Ontology shortname, e.g. UBERON, CL, NCBITaxon, MONDO, CHEBI, BTO." + }, + "concept_mapping_provenance": { + "type": "string", + "enum": ["tool", "llm_knowledge", "unmapped", "skipped"], + "description": "Where the mapping came from. 'tool' = concept-mapping tool returned this. 'llm_knowledge' = LLM filled it in. 'unmapped' = no match found. 'skipped' = alignment stage was disabled." + }, + "alignment_method": { + "type": ["string", "null"], + "enum": ["direct_tool_call", "llm_agent", "skipped", null], + "description": "How the alignment stage was run. Useful for debugging unexpected mappings." + } + }, + "additionalProperties": true +} diff --git a/skills/structsense-skills/schemas/judged-item.schema.json b/skills/structsense-skills/schemas/judged-item.schema.json new file mode 100644 index 0000000..1a2298b --- /dev/null +++ b/skills/structsense-skills/schemas/judged-item.schema.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://structsense.skills/judged-item.schema.json", + "title": "Fragment: judge stage fields", + "description": "Fields that the judge stage ADDS to any aligned item. Compose into a larger schema with allOf.", + "type": "object", + "required": ["judge_score", "remarks"], + "properties": { + "judge_score": { + "type": "number", + "minimum": 0, + "maximum": 1, + "description": "Confidence that the extraction + alignment is correct. 1.0 = unambiguous; 0.0 = obviously wrong." + }, + "remarks": { + "type": "string", + "minLength": 1, + "maxLength": 280, + "description": "One short sentence explaining the score." + }, + "judge_method": { + "type": "string", + "enum": ["llm", "auto_approved"], + "description": "How the score was produced. 'auto_approved' = judge stage was skipped." + } + }, + "additionalProperties": true +} diff --git a/skills/structsense-skills/schemas/ner-output.schema.json b/skills/structsense-skills/schemas/ner-output.schema.json new file mode 100644 index 0000000..43f84bf --- /dev/null +++ b/skills/structsense-skills/schemas/ner-output.schema.json @@ -0,0 +1,166 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://structsense.skills/ner-output.schema.json", + "title": "NER extractor output", + "description": "paper_title / doi / source_path live ONCE at the top level under source_metadata. They are NOT repeated on every entity. paper_location is per-entity because it varies (different mentions appear in different sections / pages).", + "type": "object", + "required": ["entities", "key_terms"], + "properties": { + "source_metadata": { + "type": "object", + "description": "Document-level metadata. Populated ONCE per run, not per item.", + "properties": { + "paper_title": { "type": ["string", "null"] }, + "doi": { "type": ["string", "null"] }, + "source_path": { "type": ["string", "null"] } + }, + "additionalProperties": true + }, + "entities": { + "type": "array", + "description": "Raw mentions — one item per occurrence (exhaustive). Each item carries a `source_model` recording which NER model surfaced it (HF model id, or `llm_ner:<llm_model>`).", + "items": { "$ref": "#/$defs/Entity" } + }, + "key_terms": { + "type": "array", + "items": { "$ref": "#/$defs/KeyTerm" } + }, + "entities_grouped": { + "type": "array", + "description": "Per-entity index — mentions collapsed by canonical (entity.lower(), label). Each group lists every contributing source_model, the deduplicated sentences from all locations, and judge_score summaries.", + "items": { "$ref": "#/$defs/EntityGroup" } + }, + "key_terms_grouped": { + "type": "array", + "items": { "$ref": "#/$defs/EntityGroup" } + }, + "ensemble_models": { + "type": "array", + "description": "Per-model summary from the NER ensemble run. Records which models contributed and which were skipped (e.g. missing weights).", + "items": { + "type": "object", + "properties": { + "source_model": { "type": "string" }, + "count": { "type": "integer", "minimum": 0 }, + "skipped_reason": { "type": ["string", "null"] } + } + } + }, + "task_type": { "type": "string", "enum": ["ner"] }, + "elapsed_time": { "type": "number" }, + "errors": { "type": "array", "items": { "type": "string" } }, + "stats": { "type": "object", "additionalProperties": true } + }, + "additionalProperties": true, + "$defs": { + "Entity": { + "type": "object", + "required": ["entity", "label", "sentence", "start", "end"], + "properties": { + "entity": { "type": "string", "minLength": 1 }, + "label": { + "type": "string", + "description": "Entity type. Default taxonomy: Gene, Protein, Species, CellType, BrainRegion, Disease, Method, Chemical, Measurement, Other. Override per task." + }, + "sentence": { "type": "string", "minLength": 1 }, + "start": { "type": "integer", "minimum": 0 }, + "end": { "type": "integer", "minimum": 0 }, + "paper_location": { "type": ["string", "null"] }, + + "source_model": { + "type": ["string", "null"], + "description": "Which model surfaced this mention. HF model id (e.g. 'd4data/biomedical-ner-all'), or 'llm_ner:<llm_model>'." + }, + "source_score": { + "type": ["number", "null"], + "minimum": 0, "maximum": 1, + "description": "The source model's own confidence (0–1) if it emits one (HF NER models do; LLM extractors usually don't)." + }, + + "ontology_id": { "type": ["string", "null"] }, + "ontology_label": { "type": ["string", "null"] }, + "ontology": { "type": ["string", "null"] }, + "concept_mapping_provenance": { + "type": ["string", "null"], + "enum": ["tool", "llm_knowledge", "unmapped", "skipped", null] + }, + "alignment_method": { + "type": ["string", "null"], + "enum": ["direct_tool_call", "llm_agent", "skipped", null] + }, + + "judge_score": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }, + "remarks": { "type": ["string", "null"] }, + "judge_method": { + "type": ["string", "null"], + "enum": ["llm", "auto_approved", null] + } + }, + "additionalProperties": true + }, + "KeyTerm": { + "type": "object", + "required": ["term", "sentence", "start", "end"], + "properties": { + "term": { "type": "string", "minLength": 1 }, + "sentence": { "type": "string", "minLength": 1 }, + "start": { "type": "integer", "minimum": 0 }, + "end": { "type": "integer", "minimum": 0 }, + "paper_location": { "type": ["string", "null"] }, + "source_model": { "type": ["string", "null"] }, + "source_score": { "type": ["number", "null"] }, + "ontology_id": { "type": ["string", "null"] }, + "ontology_label": { "type": ["string", "null"] }, + "ontology": { "type": ["string", "null"] }, + "concept_mapping_provenance": { "type": ["string", "null"] }, + "judge_score": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }, + "remarks": { "type": ["string", "null"] } + }, + "additionalProperties": true + }, + "EntityGroup": { + "type": "object", + "description": "An entity index entry: one canonical (entity, label) with all mentions collapsed.", + "required": ["mention_count"], + "properties": { + "entity": { "type": ["string", "null"] }, + "term": { "type": ["string", "null"] }, + "label": { "type": ["string", "null"] }, + "mention_count": { "type": "integer", "minimum": 1 }, + "source_models": { "type": "array", "items": { "type": "string" } }, + "source_model_counts": { + "type": "object", + "additionalProperties": { "type": "integer", "minimum": 1 } + }, + "consensus_count": { + "type": "integer", "minimum": 1, + "description": "Distinct source_models that surfaced this entity. Useful as a confidence proxy." + }, + "ontology_id": { "type": ["string", "null"] }, + "ontology_label": { "type": ["string", "null"] }, + "ontology": { "type": ["string", "null"] }, + "concept_mapping_provenance": { "type": ["string", "null"] }, + "alignment_method": { "type": ["string", "null"] }, + "judge_score_max": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }, + "judge_score_avg": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }, + "judge_score_min": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }, + "sentences": { + "type": "array", + "items": { + "type": "object", + "properties": { + "text": { "type": "string" }, + "paper_locations": { "type": "array", "items": { "type": "string" } } + } + } + }, + "mentions": { + "type": "array", + "description": "Slim per-occurrence views (offsets, sentence, source_model, judge_score).", + "items": { "type": "object", "additionalProperties": true } + } + }, + "additionalProperties": true + } + } +} diff --git a/skills/structsense-skills/schemas/resource-output.schema.json b/skills/structsense-skills/schemas/resource-output.schema.json new file mode 100644 index 0000000..b7392c4 --- /dev/null +++ b/skills/structsense-skills/schemas/resource-output.schema.json @@ -0,0 +1,95 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://structsense.skills/resource-output.schema.json", + "title": "Research resource extraction output", + "type": "object", + "required": ["extracted_resources"], + "properties": { + "extracted_resources": { + "type": "object", + "patternProperties": { + "^[1-9][0-9]*$": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/Resource" } + } + }, + "additionalProperties": false + }, + "task_type": { "type": "string", "enum": ["resource"] }, + "elapsed_time": { "type": "number" }, + "errors": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": true, + "$defs": { + "Resource": { + "type": "object", + "required": ["name", "description", "type"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "description": { "type": "string", "minLength": 1, "maxLength": 1000 }, + "type": { + "type": "string", + "enum": ["Model", "Dataset", "Tool", "Benchmark", "Leaderboard", "Paper"] + }, + "category": { "type": ["string", "null"] }, + "target": { "type": ["string", "null"] }, + "specific_target": { "type": ["string", "null"] }, + "url": { "type": ["string", "null"], "format": "uri" }, + "mentions": { + "type": "object", + "properties": { + "datasets": { "type": "array", "items": { "type": "string" } }, + "benchmarks": { "type": "array", "items": { "type": "string" } }, + "models": { "type": "array", "items": { "type": "string" } }, + "papers": { "type": "array", "items": { "type": "string" } }, + "tools": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": false + }, + + "mapped_target_concept": { + "type": "array", + "items": { "$ref": "#/$defs/Concept" } + }, + "mapped_specific_target_concept": { + "type": "array", + "items": { + "type": "object", + "required": ["specific_target", "mapped_target_concept"], + "properties": { + "specific_target": { "type": "string" }, + "mapped_target_concept": { "$ref": "#/$defs/Concept" } + } + } + }, + "concept_mapping_provenance": { + "type": ["string", "null"], + "enum": ["tool", "llm_knowledge", "unmapped", "skipped", null] + }, + "alignment_method": { + "type": ["string", "null"], + "enum": ["direct_tool_call", "llm_agent", "skipped", null] + }, + + "judge_score": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }, + "remarks": { "type": ["string", "null"] }, + "judge_method": { + "type": ["string", "null"], + "enum": ["llm", "auto_approved", null] + } + }, + "additionalProperties": true + }, + "Concept": { + "type": "object", + "required": ["id", "label", "ontology"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "label": { "type": "string", "minLength": 1 }, + "ontology": { "type": "string", "minLength": 1 } + }, + "additionalProperties": false + } + } +} diff --git a/skills/structsense-skills/scripts/bioportal_map.py b/skills/structsense-skills/scripts/bioportal_map.py new file mode 100644 index 0000000..cdb8169 --- /dev/null +++ b/skills/structsense-skills/scripts/bioportal_map.py @@ -0,0 +1,165 @@ +"""BioPortal concept mapping client with rate limiting + LRU caching. + +Maps free-text terms to ontology IRIs via the BioPortal REST API. + +Usage: + from bioportal_map import BioPortalMapper + + mapper = BioPortalMapper(api_key=os.environ["BIOPORTAL_API_KEY"]) + result = mapper.map_one("hippocampus", ontologies=["UBERON"]) + # -> {"ontology_id": "...", "ontology_label": "...", "ontology": "UBERON", + # "concept_mapping_provenance": "tool"} + + batch = mapper.map_batch(["hippocampus", "mouse"], ontologies=["UBERON","NCBITAXON"]) + +Get a free key at https://bioportal.bioontology.org/account. +Adapted from structsense conceptmappingtool.py. +""" +from __future__ import annotations + +import os +import time +import threading +import logging +from functools import lru_cache +from typing import Iterable, Optional + +import requests + +logger = logging.getLogger("BioPortalMapper") + +BIOPORTAL_SEARCH = "https://data.bioontology.org/search" +MAX_QUERY_LENGTH = 500 + + +class BioPortalMapper: + """Throttled BioPortal client. + + Constructor args: + api_key: BioPortal API key (or set BIOPORTAL_API_KEY env var). + request_interval: seconds between requests. Default 0.7 (env override: + BIOPORTAL_REQUEST_INTERVAL). + backoff_after_429: initial backoff seconds after a 429. Default 2.0. + cache_size: LRU cache entries for repeat terms. Default 2000. + """ + + def __init__( + self, + api_key: Optional[str] = None, + request_interval: Optional[float] = None, + backoff_after_429: Optional[float] = None, + cache_size: int = 2000, + ): + self.api_key = api_key or os.environ.get("BIOPORTAL_API_KEY") + if not self.api_key: + raise ValueError("BIOPORTAL_API_KEY is required") + self.request_interval = float( + request_interval + if request_interval is not None + else os.environ.get("BIOPORTAL_REQUEST_INTERVAL", 0.7) + ) + self.backoff = float( + backoff_after_429 + if backoff_after_429 is not None + else os.environ.get("BIOPORTAL_BACKOFF_AFTER_429", 2.0) + ) + self._lock = threading.Lock() + self._last_request_time = 0.0 + # Wrap _search_uncached in an LRU cache sized by user. + self._search = lru_cache(maxsize=cache_size)(self._search_uncached) + + # ------------------------------------------------------------------ + # Throttling + # ------------------------------------------------------------------ + def _throttle(self) -> None: + with self._lock: + elapsed = time.monotonic() - self._last_request_time + wait = self.request_interval - elapsed + if wait > 0: + time.sleep(wait) + self._last_request_time = time.monotonic() + + # ------------------------------------------------------------------ + # Search (cached) + # ------------------------------------------------------------------ + def _search_uncached(self, term: str, ontologies: tuple, max_results: int): + params = { + "q": term[:MAX_QUERY_LENGTH], + "apikey": self.api_key, + "display_context": "false", + "include": "prefLabel,definition", + "pagesize": max_results, + } + if ontologies: + params["ontologies"] = ",".join(ontologies) + for attempt in range(4): + self._throttle() + try: + resp = requests.get(BIOPORTAL_SEARCH, params=params, timeout=15) + except requests.RequestException as e: + logger.warning("BioPortal request failed (%s); retrying", e) + time.sleep(self.backoff * (2 ** attempt)) + continue + if resp.status_code == 429: + logger.warning("BioPortal 429; backing off") + time.sleep(self.backoff * (2 ** attempt)) + continue + if resp.status_code >= 500: + logger.warning("BioPortal %s; retrying", resp.status_code) + time.sleep(self.backoff * (2 ** attempt)) + continue + resp.raise_for_status() + return resp.json().get("collection", []) or [] + return [] + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + def map_one(self, term: str, ontologies: Optional[Iterable[str]] = None, + max_results: int = 1) -> dict: + """Map a single term. Returns the canonical mapping dict (with + provenance) — `unmapped` when no results. + """ + ont_tuple = tuple(ontologies or ()) + hits = self._search(term, ont_tuple, max_results) + if not hits: + return { + "term": term, + "ontology_id": None, + "ontology_label": None, + "ontology": None, + "concept_mapping_provenance": "unmapped", + } + top = hits[0] + return { + "term": term, + "ontology_id": top.get("@id"), + "ontology_label": top.get("prefLabel"), + "ontology": self._ontology_shortname(top), + "concept_mapping_provenance": "tool", + } + + def map_batch(self, terms: Iterable[str], + ontologies: Optional[Iterable[str]] = None, + max_results: int = 1) -> list[dict]: + """Map a list of terms. Returns a parallel list of mapping dicts.""" + return [self.map_one(t, ontologies, max_results) for t in terms] + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + @staticmethod + def _ontology_shortname(hit: dict) -> Optional[str]: + # BioPortal nests the source ontology under "links.ontology" as a URL. + link = (hit.get("links") or {}).get("ontology") + if isinstance(link, str): + return link.rstrip("/").rsplit("/", 1)[-1] + return None + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + m = BioPortalMapper() + for r in m.map_batch(["hippocampus", "mouse", "BDNF"], + ontologies=["UBERON", "NCBITAXON", "HGNC"]): + print(r) diff --git a/skills/structsense-skills/scripts/chunking.py b/skills/structsense-skills/scripts/chunking.py new file mode 100644 index 0000000..8228724 --- /dev/null +++ b/skills/structsense-skills/scripts/chunking.py @@ -0,0 +1,136 @@ +"""Sentence-aligned chunking for long-document extraction. + +Splits text on sentence boundaries, accumulating sentences until the chunk +would exceed `max_chars`. Returns chunks with their character offset in the +original text so per-chunk extraction results can be re-anchored to global +offsets after merging. + +Two backends: +- spaCy (preferred — better sentence segmentation on scientific text) +- regex fallback (no extra dependency) + +Adapted from the structsense pipeline pattern. +""" +from __future__ import annotations + +import re +from typing import Iterable, List, Tuple, Dict, Any + +# --------------------------------------------------------------------------- +# Sentence segmentation +# --------------------------------------------------------------------------- + +# Very basic English sentence splitter. Good enough for chunking; do not use +# for linguistic analysis. +_SENT_RE = re.compile(r"(?<=[.!?])\s+(?=[A-Z(\"'\[])") + +def _sentences_regex(text: str) -> List[Tuple[int, int, str]]: + """Return [(start, end, sentence_text)] from the start of `text`.""" + spans: List[Tuple[int, int, str]] = [] + cursor = 0 + for match in _SENT_RE.finditer(text): + end = match.start() + if end > cursor: + spans.append((cursor, end, text[cursor:end])) + cursor = match.end() + if cursor < len(text): + spans.append((cursor, len(text), text[cursor:])) + return spans + + +def _sentences_spacy(text: str, nlp) -> List[Tuple[int, int, str]]: + doc = nlp(text) + return [(s.start_char, s.end_char, s.text) for s in doc.sents] + + +def sentence_spans(text: str, nlp=None) -> List[Tuple[int, int, str]]: + """Split text into sentence spans. Pass an initialized spaCy `nlp` to use + spaCy; otherwise falls back to a regex splitter. + """ + if nlp is not None: + return _sentences_spacy(text, nlp) + return _sentences_regex(text) + + +# --------------------------------------------------------------------------- +# Chunking +# --------------------------------------------------------------------------- + +def chunk_by_sentences(text: str, max_chars: int = 2000, nlp=None) -> List[Dict[str, Any]]: + """Greedy sentence-aligned chunking. + + Args: + text: source text. + max_chars: maximum characters per chunk. + nlp: optional spaCy `Language` object for sentence splitting. + + Returns: + List of {"text": str, "start": int}. `start` is the chunk's character + offset in the original `text`. The concatenation of all chunk texts + (with inter-sentence whitespace) reconstructs `text`. + """ + if max_chars <= 0 or len(text) <= max_chars: + return [{"text": text, "start": 0}] + + spans = sentence_spans(text, nlp=nlp) + if not spans: + return [{"text": text, "start": 0}] + + chunks: List[Dict[str, Any]] = [] + cur_start: int | None = None + cur_end: int | None = None + + for s_start, s_end, _ in spans: + if cur_start is None: + cur_start, cur_end = s_start, s_end + continue + if s_end - cur_start <= max_chars: + cur_end = s_end + else: + chunks.append({"text": text[cur_start:cur_end], "start": cur_start}) + cur_start, cur_end = s_start, s_end + + if cur_start is not None: + chunks.append({"text": text[cur_start:cur_end], "start": cur_start}) + + return chunks + + +def reanchor_items(items: Iterable[dict], chunk_start: int, + start_key: str = "start", end_key: str = "end") -> List[dict]: + """Add `chunk_start` to every item's start/end so they reference the + original full text instead of the chunk. + """ + out = [] + for it in items: + it = dict(it) + if start_key in it: + it[start_key] = it[start_key] + chunk_start + if end_key in it: + it[end_key] = it[end_key] + chunk_start + out.append(it) + return out + + +def dedupe(items: Iterable[dict], key_fields=("entity", "start", "end")) -> List[dict]: + """Deduplicate by tuple of field values. Preserves first occurrence.""" + seen = set() + out = [] + for it in items: + k = tuple(it.get(f) for f in key_fields) + if k in seen: + continue + seen.add(k) + out.append(it) + return out + + +if __name__ == "__main__": + sample = ( + "The hippocampus is a brain region. Studies in mice show its role in memory. " + "BDNF is upregulated. CA1 pyramidal cells fire in response to novelty." + ) * 50 + chunks = chunk_by_sentences(sample, max_chars=400) + print(f"{len(chunks)} chunks") + for i, c in enumerate(chunks): + print(f"[{i}] start={c['start']} chars={len(c['text'])}: {c['text'][:60]}...") diff --git a/skills/structsense-skills/scripts/group_by_entity.py b/skills/structsense-skills/scripts/group_by_entity.py new file mode 100644 index 0000000..99624fe --- /dev/null +++ b/skills/structsense-skills/scripts/group_by_entity.py @@ -0,0 +1,302 @@ +"""Group raw NER mentions into a per-entity index. + +The extractor emits one item per occurrence (exhaustive). For downstream +consumers ("show me everything about BDNF in this paper") it's often +cleaner to also have a view where mentions are grouped by canonical entity, +with all containing sentences and paper locations merged. + +This module is pure post-processing — no LLM calls. It runs after the judge +stage and writes a sibling `entities_grouped` list to the pipeline result. + +Output shape per group: + + { + "entity": "BDNF", # canonical surface form + "label": "Gene", + "mention_count": 17, + "ontology_id": "http://identifiers.org/hgnc/1033", + "ontology_label": "BDNF", + "ontology": "HGNC", + "concept_mapping_provenance": "tool", + "alignment_method": "direct_tool_call", + "judge_score_max": 1.0, + "judge_score_avg": 0.92, + "judge_score_min": 0.7, + "mentions": [ + {"start": 12, "end": 16, "sentence": "...", "paper_location": "Methods"}, + {"start": 348, "end": 352, "sentence": "...", "paper_location": "Results"}, + ... + ], + "sentences": [ + {"text": "...", "paper_locations": ["Methods", "Discussion"]}, + {"text": "...", "paper_locations": ["Results"]}, + ... + ] + } +""" +from __future__ import annotations + +from collections import defaultdict +from statistics import mean +from typing import Any, Iterable, Optional + + +def _canonical_key(entity: str, label: Optional[str]) -> tuple[str, str]: + """Group key: case-insensitive entity + label. + + Case-sensitive variants (BDNF vs bdnf) collapse into one group; the + canonical surface form is picked as the most common casing seen. + Different labels (e.g. 'Pvalb' as Gene vs as LineageMarker) intentionally + do NOT collapse — they're semantically different. + """ + return ((entity or "").strip().lower(), (label or "").strip()) + + +def _pick_canonical_surface(surfaces: list[str]) -> str: + """Return the most common surface form (preserving original casing).""" + counts: dict[str, int] = defaultdict(int) + for s in surfaces: + counts[s] += 1 + return max(counts.items(), key=lambda kv: (kv[1], len(kv[0])))[0] + + +def _safe(v): + return None if v is None else v + + +def _summarize_judge(scores: list[float]) -> dict[str, Optional[float]]: + finite = [s for s in scores if isinstance(s, (int, float))] + if not finite: + return {"judge_score_max": None, "judge_score_avg": None, + "judge_score_min": None} + return { + "judge_score_max": round(max(finite), 3), + "judge_score_avg": round(mean(finite), 3), + "judge_score_min": round(min(finite), 3), + } + + +def _pick_best_alignment(mentions: list[dict]) -> dict: + """Of all mentions for this entity, return the one with the highest + judge_score AND a non-null ontology_id. Falls back to any non-null + mapping, then to the first mention. + """ + candidates = [m for m in mentions if m.get("ontology_id")] + if candidates: + return max(candidates, key=lambda m: m.get("judge_score") or 0.0) + return mentions[0] + + +def group_mentions_by_entity( + items: Iterable[dict], + *, + surface_key: str = "entity", + label_key: str = "label", +) -> list[dict]: + """Group raw mention items by canonical (entity.lower(), label). + + Returns a list of grouped dicts (see module docstring for shape). + """ + buckets: dict[tuple[str, str], list[dict]] = defaultdict(list) + for it in items: + if not it.get(surface_key): + continue + k = _canonical_key(it[surface_key], it.get(label_key)) + buckets[k].append(it) + + out: list[dict] = [] + for (lower_form, label), mentions in buckets.items(): + canonical = _pick_canonical_surface([m[surface_key] for m in mentions]) + best = _pick_best_alignment(mentions) + + # merge sentences by exact text; aggregate the locations they appear in. + sent_to_locs: dict[str, list[str]] = defaultdict(list) + for m in mentions: + sent = m.get("sentence") + loc = m.get("paper_location") + if not sent: + continue + if loc and loc not in sent_to_locs[sent]: + sent_to_locs[sent].append(loc) + elif not loc and not sent_to_locs[sent]: + sent_to_locs[sent] = [] + sentences_merged = [ + {"text": s, "paper_locations": locs} + for s, locs in sent_to_locs.items() + ] + + mentions_view = [ + { + "start": m.get("start"), + "end": m.get("end"), + "sentence": m.get("sentence"), + "paper_location": m.get("paper_location"), + "source_model": m.get("source_model"), + "source_score": m.get("source_score"), + "judge_score": m.get("judge_score"), + "concept_mapping_provenance": m.get("concept_mapping_provenance"), + "alignment_method": m.get("alignment_method"), + } + for m in mentions + ] + + # Which models surfaced this entity (preserves provenance across the + # whole ensemble + LLM). consensus_count = how many distinct models + # found it; useful as a confidence proxy. + source_models = sorted({m.get("source_model") for m in mentions + if m.get("source_model")}) + source_counts: dict[str, int] = defaultdict(int) + for m in mentions: + if m.get("source_model"): + source_counts[m["source_model"]] += 1 + + out.append({ + surface_key: canonical, + label_key: label or None, + "mention_count": len(mentions), + + "source_models": source_models, + "source_model_counts": dict(source_counts), + "consensus_count": len(source_models), + + "ontology_id": _safe(best.get("ontology_id")), + "ontology_label": _safe(best.get("ontology_label")), + "ontology": _safe(best.get("ontology")), + "concept_mapping_provenance": _safe(best.get("concept_mapping_provenance")), + "alignment_method": _safe(best.get("alignment_method")), + + **_summarize_judge([m.get("judge_score") for m in mentions]), + + "sentences": sentences_merged, + "mentions": mentions_view, + }) + + # sort: most-mentioned first; ties broken by entity name. + out.sort(key=lambda g: (-g["mention_count"], g.get(surface_key) or "")) + return out + + +# --------------------------------------------------------------------------- +# Unify ontology across all mentions of the same entity +# --------------------------------------------------------------------------- + +_ONTOLOGY_FIELDS = ( + "ontology_id", "ontology_label", "ontology", "concept_mapping_provenance", +) + + +def _ontology_score(ent: dict) -> int: + """Score an entity's ontology mapping quality. Higher = better. + + Tool-backed mapping (provenance == 'tool') beats LLM knowledge beats + nothing. A real IRI beats placeholders like 'N/A' / 'none'. + """ + s = 0 + if ent.get("concept_mapping_provenance") == "tool": + s += 100 + elif ent.get("concept_mapping_provenance") == "llm_knowledge": + s += 25 + oid = str(ent.get("ontology_id") or "").strip().lower() + if oid and oid not in ("n/a", "none", "null", ""): + s += 50 + if str(ent.get("ontology_label") or "").strip().lower() not in ("n/a", "none", "null", ""): + s += 10 + return s + + +def unify_ontology_across_entities(entities: list[dict], + surface_key: str = "entity", + ) -> list[dict]: + """Make every occurrence of the same (entity, label) share ONE ontology mapping. + + When the same surface form is processed in different parallel chunks (or + by different NER models in the ensemble), each may produce a different + ontology ID. This pass picks the *best* mapping per (entity, label) and + applies it back to every occurrence. + + The "best" is determined by `_ontology_score`: + tool-mapped > llm_knowledge-mapped > unmapped + with a real IRI > with placeholders. + + Individual mentions are preserved — nothing is dropped or deduplicated. + Only the four `_ONTOLOGY_FIELDS` are normalized. + + Mutates the list in place and returns it. + """ + best: dict[tuple, dict] = {} + + for ent in entities: + if not isinstance(ent, dict): + continue + surface = (ent.get(surface_key) or ent.get("term") + or ent.get("name") or "") + key = (str(surface).lower().strip(), + str(ent.get("label") or "").lower().strip()) + if not key[0]: + continue + s = _ontology_score(ent) + if key not in best or s > _ontology_score(best[key]): + best[key] = {f: ent[f] for f in _ONTOLOGY_FIELDS if f in ent} + + for ent in entities: + if not isinstance(ent, dict): + continue + surface = (ent.get(surface_key) or ent.get("term") + or ent.get("name") or "") + key = (str(surface).lower().strip(), + str(ent.get("label") or "").lower().strip()) + if key in best: + ent.update(best[key]) + + return entities + + +def attach_grouped_views(result: dict, *, unify_ontology: bool = True) -> dict: + """Mutate ``result`` to add `entities_grouped` and `key_terms_grouped`. + + The original `entities` / `key_terms` lists (raw, one-per-mention) are + preserved as the authoritative record. When ``unify_ontology=True`` (the + default), we first normalize ontology fields across mentions of the same + (entity, label) so all occurrences share one consistent mapping (best one + wins — tool-mapped beats LLM, real IRI beats placeholders). + """ + if "entities" in result: + if unify_ontology: + unify_ontology_across_entities(result["entities"] or [], + surface_key="entity") + result["entities_grouped"] = group_mentions_by_entity( + result.get("entities") or [], + surface_key="entity", label_key="label", + ) + if "key_terms" in result: + if unify_ontology: + unify_ontology_across_entities(result["key_terms"] or [], + surface_key="term") + result["key_terms_grouped"] = group_mentions_by_entity( + result.get("key_terms") or [], + surface_key="term", label_key="label", + ) + return result + + +if __name__ == "__main__": + import json + sample = [ + {"entity": "BDNF", "label": "Gene", "sentence": "BDNF is upregulated.", + "start": 0, "end": 4, "paper_location": "Results", "judge_score": 1.0, + "ontology_id": "HGNC:1033", "ontology_label": "BDNF", "ontology": "HGNC", + "concept_mapping_provenance": "tool"}, + {"entity": "BDNF", "label": "Gene", "sentence": "BDNF protein levels increase.", + "start": 52, "end": 56, "paper_location": "Discussion", "judge_score": 0.95, + "ontology_id": "HGNC:1033", "ontology_label": "BDNF", "ontology": "HGNC", + "concept_mapping_provenance": "tool"}, + {"entity": "bdnf", "label": "Gene", "sentence": "BDNF is upregulated.", + "start": 200, "end": 204, "paper_location": "Methods", "judge_score": 0.85, + "concept_mapping_provenance": "tool"}, + {"entity": "hippocampus", "label": "BrainRegion", + "sentence": "We recorded in the hippocampus.", + "start": 25, "end": 36, "paper_location": "Methods", "judge_score": 0.9, + "ontology_id": "UBERON_0002421", "ontology_label": "hippocampal formation", + "ontology": "UBERON", "concept_mapping_provenance": "tool"}, + ] + print(json.dumps(group_mentions_by_entity(sample), indent=2)) diff --git a/skills/structsense-skills/scripts/input_loader.py b/skills/structsense-skills/scripts/input_loader.py new file mode 100644 index 0000000..6898699 --- /dev/null +++ b/skills/structsense-skills/scripts/input_loader.py @@ -0,0 +1,291 @@ +"""Read PDF / CSV / TXT input and return plain text with reproducible offsets. + +This is the most important "missing" piece for users running NER on papers: +the skill assumes plain-text input, but real users hand it PDFs. This module +extracts text deterministically, with multiple fallbacks so it works whether +or not the user has GROBID, PyMuPDF, or pdfminer installed. + +Backend selection order: + +1. **GROBID** (if reachable) — best for academic PDFs. Preserves section + structure (Introduction, Methods, Results, …). Pointed at by the + `GROBID_SERVER_URL_OR_EXTERNAL_SERVICE` env var or `--grobid-url` flag. +2. **PyMuPDF** (`fitz`) — fast, accurate raw-text extractor. Works on any + PDF type (papers, questionnaires, reports, forms). Falls back to it when + GROBID is unreachable or returns no sections. +3. **pdfminer.six** — pure-Python, slower, no system deps. Last-resort + fallback when neither GROBID nor PyMuPDF is available. + +CSV/TXT just read the bytes (CSV is round-tripped through pandas if available, +otherwise read verbatim). + +For NER pipelines: **always write the extracted text to disk before +extraction**, so character offsets in the result file refer to a stable text. +Re-extracting the PDF later might yield different offsets if a library +version changes. + +Adapted from structsense `utils.process_file`. +""" +from __future__ import annotations + +import logging +import os +from pathlib import Path +from typing import Optional, Tuple, Union + +logger = logging.getLogger("input_loader") + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + +def process_file( + source_path: Union[str, Path], + *, + grobid_url: Optional[str] = None, + prefer_grobid: bool = True, +) -> str: + """Read a file and return its content as plain text. + + Supported extensions: `.pdf`, `.csv`, `.txt`, `.md`. + + Args: + source_path: file to read. + grobid_url: GROBID server URL. Defaults to the + `GROBID_SERVER_URL_OR_EXTERNAL_SERVICE` env var, or + `http://localhost:8070`. + prefer_grobid: if True (default), try GROBID first for PDFs. + Set False to skip GROBID and go straight to PyMuPDF/pdfminer + (useful for non-academic PDFs). + + Raises: + FileNotFoundError: if `source_path` doesn't exist. + ValueError: if the extension is unsupported, or all extractors fail. + """ + p = Path(source_path) + if not p.is_file(): + raise FileNotFoundError(f"file not found: {p}") + + ext = p.suffix.lower() + if ext == ".pdf": + return _read_pdf(p, grobid_url=grobid_url, prefer_grobid=prefer_grobid) + if ext == ".csv": + return _read_csv(p) + if ext in (".txt", ".md"): + return p.read_text(encoding="utf-8", errors="replace") + + raise ValueError(f"unsupported extension {ext!r} (supported: .pdf, .csv, .txt, .md)") + + +def process_file_to_text_file( + source_path: Union[str, Path], + out_path: Optional[Union[str, Path]] = None, + **kwargs, +) -> Tuple[Path, str]: + """Extract text and write it to `<source_stem>.txt` for stable offsets. + + Returns ``(out_path, text)``. Useful as the first step of an NER pipeline + so every subsequent stage references the same canonical text file. + """ + src = Path(source_path) + text = process_file(src, **kwargs) + out = Path(out_path) if out_path else src.with_suffix(".txt") + out.write_text(text, encoding="utf-8") + logger.info("wrote %d chars to %s", len(text), out) + return out, text + + +# --------------------------------------------------------------------------- +# PDF backends +# --------------------------------------------------------------------------- + +def _read_pdf(path: Path, *, grobid_url: Optional[str], + prefer_grobid: bool) -> str: + errors: list[str] = [] + + if prefer_grobid: + text = _try_grobid(path, grobid_url=grobid_url, errors=errors) + if text and text.strip(): + return text + + text = _try_pymupdf(path, errors=errors) + if text and text.strip(): + return text + + text = _try_pdfminer(path, errors=errors) + if text and text.strip(): + return text + + raise ValueError( + f"all PDF extractors failed for {path.name}: " + " | ".join(errors) + ) + + +def _try_grobid(path: Path, *, grobid_url: Optional[str], + errors: list[str]) -> Optional[str]: + url = (grobid_url + or os.environ.get("GROBID_SERVER_URL_OR_EXTERNAL_SERVICE") + or "http://localhost:8070") + try: + import requests + except ImportError as e: + errors.append(f"grobid: requests not installed ({e})") + return None + # Quick probe — if not reachable, fall back fast. + try: + r = requests.get(f"{url.rstrip('/')}/api/isalive", timeout=2) + if not r.ok: + errors.append(f"grobid {url}: /isalive returned {r.status_code}") + return None + except requests.RequestException as e: + errors.append(f"grobid {url}: unreachable ({e})") + return None + + # Submit the PDF + try: + with open(path, "rb") as fh: + files = {"input": (path.name, fh, "application/pdf")} + r = requests.post( + f"{url.rstrip('/')}/api/processFulltextDocument", + files=files, timeout=120, + ) + if not r.ok: + errors.append(f"grobid: HTTP {r.status_code}") + return None + xml_text = r.text + except requests.RequestException as e: + errors.append(f"grobid: request failed ({e})") + return None + + # Parse the TEI XML to plain text (preserving section headings + paragraphs) + return _tei_to_text(xml_text, errors=errors) + + +def _tei_to_text(xml_text: str, *, errors: list[str]) -> Optional[str]: + """Convert TEI XML from GROBID into plain text with section headings.""" + try: + from xml.etree import ElementTree as ET + except ImportError as e: + errors.append(f"tei parser: {e}") + return None + + try: + ns = {"tei": "http://www.tei-c.org/ns/1.0"} + root = ET.fromstring(xml_text) + except ET.ParseError as e: + errors.append(f"grobid TEI parse error: {e}") + return None + + parts: list[str] = [] + + # Title + for t in root.iter("{http://www.tei-c.org/ns/1.0}title"): + if t.text and t.text.strip(): + parts.append(t.text.strip()) + break + + # Abstract + for ab in root.iter("{http://www.tei-c.org/ns/1.0}abstract"): + ab_text = " ".join((p.text or "") for p in ab.iter("{http://www.tei-c.org/ns/1.0}p") if (p.text or "").strip()) + if ab_text.strip(): + parts.append("Abstract\n" + ab_text.strip()) + break + + # Body sections + for div in root.iter("{http://www.tei-c.org/ns/1.0}div"): + head = div.find("tei:head", ns) + head_text = (head.text.strip() if head is not None and head.text else "") + paras = [] + for p in div.findall("tei:p", ns): + if p.text and p.text.strip(): + paras.append(p.text.strip()) + if head_text and paras: + parts.append(f"{head_text}\n" + "\n".join(paras)) + elif paras: + parts.append("\n".join(paras)) + + if not parts: + return None + + return "\n\n".join(parts).strip() + + +def _try_pymupdf(path: Path, *, errors: list[str]) -> Optional[str]: + try: + import fitz # PyMuPDF + except ImportError as e: + errors.append(f"pymupdf: not installed ({e}); pip install pymupdf") + return None + try: + doc = fitz.open(str(path)) + except Exception as e: + errors.append(f"pymupdf: open failed ({e})") + return None + try: + pages = [] + for page in doc: + t = page.get_text() + if t and t.strip(): + pages.append(t) + return "\n\n".join(pages).strip() + finally: + doc.close() + + +def _try_pdfminer(path: Path, *, errors: list[str]) -> Optional[str]: + try: + from pdfminer.high_level import extract_text + except ImportError as e: + errors.append(f"pdfminer: not installed ({e}); pip install pdfminer.six") + return None + try: + return extract_text(str(path)) + except Exception as e: + errors.append(f"pdfminer: extract failed ({e})") + return None + + +# --------------------------------------------------------------------------- +# CSV +# --------------------------------------------------------------------------- + +def _read_csv(path: Path) -> str: + """Round-trip the CSV through pandas (if available) for clean re-serialization. + + Falls back to raw file read if pandas isn't installed. + """ + try: + import pandas as pd + return pd.read_csv(path).to_csv(index=False) + except ImportError: + return path.read_text(encoding="utf-8", errors="replace") + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + import argparse, sys + ap = argparse.ArgumentParser( + description="Extract text from a PDF / CSV / TXT and write a " + "<stem>.txt next to the source so subsequent NER stages " + "have stable character offsets." + ) + ap.add_argument("source", help="path to .pdf / .csv / .txt / .md") + ap.add_argument("--out", default=None, help="output text path (default: <stem>.txt)") + ap.add_argument("--grobid-url", default=None, + help="GROBID server URL (default: $GROBID_SERVER_URL_OR_EXTERNAL_SERVICE or http://localhost:8070)") + ap.add_argument("--no-grobid", action="store_true", + help="Skip GROBID; use PyMuPDF / pdfminer only.") + args = ap.parse_args() + + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s") + out_path, text = process_file_to_text_file( + args.source, + out_path=args.out, + grobid_url=args.grobid_url, + prefer_grobid=not args.no_grobid, + ) + print(f"{out_path}\t{len(text)} chars", file=sys.stderr) diff --git a/skills/structsense-skills/scripts/iri_validation.py b/skills/structsense-skills/scripts/iri_validation.py new file mode 100644 index 0000000..d047c1a --- /dev/null +++ b/skills/structsense-skills/scripts/iri_validation.py @@ -0,0 +1,326 @@ +"""Strict IRI validation for ontology mappings. + +Purpose: prevent LLM-hallucinated IRIs from entering the canonical output. +The skill's policy is **no LLM-knowledge mappings, ever** — every mapping +must come from a tool call (local hybrid or BioPortal) and pass these checks: + +1. Structural sanity — the IRI matches a known pattern for its ontology. +2. Optional resolvability — HEAD/GET returns 2xx (slow; off by default). + +Items whose `ontology_id` fails validation are **demoted to unmapped** +(ontology fields nulled, `concept_mapping_provenance: "unmapped"`, +`alignment_method: "validation_failed"`). The item itself is preserved so +exhaustive extraction remains intact. +""" +from __future__ import annotations + +import logging +import re +from typing import Optional + +logger = logging.getLogger("iri_validation") + +# --------------------------------------------------------------------------- +# Per-ontology IRI patterns. Keys are the canonical ontology shortnames +# (case-insensitive); values are regexes that the FULL IRI must match. +# Most biomedical ontologies use OBO PURLs of the form +# http://purl.obolibrary.org/obo/<PREFIX>_<NUMBER> +# but several have their own URL spaces. +# --------------------------------------------------------------------------- + +_OBO_PREFIX_PATTERN = ( + r"^https?://purl\.obolibrary\.org/obo/{prefix}_[0-9]+(?:[/#?].*)?$" +) + +# Curie patterns ("NCBITaxon:10090") accepted alongside full IRIs +_CURIE_PATTERN = r"^{prefix}:[0-9A-Za-z\-_]+$" + +_IRI_PATTERNS: dict[str, tuple[re.Pattern, ...]] = { + # Anatomy + "uberon": (re.compile(_OBO_PREFIX_PATTERN.format(prefix="UBERON")), + re.compile(_CURIE_PATTERN.format(prefix="UBERON"))), + # Cells + "cl": (re.compile(_OBO_PREFIX_PATTERN.format(prefix="CL")), + re.compile(_CURIE_PATTERN.format(prefix="CL"))), + "pcl": (re.compile(_OBO_PREFIX_PATTERN.format(prefix="PCL")), + re.compile(_CURIE_PATTERN.format(prefix="PCL"))), + # Species + "ncbitaxon": (re.compile(_OBO_PREFIX_PATTERN.format(prefix="NCBITaxon")), + re.compile(_CURIE_PATTERN.format(prefix="NCBITaxon"))), + # Diseases + "mondo": (re.compile(_OBO_PREFIX_PATTERN.format(prefix="MONDO")), + re.compile(_CURIE_PATTERN.format(prefix="MONDO"))), + "doid": (re.compile(_OBO_PREFIX_PATTERN.format(prefix="DOID")), + re.compile(_CURIE_PATTERN.format(prefix="DOID"))), + "hp": (re.compile(_OBO_PREFIX_PATTERN.format(prefix="HP")), + re.compile(_CURIE_PATTERN.format(prefix="HP"))), + # Chemistry / drugs + "chebi": (re.compile(_OBO_PREFIX_PATTERN.format(prefix="CHEBI")), + re.compile(_CURIE_PATTERN.format(prefix="CHEBI"))), + "dron": (re.compile(_OBO_PREFIX_PATTERN.format(prefix="DRON")), + re.compile(_CURIE_PATTERN.format(prefix="DRON"))), + # Tissue / cell line + "bto": (re.compile(_OBO_PREFIX_PATTERN.format(prefix="BTO")), + re.compile(_CURIE_PATTERN.format(prefix="BTO"))), + # Methods / assays + "obi": (re.compile(_OBO_PREFIX_PATTERN.format(prefix="OBI")), + re.compile(_CURIE_PATTERN.format(prefix="OBI"))), + # Phenotypes + "mp": (re.compile(_OBO_PREFIX_PATTERN.format(prefix="MP")), + re.compile(_CURIE_PATTERN.format(prefix="MP"))), + # Genes + "hgnc": (re.compile(r"^https?://identifiers\.org/hgnc/[0-9]+$"), + re.compile(r"^https?://www\.genenames\.org/(?:cgi-bin/gene_symbol_report\?(?:hgnc_id=)?HGNC:|data/gene-symbol-report/)?HGNC:[0-9]+$"), + re.compile(r"^HGNC:[0-9]+$")), + "ncbigene": (re.compile(r"^https?://www\.ncbi\.nlm\.nih\.gov/gene/[0-9]+$"), + re.compile(r"^NCBIGene:[0-9]+$")), + "mgi": (re.compile(r"^MGI:[0-9]+$"), + re.compile(r"^https?://www\.informatics\.jax\.org/marker/MGI:[0-9]+$")), + # Proteins + "uniprot": (re.compile(r"^https?://(?:www\.)?uniprot\.org/uniprot(?:kb)?/[A-Z0-9]{6,10}$"), + re.compile(r"^UniProtKB:[A-Z0-9]{6,10}$")), + "pr": (re.compile(_OBO_PREFIX_PATTERN.format(prefix="PR")), + re.compile(_CURIE_PATTERN.format(prefix="PR"))), + # Cellular components / GO + "go": (re.compile(_OBO_PREFIX_PATTERN.format(prefix="GO")), + re.compile(_CURIE_PATTERN.format(prefix="GO"))), + # EFO (general experimental factor) + "efo": (re.compile(r"^https?://www\.ebi\.ac\.uk/efo/EFO_[0-9]+$"), + re.compile(r"^EFO:[0-9]+$")), + # NIFSTD (neuroscience) + "nifstd": (re.compile(r"^https?://uri\.neuinfo\.org/nif/nifstd/[A-Za-z0-9_]+$"),), + # CIDO (COVID) + "cido": (re.compile(_OBO_PREFIX_PATTERN.format(prefix="CIDO")),), +} + +# Generic permissive pattern — used when no per-ontology pattern is registered. +# Requires either a known scheme + path with at least one alphanumeric run, +# or a CURIE-like "PREFIX:LOCAL". +_GENERIC_IRI = re.compile( + r"^(" + r"https?://[A-Za-z0-9_\-.]+(?:/[^\s]*)?" # http(s) URL + r"|[A-Za-z][A-Za-z0-9_]*:[A-Za-z0-9_\-]+" # CURIE + r")$" +) + + +def is_well_formed_iri(iri: Optional[str]) -> bool: + """True if `iri` looks like a real IRI/CURIE (structural check only).""" + if not iri or not isinstance(iri, str): + return False + s = iri.strip() + if not s or s.lower() in ("n/a", "none", "null", "unmapped"): + return False + return bool(_GENERIC_IRI.match(s)) + + +def matches_ontology(iri: Optional[str], ontology: Optional[str]) -> Optional[bool]: + """Stricter check: does the IRI match the patterns registered for the + declared ontology? + + Returns: + True — IRI matches one of the known patterns for that ontology. + False — ontology is registered but the IRI does NOT match any of + its patterns (likely fabricated). + None — ontology is unknown to the registry; no opinion. + """ + if not iri or not ontology: + return None + key = ontology.strip().lower() + patterns = _IRI_PATTERNS.get(key) + if not patterns: + return None + return any(p.match(iri.strip()) for p in patterns) + + +def matches_any_known_ontology(iri: Optional[str]) -> bool: + """Does this IRI match the pattern for ANY ontology we know about? + + Used to accept legitimate cross-ontology mappings (e.g. CIDO results that + reuse an HP IRI as a synonym) without requiring the declared `ontology` + field to match the IRI prefix. + """ + if not iri: + return False + s = iri.strip() + for patterns in _IRI_PATTERNS.values(): + if any(p.match(s) for p in patterns): + return True + return False + + +def validate_item(item: dict, *, strict: bool = True) -> tuple[bool, Optional[str]]: + """Check the ontology fields of one extracted item. + + Policy: + - `concept_mapping_provenance == "llm_knowledge"` is ALWAYS rejected. + The skill does not permit LLM-fabricated mappings. + - `ontology_id` must be structurally well-formed (real URL or CURIE). + - In `strict=True` mode, the IRI must additionally match the registered + pattern for SOME known ontology — its own prefix's pattern. The + declared `ontology` field does NOT have to match the IRI prefix (real + mappers legitimately report cross-ontology synonyms, e.g. CIDO results + that reuse HP IRIs). + - Unknown-ontology IRIs (no per-prefix pattern) are accepted as long as + they're well-formed — we can't assert what we don't know. + + Returns: + (ok, reason). ok=True keeps the mapping. ok=False demotes to unmapped. + """ + oid = item.get("ontology_id") + prov = (item.get("concept_mapping_provenance") or "").strip().lower() + + # explicit unmapped/skipped — nothing to validate + if prov in ("unmapped", "skipped") and not oid: + return True, None + if not oid: + return False, "no ontology_id but provenance not unmapped/skipped" + + # Hard rule #1: zero LLM-knowledge mappings allowed. + if prov == "llm_knowledge": + return False, ("concept_mapping_provenance=llm_knowledge is not " + "permitted; all mappings must come from a tool call") + + # Hard rule #2: must be a well-formed IRI/CURIE. + if not is_well_formed_iri(oid): + return False, f"ontology_id {oid!r} is not a well-formed IRI/CURIE" + + # Hard rule #3 (strict only): IRI must match SOME known ontology pattern. + # We don't require the declared `ontology` field to match the IRI prefix — + # cross-ontology lookups legitimately reuse IRIs across ontology sources. + if strict and not matches_any_known_ontology(oid): + # If the IRI looks like an OBO PURL or identifiers.org URL of a known + # form, accept it even if no per-prefix pattern is registered. + if not _looks_like_real_ontology_iri(oid): + return False, (f"ontology_id {oid!r} does not match any known " + f"ontology pattern; possibly hallucinated") + return True, None + + +# Permissive structural acceptance: catches real-world ontology IRIs whose +# specific prefix isn't yet in _IRI_PATTERNS, while still rejecting strings +# that don't look like ontology IRIs at all. +_OBO_GENERIC = re.compile(r"^https?://purl\.obolibrary\.org/obo/[A-Za-z]+_[0-9]+(?:[/#?].*)?$") +_BIOPORTAL_PURL = re.compile(r"^https?://purl\.bioontology\.org/ontology/[A-Za-z0-9_\-]+/[A-Za-z0-9_\-]+$") +_IDENTIFIERS_ORG = re.compile(r"^https?://identifiers\.org/[A-Za-z0-9._]+(?:/[A-Za-z0-9._:-]+)?$") +_EBI_OLS = re.compile(r"^https?://www\.ebi\.ac\.uk/[A-Za-z0-9/_-]+/[A-Za-z0-9_]+_[0-9]+$") +_SEMANTIC_WEB = re.compile(r"^https?://www\.semanticweb\.org/.+#[A-Za-z0-9_]+$") +_GENERIC_OWL_IRI = re.compile(r"^https?://[A-Za-z0-9./_-]+/[A-Za-z0-9._-]+_[0-9]+$") + + +def _looks_like_real_ontology_iri(iri: str) -> bool: + s = iri.strip() + return any(p.match(s) for p in ( + _OBO_GENERIC, _BIOPORTAL_PURL, _IDENTIFIERS_ORG, _EBI_OLS, + _SEMANTIC_WEB, _GENERIC_OWL_IRI, + )) + + +def demote_to_unmapped(item: dict, reason: Optional[str] = None) -> dict: + """In-place: null out ontology fields and mark the item as failed validation.""" + item["ontology_id"] = None + item["ontology_label"] = None + item["ontology"] = None + item["concept_mapping_provenance"] = "unmapped" + item["alignment_method"] = "validation_failed" + if reason: + # Append to remarks if present so the audit trail is preserved. + prev = item.get("remarks") or "" + if prev and reason not in prev: + item["remarks"] = (prev + " | " + reason).strip(" |") + elif not prev: + item["remarks"] = reason + return item + + +def validate_all(result: dict, *, strict: bool = True) -> dict: + """Walk every entity / key_term / resource item and demote any whose + ontology mapping fails validation (hallucinated, malformed, or + llm_knowledge). Returns the mutated result. + + Adds `result["validation"]` with counts: passed / demoted / by_reason. + """ + counts = {"passed": 0, "demoted": 0} + by_reason: dict[str, int] = {} + + for key in ("entities", "key_terms"): + for it in result.get(key, []) or []: + if not isinstance(it, dict): + continue + ok, reason = validate_item(it, strict=strict) + if ok: + counts["passed"] += 1 + else: + demote_to_unmapped(it, reason) + counts["demoted"] += 1 + if reason: + by_reason[_canonical_reason(reason)] = ( + by_reason.get(_canonical_reason(reason), 0) + 1) + + # Resources nested by index — walk shallowly + res = (result.get("extracted_resources") or result.get("aligned_resources") + or result.get("judge_resource") or {}) + if isinstance(res, dict): + for items in res.values(): + for it in items or []: + if not isinstance(it, dict): + continue + ok, reason = validate_item(it, strict=strict) + if ok: + counts["passed"] += 1 + else: + demote_to_unmapped(it, reason) + counts["demoted"] += 1 + if reason: + by_reason[_canonical_reason(reason)] = ( + by_reason.get(_canonical_reason(reason), 0) + 1) + + result["validation"] = { + "strict_iri_validation": strict, + "counts": counts, + "demoted_by_reason": by_reason, + } + return result + + +def _canonical_reason(reason: str) -> str: + """Bucket the free-text reason into a short tag for the stats.""" + r = reason.lower() + if "llm_knowledge" in r: + return "llm_knowledge_rejected" + if "well-formed" in r: + return "malformed_iri" + if "does not match" in r and "pattern" in r: + return "wrong_ontology_pattern" + if "no ontology_id" in r: + return "missing_ontology_id" + return "other" + + +if __name__ == "__main__": + import json + cases = [ + {"entity": "kidney disease", "ontology": "CIDO", + "ontology_id": "http://purl.obolibrary.org/obo/HP_0012622", + "concept_mapping_provenance": "tool"}, + {"entity": "BDNF", "ontology": "HGNC", + "ontology_id": "HGNC:1033", "concept_mapping_provenance": "tool"}, + {"entity": "hippocampus", "ontology": "UBERON", + "ontology_id": "http://purl.obolibrary.org/obo/UBERON_0002421", + "concept_mapping_provenance": "tool"}, + # Fabricated by LLM + {"entity": "made-up", "ontology": "UBERON", + "ontology_id": "http://purl.obolibrary.org/obo/MADE_UP_999", + "concept_mapping_provenance": "llm_knowledge"}, + # Wrong pattern for declared ontology + {"entity": "weird", "ontology": "NCBITaxon", + "ontology_id": "http://example.com/random-uri", + "concept_mapping_provenance": "tool"}, + # Unknown ontology — accepted if well-formed + {"entity": "x", "ontology": "RandomOnto", + "ontology_id": "http://random.example.org/X_42", + "concept_mapping_provenance": "tool"}, + ] + for c in cases: + ok, reason = validate_item(c) + print(f" {c['entity']:15s} ontology={c['ontology']:12s} ok={ok!s:5s} {reason or ''}") diff --git a/skills/structsense-skills/scripts/json_repair.py b/skills/structsense-skills/scripts/json_repair.py new file mode 100644 index 0000000..1ee0824 --- /dev/null +++ b/skills/structsense-skills/scripts/json_repair.py @@ -0,0 +1,164 @@ +"""Multi-tier JSON repair for LLM output. + +Tier 1: strict json.loads +Tier 2: strip markdown fences, then strict json.loads +Tier 3: json-repair library (handles trailing commas, single quotes, + Python None/True/False, truncated trailing braces) +Tier 4: truncate to last balanced brace/bracket + +Combined with schema-aware repair via repair_to_schema(), which sends an +LLM the validator error and asks for a fixed JSON. + +Adapted from the structsense json_repair_tool.py. +""" +from __future__ import annotations + +import json +import re +from typing import Any, Optional, Callable + +try: + from json_repair import repair_json as _jr_repair +except ImportError: # pragma: no cover + _jr_repair = None + +_FENCE_RE = re.compile(r"^```(?:json)?\s*\n?(.*?)\n?```\s*$", re.DOTALL) + + +def strip_fences(s: str) -> str: + """Remove a surrounding ```json ... ``` fence if present.""" + s = s.strip() + m = _FENCE_RE.match(s) + return m.group(1).strip() if m else s + + +def truncate_to_last_balanced(s: str) -> str: + """Walk the string, tracking brace/bracket depth (ignoring strings). + Return the substring up to and including the last position where depth + returned to 0. Useful for token-limit truncations. + """ + depth = 0 + last_balanced = -1 + in_string = False + escape = False + for i, ch in enumerate(s): + if escape: + escape = False + continue + if ch == "\\": + escape = True + continue + if ch == '"': + in_string = not in_string + continue + if in_string: + continue + if ch in "{[": + depth += 1 + elif ch in "}]": + depth -= 1 + if depth == 0: + last_balanced = i + return s[: last_balanced + 1] if last_balanced > 0 else s + + +def parse_or_repair(raw: str) -> Optional[Any]: + """Run the four-tier repair pipeline. Returns parsed object or None.""" + if not raw or not raw.strip(): + return None + + # Tier 1: strict + try: + return json.loads(raw) + except json.JSONDecodeError: + pass + + # Tier 2: strip fences + stripped = strip_fences(raw) + if stripped != raw: + try: + return json.loads(stripped) + except json.JSONDecodeError: + pass + + # Tier 3: json-repair library + if _jr_repair is not None: + try: + return _jr_repair(stripped, return_objects=True) + except Exception: # pragma: no cover + pass + + # Tier 4: truncate to last balanced and try again + truncated = truncate_to_last_balanced(stripped) + if truncated and truncated != stripped: + try: + return json.loads(truncated) + except json.JSONDecodeError: + if _jr_repair is not None: + try: + return _jr_repair(truncated, return_objects=True) + except Exception: + pass + + return None + + +def repair_to_schema( + parsed: Any, + schema: dict, + llm_call: Callable[[str], str], + max_attempts: int = 2, +): + """Validate `parsed` against `schema`. If invalid, ask the LLM to repair + using the validator's error path. Returns (parsed, errors). + + `llm_call(prompt: str) -> str` should call your model and return raw text + (we'll re-run parse_or_repair on the result). + """ + try: + import jsonschema + except ImportError as e: + raise RuntimeError( + "repair_to_schema requires the jsonschema package: pip install jsonschema" + ) from e + + validator = jsonschema.Draft202012Validator(schema) + errors_history: list[str] = [] + current = parsed + + for _ in range(max_attempts + 1): + errs = sorted(validator.iter_errors(current), key=lambda e: list(e.absolute_path)) + if not errs: + return current, [] + first = errs[0] + path = ".".join(str(p) for p in first.absolute_path) + msg = f"{first.message} at path: {path or '<root>'}" + errors_history.append(msg) + + prompt = ( + "The JSON below is INVALID. Fix it to match the JSON Schema. " + "Output ONLY the fixed JSON — no prose, no markdown fences.\n\n" + f"SCHEMA:\n{json.dumps(schema, indent=2)}\n\n" + f"ERROR:\n{msg}\n\n" + f"INVALID JSON:\n{json.dumps(current, indent=2)}" + ) + repaired_raw = llm_call(prompt) + repaired = parse_or_repair(repaired_raw) + if repaired is None: + return current, errors_history + ["repair attempt returned unparseable text"] + current = repaired + + return current, errors_history + + +if __name__ == "__main__": + bad = """```json + { + "entities": [ + {'entity': 'BDNF', 'label': 'Gene',}, + {"entity": "hippocampus", "label": "BrainRegion",} + ], + "key_terms": [None, True, False] + """ + obj = parse_or_repair(bad) + print(json.dumps(obj, indent=2)) diff --git a/skills/structsense-skills/scripts/llm_client.py b/skills/structsense-skills/scripts/llm_client.py new file mode 100644 index 0000000..6d3177a --- /dev/null +++ b/skills/structsense-skills/scripts/llm_client.py @@ -0,0 +1,182 @@ +"""Minimal multi-provider LLM client. + +Wraps OpenAI / OpenRouter / Anthropic / Ollama behind one call() interface so +the rest of the pipeline doesn't care which provider is in use. + +Usage: + from llm_client import call + + raw = call( + model="openrouter/anthropic/claude-sonnet-4-6", + system=PROMPT_EXTRACTOR, + user=USER_TEXT, + json_mode=True, + temperature=0, + ) + +Model string conventions (same as CrewAI / structsense): + openai/<model> -> OpenAI direct + openrouter/<provider>/<model> -> OpenRouter + anthropic/<model> -> Anthropic direct + ollama/<model> -> local Ollama + gemini/<model> -> Google Gemini +""" +from __future__ import annotations + +import os +import json +import logging +from typing import Optional + +logger = logging.getLogger("llm_client") + + +def _provider_and_model(model: str) -> tuple[str, str]: + """Split 'openrouter/anthropic/claude-...' into ('openrouter', 'anthropic/claude-...').""" + if "/" not in model: + raise ValueError(f"model must include provider prefix, got {model!r}") + provider, _, rest = model.partition("/") + return provider.lower(), rest + + +def call( + model: str, + system: str, + user: str, + *, + temperature: float = 0.0, + max_tokens: int = 4096, + json_mode: bool = False, + response_schema: Optional[dict] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + extra: Optional[dict] = None, +) -> str: + """Single, synchronous LLM call. Returns the raw text reply.""" + provider, real_model = _provider_and_model(model) + extra = extra or {} + + if provider in {"openai", "openrouter"}: + return _openai_compatible( + provider, real_model, system, user, temperature, max_tokens, + json_mode, response_schema, api_key, base_url, extra, + ) + if provider == "anthropic": + return _anthropic(real_model, system, user, temperature, max_tokens, + api_key, extra) + if provider == "ollama": + return _ollama(real_model, system, user, temperature, max_tokens, + json_mode, base_url, extra) + if provider == "gemini": + return _gemini(real_model, system, user, temperature, max_tokens, + json_mode, api_key, extra) + raise ValueError(f"unsupported provider: {provider!r}") + + +# --------------------------------------------------------------------------- +# OpenAI-compatible (OpenAI direct, OpenRouter, vLLM, …) +# --------------------------------------------------------------------------- +def _openai_compatible(provider, real_model, system, user, temperature, + max_tokens, json_mode, response_schema, api_key, + base_url, extra): + from openai import OpenAI # pip install openai + if provider == "openrouter": + api_key = api_key or os.environ.get("OPENROUTER_API_KEY") + base_url = base_url or "https://openrouter.ai/api/v1" + else: + api_key = api_key or os.environ.get("OPENAI_API_KEY") + client = OpenAI(api_key=api_key, base_url=base_url) + + kwargs = dict( + model=real_model, + messages=[{"role": "system", "content": system}, + {"role": "user", "content": user}], + temperature=temperature, + max_tokens=max_tokens, + ) + if response_schema is not None: + kwargs["response_format"] = { + "type": "json_schema", + "json_schema": {"name": "out", "schema": response_schema, "strict": True}, + } + elif json_mode: + kwargs["response_format"] = {"type": "json_object"} + kwargs.update(extra) + + resp = client.chat.completions.create(**kwargs) + return resp.choices[0].message.content or "" + + +# --------------------------------------------------------------------------- +# Anthropic +# --------------------------------------------------------------------------- +def _anthropic(real_model, system, user, temperature, max_tokens, api_key, extra): + from anthropic import Anthropic # pip install anthropic + client = Anthropic(api_key=api_key or os.environ.get("ANTHROPIC_API_KEY")) + kwargs = dict( + model=real_model, + system=system, + max_tokens=max_tokens, + temperature=temperature, + messages=[{"role": "user", "content": user}], + ) + kwargs.update(extra) + resp = client.messages.create(**kwargs) + parts = [] + for block in resp.content: + if getattr(block, "type", None) == "text": + parts.append(block.text) + return "".join(parts) + + +# --------------------------------------------------------------------------- +# Ollama +# --------------------------------------------------------------------------- +def _ollama(real_model, system, user, temperature, max_tokens, json_mode, + base_url, extra): + import requests + base = base_url or "http://localhost:11434" + payload = { + "model": real_model, + "messages": [{"role": "system", "content": system}, + {"role": "user", "content": user}], + "options": {"temperature": temperature, "num_predict": max_tokens}, + "stream": False, + } + if json_mode: + payload["format"] = "json" + payload.update(extra) + r = requests.post(f"{base}/api/chat", json=payload, timeout=600) + r.raise_for_status() + return ((r.json() or {}).get("message") or {}).get("content") or "" + + +# --------------------------------------------------------------------------- +# Gemini +# --------------------------------------------------------------------------- +def _gemini(real_model, system, user, temperature, max_tokens, json_mode, + api_key, extra): + import google.generativeai as genai # pip install google-generativeai + genai.configure(api_key=api_key or os.environ.get("GEMINI_API_KEY")) + config = { + "temperature": temperature, + "max_output_tokens": max_tokens, + } + if json_mode: + config["response_mime_type"] = "application/json" + config.update(extra) + model = genai.GenerativeModel(real_model, system_instruction=system, + generation_config=config) + resp = model.generate_content(user) + return resp.text or "" + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + out = call( + model=os.environ.get("DEMO_MODEL", "ollama/qwen2.5:7b"), + system="You output strict JSON only. No prose, no fences.", + user='Say hello in JSON: {"greeting": "..."}', + json_mode=True, + ) + print(out) diff --git a/skills/structsense-skills/scripts/local_hybrid_map.py b/skills/structsense-skills/scripts/local_hybrid_map.py new file mode 100644 index 0000000..3bc67b2 --- /dev/null +++ b/skills/structsense-skills/scripts/local_hybrid_map.py @@ -0,0 +1,252 @@ +"""Client for a local hybrid (BM25 + dense) concept mapping service. + +The pipeline pattern in this skill is designed around batched ontology lookups. +For thousands of terms, a single HTTP POST to a local service is orders of +magnitude faster than per-term API calls. + +Reference service: https://github.com/sensein/search_hybrid +OpenAPI / interactive docs (default): http://localhost:8000/docs + +Endpoint (default port 8000): + POST /map/batch + { "terms": ["hippocampus", "mouse", ...], "max_results": 1 } + +Health check tried in order: GET /health, then GET /docs (every FastAPI-based +service exposes /docs by default, so this is a reliable "service is up" probe +even when /health hasn't been wired up). + +If the default URL doesn't work, set LOCAL_CONCEPT_MAPPING_URL or pass +base_url=... to the constructor. **Ask the user** for their URL if neither +the default nor BioPortal fallback succeeds — the port and host can vary +based on how they deployed the service. + +Usage: + from local_hybrid_map import LocalHybridMapper + mapper = LocalHybridMapper("http://localhost:8000") + mappings = mapper.map_batch(["hippocampus", "mouse"], max_results=1) +""" +from __future__ import annotations + +import os +import logging +from typing import Iterable, Optional + +import requests + +logger = logging.getLogger("LocalHybridMapper") + +DEFAULT_URL = os.environ.get("LOCAL_CONCEPT_MAPPING_URL", "http://localhost:8000") +DEFAULT_TIMEOUT = float(os.environ.get("LOCAL_CONCEPT_MAPPING_TIMEOUT", 30)) + + +class LocalHybridMapper: + def __init__(self, base_url: str = DEFAULT_URL, + timeout: float = DEFAULT_TIMEOUT, + api_key: Optional[str] = None, + rerank_model: Optional[str] = None): + self.base_url = base_url.rstrip("/") + self.timeout = timeout + self.api_key = api_key or os.environ.get("LOCAL_CONCEPT_MAPPING_API_KEY") \ + or os.environ.get("OPENROUTER_API_KEY") + self.rerank_model = rerank_model or os.environ.get("LOCAL_CONCEPT_MAPPING_MODEL") \ + or os.environ.get("OPENROUTER_MODEL") + + def health(self) -> bool: + """Return True if the service is reachable. + + Tries /health first; falls back to /docs (always present on + FastAPI-based services like search_hybrid). + """ + for path in ("/health", "/docs"): + try: + r = requests.get(f"{self.base_url}{path}", + timeout=min(self.timeout, 5)) + if r.ok: + return True + except requests.RequestException: + continue + return False + + def map_batch(self, terms, max_results: int = 5, + ontologies: Optional[Iterable[str]] = None, + contexts: Optional[Iterable[Optional[str]]] = None + ) -> list[dict]: + """One POST to /map/batch. + + Accepts either: + - a flat iterable of strings (e.g. ["kidney disease", "T2DM", ...]), + plus an optional parallel `contexts` iterable, OR + - an iterable of dicts already shaped as the API expects: + [{"text": "kidney disease", "context": "progressive decline in GFR"}, + {"text": "T2DM", "context": "type 2 diabetes with insulin resistance"}, + {"text": "astrocyte"}] + + API contract (verified against the reference search_hybrid deployment): + + POST /map/batch + { + "max_results": 5, + "text": [{"text": "...", "context": "..."}, ...] + } + + Returns parallel list of canonical mapping dicts: + {"term", "ontology_id", "ontology_label", "ontology", + "score", "concept_mapping_provenance"} + """ + # normalize input to the API's expected shape + items: list[dict] = [] + plain_terms: list[str] = [] + terms = list(terms) + if terms and isinstance(terms[0], dict): + # already shaped — pass through, capture surface forms for the + # output's "term" field. + for it in terms: + if not isinstance(it, dict) or not it.get("text"): + continue + items.append({k: v for k, v in it.items() + if k in ("text", "context") and v is not None}) + plain_terms.append(it["text"]) + else: + ctx_list = list(contexts) if contexts is not None else [] + for i, t in enumerate(terms): + if not t: + continue + entry = {"text": str(t)} + if i < len(ctx_list) and ctx_list[i]: + entry["context"] = str(ctx_list[i]) + items.append(entry) + plain_terms.append(str(t)) + + if not items: + return [] + + body: dict = { + "text": items, + "max_results": max(1, min(20, int(max_results))), + } + if ontologies: + body["ontologies"] = list(ontologies) + if self.rerank_model: + body["rerank_model"] = self.rerank_model + if self.api_key: + body["api_key"] = self.api_key # used only for LLM re-ranking + + r = requests.post(f"{self.base_url}/map/batch", json=body, timeout=self.timeout) + r.raise_for_status() + payload = r.json() or {} + + # Real server shape (verified against the user's deployment): + # { + # "query": "...", + # "type": "batch", + # "results": { + # "<input text>": [ + # {"rank": 1, "ontology_id": ..., "ontology_label": ..., + # "ontology": ..., "final_score": ..., "llm_score": ..., + # "late_interaction_score": ..., "original_score": ...}, + # ... + # ], + # ... + # } + # } + # + # Also tolerated for forward/backward compatibility: + # results: list[{text, candidates: [...]}] + # results: list[{...top candidate flat...}] + # mappings: ... + + results = payload.get("results") + if results is None: + results = payload.get("mappings") + + out: list[dict] = [] + + if isinstance(results, dict): + # Dict keyed by the input surface form (the user-verified shape). + for term in plain_terms: + cands = results.get(term) or [] + if not cands: + out.append(_unmapped(term)) + continue + out.append(_to_canonical(term, cands[0])) + return out + + # Fallback: list shape, parallel to inputs + if isinstance(results, list): + for term, row in zip(plain_terms, results): + if not isinstance(row, dict): + out.append(_unmapped(term)) + continue + cands = (row.get("candidates") or row.get("matches") + or row.get("mappings") or []) + if not cands: + if row.get("ontology_id") or row.get("id"): + out.append(_to_canonical(term, row)) + continue + out.append(_unmapped(term)) + continue + out.append(_to_canonical(term, cands[0])) + return out + + # Unknown shape — mark everything unmapped but don't crash. + return [_unmapped(t) for t in plain_terms] + + def map_one(self, term, context: Optional[str] = None, **kwargs) -> dict: + if isinstance(term, dict): + return self.map_batch([term], **kwargs)[0] + return self.map_batch([term], + contexts=[context] if context else None, + **kwargs)[0] + + +def _to_canonical(term: str, cand: dict) -> dict: + """Map an API candidate dict to the skill's canonical mapping shape. + + Tolerant of common field-name variations across server versions: + - ontology_id / id / iri + - ontology_label / label / prefLabel + - ontology / source / ontology_name + - score / final_score (preferred when available) + """ + score = cand.get("final_score") + if score is None: + score = cand.get("score") + return { + "term": term, + "ontology_id": (cand.get("ontology_id") or cand.get("id") + or cand.get("iri")), + "ontology_label": (cand.get("ontology_label") or cand.get("label") + or cand.get("prefLabel")), + "ontology": (cand.get("ontology") or cand.get("source") + or cand.get("ontology_name")), + "score": score, + "concept_mapping_provenance": "tool", + } + + +def _unmapped(term: str) -> dict: + return { + "term": term, + "ontology_id": None, + "ontology_label": None, + "ontology": None, + "concept_mapping_provenance": "unmapped", + } + + +if __name__ == "__main__": + import json + logging.basicConfig(level=logging.INFO) + m = LocalHybridMapper() + if not m.health(): + print(f"Service not reachable at {m.base_url}") + raise SystemExit(1) + + # Identical to the user-verified curl payload + items = [ + {"text": "kidney disease", "context": "progressive decline in GFR"}, + {"text": "T2DM", "context": "type 2 diabetes with insulin resistance"}, + {"text": "astrocyte"}, + ] + out = m.map_batch(items, max_results=5) + print(json.dumps(out, indent=2, default=str)) diff --git a/skills/structsense-skills/scripts/mask_pass.py b/skills/structsense-skills/scripts/mask_pass.py new file mode 100644 index 0000000..46b6c6b --- /dev/null +++ b/skills/structsense-skills/scripts/mask_pass.py @@ -0,0 +1,208 @@ +"""Mask-mode passes for NER recall and verification. + +Two functions: + +- mask_for_recall(text, items) + Replace every extracted span in `text` with a placeholder of the form + [E<i>], padded to preserve character offsets. Use the resulting masked + text as input to the mask-recall-pass.md prompt. Returns (masked_text, + placeholder_map) where placeholder_map records what each [En] replaced. + +- mask_one_for_verify(text, item, context_chars=300) + Replace just `text[item.start:item.end]` with `[MASK]`, padded to keep + offsets. Return the masked sentence + surrounding context for use with + mask-verify-pass.md. + +- map_masked_offsets_to_original(masked_text, items, placeholder_map) + After the mask-recall pass returns new items with offsets in the + MASKED text, translate those offsets back to the ORIGINAL text. + +These are pure functions — they do not call any LLM. Plug them into +your own driver alongside scripts/llm_client.py and scripts/json_repair.py. +""" +from __future__ import annotations + +from typing import Iterable + + +def _placeholder(i: int, width: int) -> str: + """Return a placeholder string [E<i>] padded with spaces to `width` chars.""" + raw = f"[E{i}]" + if len(raw) > width: + # Span is shorter than the placeholder; we can't preserve offsets in + # this case. Caller should detect and treat masked_text offsets as + # approximate. + return raw + pad = width - len(raw) + left = pad // 2 + right = pad - left + return " " * left + raw + " " * right + + +def mask_for_recall(text: str, items: list[dict], + span_keys: tuple[str, str, str] = ("entity", "start", "end") + ) -> tuple[str, list[dict]]: + """Replace every (start, end) span with a padded placeholder. + + Items can carry either 'entity' or 'term' as their surface field; pass + the relevant span_keys triple. + + Returns: + masked_text: original-length string with spans replaced. + placeholder_map: list of {"placeholder", "entity", "label", + "orig_start", "orig_end", "masked_start", + "masked_end"}. + """ + surface_key, start_key, end_key = span_keys + + # Sort spans by start ascending; resolve overlaps by keeping the longest. + spans = sorted( + ((it[start_key], it[end_key], it) for it in items + if it.get(start_key) is not None and it.get(end_key) is not None), + key=lambda s: (s[0], -(s[1] - s[0])), + ) + chosen: list[tuple[int, int, dict]] = [] + last_end = -1 + for s, e, it in spans: + if s < last_end: + continue # overlap with already-chosen longer span + chosen.append((s, e, it)) + last_end = e + + # Build the masked text in one pass. + masked: list[str] = [] + pmap: list[dict] = [] + cursor = 0 + for i, (s, e, it) in enumerate(chosen): + masked.append(text[cursor:s]) + ph = _placeholder(i, e - s) + masked.append(ph) + # masked_text offset == original offset because length is preserved + # when the placeholder fits; otherwise we record both. + masked_start = sum(len(p) for p in masked) - len(ph) + masked_end = masked_start + len(ph) + pmap.append({ + "placeholder": ph.strip(), + "entity": it.get(surface_key), + "label": it.get("label"), + "orig_start": s, + "orig_end": e, + "masked_start": masked_start, + "masked_end": masked_end, + }) + cursor = e + masked.append(text[cursor:]) + + return "".join(masked), pmap + + +def map_masked_offsets_to_original(masked_text: str, + new_items: Iterable[dict], + placeholder_map: list[dict], + original_text: str, + start_key: str = "start", + end_key: str = "end", + surface_key: str = "entity", + ) -> list[dict]: + """Translate offsets reported in `masked_text` back to `original_text`. + + Because placeholders are padded to the same length as the spans they + replace, character offsets in masked_text equal those in original_text + — UNLESS a span was shorter than its placeholder text. The function + re-verifies each item against `original_text[start:end]` and, on + mismatch, searches for the surface form in the original text. + """ + from .span_validator import repair_span # local import to avoid cycle + out = [] + for it in new_items: + s = it.get(start_key) + e = it.get(end_key) + surface = it.get(surface_key) + if s is None or e is None or surface is None: + continue + if 0 <= s < e <= len(original_text) and original_text[s:e] == surface: + out.append(it) + continue + repaired = repair_span(original_text, it) + if repaired is not None: + out.append(repaired) + return out + + +def mask_one_for_verify(text: str, item: dict, + context_chars: int = 300, + start_key: str = "start", end_key: str = "end", + surface_key: str = "entity", + ) -> dict: + """Replace just one entity with [MASK] (padded) and return the local + context for the mask-verify prompt. + + Returns: + { + "masked_sentence": the sentence with [MASK] in place of the span, + "masked_paragraph": a +/- context_chars window with [MASK] in place, + "placeholder": "[MASK]" + } + """ + s = item[start_key] + e = item[end_key] + ph = _placeholder_mask(e - s) + sent = item.get("sentence") + surface = item.get(surface_key) + if sent and surface: + local_idx = sent.find(surface) + if local_idx >= 0: + masked_sentence = sent[:local_idx] + ph + sent[local_idx + len(surface):] + else: + masked_sentence = sent + else: + masked_sentence = ph + + # paragraph window + w_start = max(0, s - context_chars) + w_end = min(len(text), e + context_chars) + window = text[w_start:s] + ph + text[e:w_end] + return { + "masked_sentence": masked_sentence, + "masked_paragraph": window, + "placeholder": "[MASK]", + } + + +def _placeholder_mask(width: int) -> str: + raw = "[MASK]" + if len(raw) > width: + return raw + pad = width - len(raw) + left = pad // 2 + right = pad - left + return " " * left + raw + " " * right + + +if __name__ == "__main__": + text = ( + "BDNF is upregulated in the hippocampus of adult mice. " + "BDNF protein levels increase following exercise." + ) + items = [ + {"entity": "BDNF", "label": "Gene", + "sentence": "BDNF is upregulated in the hippocampus of adult mice.", + "start": 0, "end": 4}, + {"entity": "hippocampus", "label": "BrainRegion", + "sentence": "BDNF is upregulated in the hippocampus of adult mice.", + "start": 25, "end": 36}, + {"entity": "mice", "label": "Species", + "sentence": "BDNF is upregulated in the hippocampus of adult mice.", + "start": 46, "end": 50}, + {"entity": "BDNF", "label": "Gene", + "sentence": "BDNF protein levels increase following exercise.", + "start": 52, "end": 56}, + ] + masked, pmap = mask_for_recall(text, items) + print("=== masked text ===") + print(masked) + print("\n=== placeholder map ===") + for p in pmap: + print(p) + print("\n=== mask-one for verify (first item) ===") + print(mask_one_for_verify(text, items[1])) diff --git a/skills/structsense-skills/scripts/model_context.py b/skills/structsense-skills/scripts/model_context.py new file mode 100644 index 0000000..5b75eda --- /dev/null +++ b/skills/structsense-skills/scripts/model_context.py @@ -0,0 +1,268 @@ +"""Model context-window registry + token-aware chunk-size math. + +Two things this module gives you: + +1. ``get_model_context_window(model_str)`` — best-known context size (tokens) + for a model string. Patterns are matched longest-first. +2. ``compute_downstream_chunk_size(...)`` — how many items per chunk to feed + into alignment / judge / humanfeedback so the *prompt + payload + output* + stays under the model's context window. + +Adapted from structsense `model_context.py`. The OpenRouter live-probe cache +is omitted (it adds complexity and a network round-trip; the static patterns +are accurate enough for chunk-sizing decisions). +""" +from __future__ import annotations + +import json +import math +import re +from typing import Any, Dict, Optional + +# --------------------------------------------------------------------------- +# Tuning constants. Override per-call via the function arguments; these are +# safe defaults that work across the major frontier models. +# --------------------------------------------------------------------------- + +# Fraction of (context - prompt_overhead) usable for the payload. The rest +# is reserved for the model's output generation. +_CONTEXT_SAFETY_FACTOR: float = 0.70 + +# Conservative overhead for ReAct/tool-loop boilerplate, JSON schema reminders, +# format instructions. Set to ~3k for agent-loop frameworks like CrewAI; set +# to ~500 for direct API calls with a focused system prompt. +_DEFAULT_FRAMEWORK_OVERHEAD_TOKENS: int = 3_000 + + +# --------------------------------------------------------------------------- +# Model family → context window (tokens). Ordered MOST SPECIFIC first so +# the first substring match wins. +# --------------------------------------------------------------------------- +_MODEL_CONTEXT_PATTERNS: list[tuple[str, int]] = [ + # Meta Llama + ("llama-4-scout", 10_000_000), + ("llama-4-maverick", 1_000_000), + ("llama-4", 1_000_000), + ("llama-3.3", 128_000), + ("llama-3.2", 128_000), + ("llama-3.1", 128_000), + ("llama-3", 8_000), + ("llama", 128_000), + + # OpenAI + ("gpt-4.1", 1_000_000), + ("gpt-5-mini", 400_000), + ("gpt-5-nano", 400_000), + ("gpt-5", 400_000), + ("gpt-4o-mini", 128_000), + ("gpt-4o", 128_000), + ("gpt-4-turbo", 128_000), + ("gpt-4", 128_000), + ("gpt-3.5", 4_000), + ("o3", 200_000), + ("o1", 128_000), + + # Anthropic Claude + ("claude-opus-4-7", 1_000_000), # opus-4-7 with extended ctx + ("claude-sonnet-4-6", 200_000), + ("claude-sonnet-4", 200_000), + ("claude-haiku-4-5", 200_000), + ("claude-haiku-4", 200_000), + ("claude-opus-4", 200_000), + ("claude-3-5-sonnet", 200_000), + ("claude-3-5-haiku", 200_000), + ("claude-3", 200_000), + ("claude", 200_000), + + # Google Gemini + ("gemini-2.5", 1_000_000), + ("gemini-2.0", 1_000_000), + ("gemini-1.5", 1_000_000), + ("gemini", 1_000_000), + + # DeepSeek + ("deepseek-r1", 128_000), + ("deepseek-v3", 128_000), + ("deepseek", 128_000), + + # Mistral + ("mistral-large", 128_000), + ("mistral-medium", 128_000), + ("mistral-small", 128_000), + ("mixtral", 32_000), + ("mistral", 128_000), + + # Qwen / Alibaba + ("qwen3-coder", 256_000), + ("qwen3", 256_000), + ("qwen2.5", 128_000), + ("qwen", 128_000), + + # xAI Grok + ("grok-3", 131_000), + ("grok", 131_000), + + # Cohere + ("command-r-plus", 128_000), + ("command-r", 128_000), + ("command", 4_000), + + # Amazon + ("nova-pro", 300_000), + ("nova", 300_000), + + # Microsoft Phi + ("phi-3.5", 128_000), + ("phi-3", 128_000), + ("phi", 128_000), + + # Perplexity Sonar + ("sonar", 127_000), +] + +_DEFAULT_CONTEXT_WINDOW: int = 128_000 + + +def _normalize_model_id(model_str: str) -> str: + """Strip provider prefixes ('openrouter/', 'openai/', 'anthropic/', etc.) + and any ':variant' suffix so substring matching is robust. + """ + m = (model_str or "").strip().lower() + for prefix in ("openrouter/", "openai/", "anthropic/", "google/", + "ollama/", "gemini/", "mistralai/"): + if m.startswith(prefix): + m = m[len(prefix):] + break + if ":" in m: + m = m.split(":")[0] + return m + + +def get_model_context_window(model_str: str) -> int: + """Return the context-window size in tokens for ``model_str``. + + Falls back to ``_DEFAULT_CONTEXT_WINDOW`` (128k) when no pattern matches. + """ + if not model_str: + return _DEFAULT_CONTEXT_WINDOW + m = _normalize_model_id(model_str) + for pattern, ctx in _MODEL_CONTEXT_PATTERNS: + if pattern in m: + return ctx + return _DEFAULT_CONTEXT_WINDOW + + +# --------------------------------------------------------------------------- +# Token estimation for structured payloads +# --------------------------------------------------------------------------- + +def estimate_payload_tokens(payload: Any) -> int: + """Estimate token count for a JSON-serializable payload. + + Uses the character count of the JSON form as a 1:1 token estimate. + For structured data (entity arrays, ontology IDs, offsets), this is + empirically accurate — every delimiter character (`{}`, `[]`, `"`, `:`, `,`) + is its own token. + + Deliberately conservative: may overestimate for text-heavy content (harmless, + just triggers extra chunking), but never underestimates (which would cause + context-overflow errors). + """ + try: + return max(1, len(json.dumps(payload, default=str))) + except Exception: + return 1 + + +def estimate_prompt_tokens(system: str = "", user_template: str = "", + framework_overhead: int = _DEFAULT_FRAMEWORK_OVERHEAD_TOKENS, + ) -> int: + """Estimate the prompt overhead in tokens. + + Strips {placeholder} markers from `user_template` since the payload that + gets injected at runtime is tracked separately via `estimate_payload_tokens`. + """ + text_parts: list[str] = [] + if system: + text_parts.append(system) + if user_template: + text_parts.append(re.sub(r"\{[^}]+\}", "", user_template)) + combined = " ".join(text_parts) + return max(1, len(combined)) + framework_overhead + + +# --------------------------------------------------------------------------- +# Downstream chunk-size math +# --------------------------------------------------------------------------- + +def compute_downstream_chunk_size( + items: list[Any], + model_str: str, + *, + max_workers: int = 8, + prompt_overhead_tokens: Optional[int] = None, + context_window_override: Optional[int] = None, + safety_factor: float = _CONTEXT_SAFETY_FACTOR, + min_chunk: int = 1, + max_chunk: int = 500, +) -> int: + """How many items per chunk to feed to a downstream agent? + + The goal: `prompt_overhead + payload(chunk) + expected_output ≤ + context_window * safety_factor`. We solve for `len(chunk)` and clamp. + + A pragmatic split also caps at `ceil(len(items) / max_workers)` so we + actually use parallelism — no point in one big chunk when 8 workers are + available. + + Returns an integer in [min_chunk, max_chunk]. Use it as + ``chunks = [items[i:i+size] for i in range(0, len(items), size)]``. + """ + if not items: + return min_chunk + + ctx = context_window_override or get_model_context_window(model_str) + overhead = prompt_overhead_tokens or _DEFAULT_FRAMEWORK_OVERHEAD_TOKENS + budget = int((ctx - overhead) * safety_factor) + if budget <= 0: + # Pathological case (overhead exceeds window) — fall back to 1. + return max(min_chunk, 1) + + # average tokens per item + sample = items[: min(50, len(items))] + avg_per_item = max(1, estimate_payload_tokens(sample) // max(1, len(sample))) + + by_budget = max(1, budget // avg_per_item) + by_workers = math.ceil(len(items) / max(1, max_workers)) + + return max(min_chunk, min(max_chunk, by_budget, by_workers)) + + +# --------------------------------------------------------------------------- +# CLI for ad-hoc inspection +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + import argparse + ap = argparse.ArgumentParser( + description="Inspect a model's known context window and recommended " + "downstream chunk size for an items-list of a given size." + ) + ap.add_argument("model", help="model string, e.g. openrouter/anthropic/claude-sonnet-4-6") + ap.add_argument("--items", type=int, default=1000, + help="number of items to chunk for the downstream agent") + ap.add_argument("--workers", type=int, default=8) + ap.add_argument("--avg-item-tokens", type=int, default=250, + help="rough average tokens per item; used to synthesize a sample") + args = ap.parse_args() + + ctx = get_model_context_window(args.model) + sample_items = [{"i": i, "pad": "x" * args.avg_item_tokens} for i in range(args.items)] + size = compute_downstream_chunk_size(sample_items, args.model, + max_workers=args.workers) + print(f"model={args.model}") + print(f"context_window: {ctx:,} tokens") + print(f"recommended chunk size: {size} items per call") + print(f" → {math.ceil(args.items / size)} chunks, " + f"≈{math.ceil((args.items / size) / args.workers)} round(s) " + f"over {args.workers} workers") diff --git a/skills/structsense-skills/scripts/ner_models.py b/skills/structsense-skills/scripts/ner_models.py new file mode 100644 index 0000000..480bd18 --- /dev/null +++ b/skills/structsense-skills/scripts/ner_models.py @@ -0,0 +1,366 @@ +"""Ensemble HuggingFace NER extractors with per-mention provenance. + +Mirrors the multi-model strategy structsense uses (`ner_tool.py`): run +several specialized biomedical NER models in parallel alongside the LLM +extractor, then merge their outputs. Every emitted item carries a +``source_model`` field so the grouped view can report which models +surfaced each entity (and downstream consumers can filter by model). + +Default model roster +-------------------- + d4data/biomedical-ner-all — broad biomedical + mobashgr/BC5CDR-chem-WLT-384-BioELECTRA-Pubmed-ENS-20-5 — chemicals + mobashgr/NCBI-disease-WLT-256-SciBERT-13INS — diseases + alvaroalon2/biobert_genetic_ner — genes / proteins + +Plus, when wired into the pipeline, the LLM extractor adds items with +``source_model="llm_ner:<model_string>"``. + +Dependencies: + pip install transformers torch + +When a model fails to load (missing weights, no GPU, etc.), the runner logs +a warning and skips that model — never crashes the pipeline. + +Returned shape per mention (matches the LLM extractor's per-mention shape +so the same downstream code handles both): + + { + "entity": "BDNF", + "label": "Gene", + "sentence": "BDNF is upregulated in the hippocampus.", + "start": 0, + "end": 4, + "paper_location": null, + "source_model": "alvaroalon2/biobert_genetic_ner", + "source_score": 0.998 # model's own confidence (0–1), if available + } +""" +from __future__ import annotations + +import concurrent.futures as cf +import logging +from dataclasses import dataclass, field +from typing import Callable, Iterable, Optional + +logger = logging.getLogger("ner_models") + + +# --------------------------------------------------------------------------- +# Default roster +# --------------------------------------------------------------------------- + +# Comprehensive default roster. The runner skips unavailable models +# gracefully (missing weights / no transformers / no GPU), so an over-broad +# default is safe — set `hf_models=` explicitly to constrain. +DEFAULT_HF_MODELS: list[str] = [ + # broad biomedical + "d4data/biomedical-ner-all", + + # BC5CDR-derived specialists + "mobashgr/BC5CDR-chem-WLT-384-BioELECTRA-Pubmed-ENS-20-5", + "mobashgr/NCBI-disease-WLT-256-SciBERT-13INS", + + # BioBERT genetic + "alvaroalon2/biobert_genetic_ner", + + # BENT family — consistent PubMedBERT base, one head per entity type. + "pruas/BENT-PubMedBERT-NER-Gene", + "pruas/BENT-PubMedBERT-NER-Chemical", + "pruas/BENT-PubMedBERT-NER-Disease", + "pruas/BENT-PubMedBERT-NER-Anatomical", + "pruas/BENT-PubMedBERT-NER-Cell-Type", + "pruas/BENT-PubMedBERT-NER-Cell-Line", + "pruas/BENT-PubMedBERT-NER-Organism", + "pruas/BENT-PubMedBERT-NER-Bioprocess", +] + + +# Domain profiles — narrower rosters that cost less to load / run when you +# know your domain. Pick by passing `profile=` to ``EnsembleConfig``. +NER_MODEL_PROFILES: dict[str, list[str]] = { + "biomedical_broad": [ + "d4data/biomedical-ner-all", + "pruas/BENT-PubMedBERT-NER-Disease", + "pruas/BENT-PubMedBERT-NER-Chemical", + "pruas/BENT-PubMedBERT-NER-Gene", + "pruas/BENT-PubMedBERT-NER-Organism", + ], + "cns_cells": [ + "pruas/BENT-PubMedBERT-NER-Cell-Type", + "pruas/BENT-PubMedBERT-NER-Cell-Line", + "pruas/BENT-PubMedBERT-NER-Anatomical", + "pruas/BENT-PubMedBERT-NER-Gene", + "alvaroalon2/biobert_genetic_ner", + "d4data/biomedical-ner-all", + ], + "pharmacology": [ + "mobashgr/BC5CDR-chem-WLT-384-BioELECTRA-Pubmed-ENS-20-5", + "pruas/BENT-PubMedBERT-NER-Chemical", + "pruas/BENT-PubMedBERT-NER-Disease", + "mobashgr/NCBI-disease-WLT-256-SciBERT-13INS", + ], + "genetic": [ + "alvaroalon2/biobert_genetic_ner", + "pruas/BENT-PubMedBERT-NER-Gene", + "d4data/biomedical-ner-all", + ], + "clinical": [ + # general clinical / EHR-style NER + "Clinical-AI-Apollo/Medical-NER", + "blaze999/Medical-NER", + "d4data/biomedical-ner-all", + "pruas/BENT-PubMedBERT-NER-Disease", + ], + "minimal": [ + # smallest sensible set: one broad + one disease + one gene + "d4data/biomedical-ner-all", + "pruas/BENT-PubMedBERT-NER-Gene", + "pruas/BENT-PubMedBERT-NER-Disease", + ], + "all": "DEFAULT", # sentinel — expanded at runtime to DEFAULT_HF_MODELS +} + + +# Map raw model-emitted labels to the skill's canonical label taxonomy where +# the mapping is unambiguous. Anything not in this map is passed through +# verbatim (the grouped view + judge stage can sort out edge cases). +_LABEL_NORMALIZATION = { + # d4data emits "Disease_disorder", "Sign_symptom", "Detailed_description"… + "Disease_disorder": "Disease", + "Sign_symptom": "Phenotype", + "Therapeutic_procedure": "Method", + "Diagnostic_procedure": "Method", + "Biological_structure": "Anatomy", + "Body_system": "Anatomy", + "Sex": "Other", + "Age": "Measurement", + "Quantitative_concept": "Measurement", + "Biological_attribute": "Measurement", + + # BC5CDR / BioBERT raw labels (some emit single-class output) + "Chemical": "Chemical", + "Drug": "Drug", + "Disease": "Disease", + "Gene": "Gene", + "Protein": "Protein", + "GENE": "Gene", + "DRUG": "Drug", + "DISEASE": "Disease", + "CHEMICAL": "Chemical", + + # BENT-PubMedBERT family — each single-head model returns a generic + # "B-<TYPE>" / "I-<TYPE>" or aggregated label. Most are unambiguous. + "Anatomical": "Anatomy", + "Cell-Type": "CellType", + "Cell_Type": "CellType", + "Cell-Line": "CellLine", + "Cell_Line": "CellLine", + "Organism": "Species", + "Bioprocess": "Phenomenon", + "Molecular": "Protein", # BENT "Molecular" tags molecular entities (mostly proteins) +} + + +def _normalize_label(raw: Optional[str]) -> Optional[str]: + if raw is None: + return None + return _LABEL_NORMALIZATION.get(raw, raw) + + +# --------------------------------------------------------------------------- +# Span helpers +# --------------------------------------------------------------------------- + +def _sentence_for(text: str, start: int, end: int) -> str: + """Return the smallest substring of ``text`` containing [start, end] that + starts after a ``. !? \\n`` boundary and ends at one. + + Lightweight — no spaCy required. + """ + # walk backwards to the last sentence terminator + s = start + while s > 0 and text[s - 1] not in ".!?\n": + s -= 1 + while s < start and text[s] in " \t\n": + s += 1 + # walk forwards to the next sentence terminator + e = end + n = len(text) + while e < n and text[e - 1] not in ".!?\n": + e += 1 + return text[s:e].strip() + + +# --------------------------------------------------------------------------- +# Model runners (lazy imports — each tolerates missing deps) +# --------------------------------------------------------------------------- + +@dataclass +class ModelResult: + source_model: str + items: list[dict] = field(default_factory=list) + skipped_reason: Optional[str] = None + + +def run_hf_token_classifier(text: str, model_name: str, + aggregation: str = "simple", + device: int = -1) -> ModelResult: + """Run a HuggingFace token-classification pipeline (BERT-style NER model).""" + try: + from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline + except ImportError as e: + return ModelResult(source_model=model_name, + skipped_reason=f"transformers not installed: {e}") + try: + tok = AutoTokenizer.from_pretrained(model_name) + mdl = AutoModelForTokenClassification.from_pretrained(model_name) + nlp = pipeline("ner", model=mdl, tokenizer=tok, + aggregation_strategy=aggregation, device=device) + except Exception as e: + return ModelResult(source_model=model_name, + skipped_reason=f"failed to load: {e}") + + items: list[dict] = [] + # Long inputs: chunk to the tokenizer's max length to avoid truncation + # errors. Each HF call returns spans relative to the chunk; we re-anchor. + try: + max_len = tok.model_max_length or 512 + except AttributeError: + max_len = 512 + # rough char-to-token ratio for biomedical text ≈ 4 + chunk_chars = max(256, (max_len - 32) * 4) + + cursor = 0 + while cursor < len(text): + # break at a whitespace boundary near chunk_chars to avoid cutting words + end = min(len(text), cursor + chunk_chars) + if end < len(text): + space = text.rfind(" ", cursor, end) + if space > cursor + chunk_chars // 2: + end = space + chunk = text[cursor:end] + try: + results = nlp(chunk) + except Exception as e: + logger.warning("hf %s failed on chunk: %s", model_name, e) + cursor = end + continue + for r in results or []: + start = int(r.get("start") or 0) + cursor + stop = int(r.get("end") or 0) + cursor + surface = text[start:stop] + items.append({ + "entity": surface, + "label": _normalize_label(r.get("entity_group") or r.get("entity")), + "sentence": _sentence_for(text, start, stop), + "start": start, + "end": stop, + "paper_location": None, + "source_model": model_name, + "source_score": float(r.get("score") or 0.0), + }) + cursor = end + return ModelResult(source_model=model_name, items=items) + + +# --------------------------------------------------------------------------- +# Ensemble driver +# --------------------------------------------------------------------------- + +@dataclass +class EnsembleConfig: + """Configure the NER ensemble. + + Pick models EITHER by ``profile`` (one of NER_MODEL_PROFILES) OR by + ``hf_models`` (explicit list). If both are set, ``hf_models`` wins. + """ + hf_models: Optional[list[str]] = None + profile: Optional[str] = None # "biomedical_broad" | "cns_cells" | … + device: int = -1 # -1 = CPU; >=0 = CUDA device index + max_workers: int = 4 + min_score: float = 0.0 # drop HF hits below this score + + def resolved_models(self) -> list[str]: + if self.hf_models is not None: + return list(self.hf_models) + if self.profile: + picked = NER_MODEL_PROFILES.get(self.profile) + if picked is None: + raise ValueError( + f"unknown NER profile {self.profile!r}. " + f"Available: {sorted(NER_MODEL_PROFILES)}") + if picked == "DEFAULT": + return list(DEFAULT_HF_MODELS) + return list(picked) + return list(DEFAULT_HF_MODELS) + + +def run_ensemble(text: str, config: Optional[EnsembleConfig] = None, + ) -> tuple[list[dict], list[dict]]: + """Run every configured model in parallel. Returns (items, per_model_meta). + + items: flat list of mentions across all models, each with source_model + per_model_meta: [{"source_model", "count", "skipped_reason"}] + """ + cfg = config or EnsembleConfig() + models = cfg.resolved_models() + jobs: list[Callable[[], ModelResult]] = [ + (lambda mid=mid: run_hf_token_classifier(text, mid, device=cfg.device)) + for mid in models + ] + + items: list[dict] = [] + meta: list[dict] = [] + with cf.ThreadPoolExecutor(max_workers=cfg.max_workers) as ex: + for res in ex.map(lambda f: f(), jobs): + if res.skipped_reason: + logger.warning("skipped %s: %s", res.source_model, res.skipped_reason) + meta.append({"source_model": res.source_model, "count": 0, + "skipped_reason": res.skipped_reason}) + continue + keep = [it for it in res.items if + it.get("source_score", 1.0) >= cfg.min_score] + items.extend(keep) + meta.append({"source_model": res.source_model, + "count": len(keep), + "skipped_reason": None}) + return items, meta + + +# --------------------------------------------------------------------------- +# Merge with LLM extractor output (the prompts/extractor-ner-*.md results) +# --------------------------------------------------------------------------- + +def annotate_llm_provenance(llm_entities: Iterable[dict], llm_model: str) -> list[dict]: + """Tag every LLM-emitted item with source_model so it merges cleanly with + ensemble output. Call this on the result of pipeline.extract(). + """ + out = [] + for it in llm_entities: + it = dict(it) + it.setdefault("source_model", f"llm_ner:{llm_model}") + out.append(it) + return out + + +def merge_ensemble_and_llm(ensemble_items: list[dict], + llm_items: list[dict]) -> list[dict]: + """Concatenate; do NOT deduplicate. Different models surfacing the same + span is signal, not noise — the grouped view will record both sources + and the consensus count. + """ + return list(ensemble_items) + list(llm_items) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + demo = ("BDNF is upregulated in the hippocampus of adult mice. " + "Patients with Alzheimer's disease show reduced BDNF protein levels. " + "We administered ketamine at 10 mg/kg.") + # Smoke test with an empty model list — proves the runner doesn't crash + # when transformers isn't installed or no models are configured. + items, meta = run_ensemble(demo, EnsembleConfig(hf_models=[])) + print("per-model:", meta) + print(f"\n{len(items)} items") + print("\nAvailable profiles:", sorted(NER_MODEL_PROFILES)) diff --git a/skills/structsense-skills/scripts/normalize_result.py b/skills/structsense-skills/scripts/normalize_result.py new file mode 100644 index 0000000..2da17d1 --- /dev/null +++ b/skills/structsense-skills/scripts/normalize_result.py @@ -0,0 +1,319 @@ +"""Normalize a pipeline result to the canonical skill shape — idempotent. + +Some models (and older runs) emit the legacy shape: + - paper_title / doi repeated on every entity + - no source_metadata block + - no source_model field + - no entities_grouped / stats + +This module rewrites such results into the canonical shape. It is safe to run +on already-canonical results (idempotent). + +Always run this in the pipeline before writing the *_final.json. It is also +exposed as a CLI so users can repair legacy files in place: + + python -m scripts.normalize_result paper_final.json \\ + --input paper.txt \\ + --llm-model openrouter/anthropic/claude-sonnet-4-6 + +What it does (each step is a no-op if already done): +1. Lift `paper_title` + `doi` off every entity/key_term into top-level + `source_metadata` (deduplicating by most-common value). +2. Strip per-entity `paper_title` / `doi` keys. +3. If an item is missing `source_model`, tag it `llm_ner:<llm_model>` (or + `llm_ner:unknown` if the model isn't known). +4. Attach `entities_grouped` + `key_terms_grouped` via group_by_entity. +5. Compute and embed `stats` via stats.compute_stats. +6. Add `source_path` to `source_metadata` if `input_path` is provided. +""" +from __future__ import annotations + +import argparse +import json +import sys +from collections import Counter +from pathlib import Path +from typing import Any, Mapping, Optional + + +# --------------------------------------------------------------------------- +# Step 1+2: source_metadata lifting +# --------------------------------------------------------------------------- + +_DOC_LEVEL_KEYS = ("paper_title", "doi", "source_path") + + +def lift_doc_metadata(result: dict, *, input_path: Optional[str] = None) -> dict: + """Move paper_title/doi off every item to top-level source_metadata. + + Deduplicates: the value most commonly seen across items wins. If items + disagree, the others are stored under + ``source_metadata.duplicate_values_seen`` for audit. + """ + sm = dict(result.get("source_metadata") or {}) + + for field in _DOC_LEVEL_KEYS: + if sm.get(field): + continue + seen: Counter = Counter() + for key in ("entities", "key_terms"): + for it in result.get(key, []) or []: + v = it.get(field) + if v not in (None, ""): + seen[v] += 1 + if not seen: + continue + top, _ = seen.most_common(1)[0] + sm[field] = top + # audit if disagreement + if len(seen) > 1: + dup = sm.setdefault("duplicate_values_seen", {}) + dup[field] = [v for v, _ in seen.most_common()][1:] + + if input_path and not sm.get("source_path"): + sm["source_path"] = input_path + + # strip per-item paper_title/doi/source_path + for key in ("entities", "key_terms"): + for it in result.get(key, []) or []: + for field in _DOC_LEVEL_KEYS: + it.pop(field, None) + + if sm: + result["source_metadata"] = sm + return result + + +# --------------------------------------------------------------------------- +# Step 3: tag source_model +# --------------------------------------------------------------------------- + +def tag_missing_source_model(result: dict, llm_model: Optional[str]) -> dict: + """Tag any item missing `source_model` as `llm_ner:<llm_model>`. + + Items that already have source_model are left alone. + """ + label = f"llm_ner:{llm_model}" if llm_model else "llm_ner:unknown" + for key in ("entities", "key_terms"): + for it in result.get(key, []) or []: + if not it.get("source_model"): + it["source_model"] = label + return result + + +# --------------------------------------------------------------------------- +# alignment_method inference +# --------------------------------------------------------------------------- + +# When the alignment was clearly done (concept_mapping_provenance is set) but +# the per-item alignment_method field is missing, infer the most likely +# method so downstream consumers see a consistent provenance trail. +_PROVENANCE_TO_METHOD = { + "tool": "direct_tool_call", # batched HTTP call to mapping service + "llm_knowledge": "llm_agent", # LLM filled it in from prior knowledge + "unmapped": "direct_tool_call", # service responded but had no hit + "skipped": "skipped", # alignment stage was explicitly disabled +} + + +def tag_missing_alignment_method(result: dict) -> dict: + """For any item with `concept_mapping_provenance` set but missing + `alignment_method`, derive the method from the provenance. + + This fixes the common case where a user (or a previous pipeline) tagged + only the provenance field, leaving downstream consumers with a + `by_method: {"missing": N}` row in stats. + """ + for key in ("entities", "key_terms"): + for it in result.get(key, []) or []: + if it.get("alignment_method"): + continue + prov = it.get("concept_mapping_provenance") + if prov in _PROVENANCE_TO_METHOD: + it["alignment_method"] = _PROVENANCE_TO_METHOD[prov] + return result + + +# --------------------------------------------------------------------------- +# task_type inference +# --------------------------------------------------------------------------- + +def infer_task_type(result: dict) -> dict: + """Set `result["task_type"]` if it's missing, by looking at what shape + the result actually contains. + """ + if result.get("task_type"): + return result + if result.get("entities") or result.get("key_terms"): + result["task_type"] = "ner" + elif result.get("extracted_resources") or result.get("aligned_resources") \ + or result.get("judge_resource"): + result["task_type"] = "resource" + elif result.get("activity") or result.get("items"): + result["task_type"] = "structured_extraction" + return result + + +# --------------------------------------------------------------------------- +# Step 4+5: grouped views + stats (delegated) +# --------------------------------------------------------------------------- + +def attach_grouped(result: dict) -> dict: + try: + from group_by_entity import attach_grouped_views + except ImportError: + # When called from outside the scripts/ dir + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from group_by_entity import attach_grouped_views # type: ignore + attach_grouped_views(result) + return result + + +def attach_stats(result: dict, *, + input_path: Optional[str] = None, + input_text_chars: Optional[int] = None, + chunk_size_chars: Optional[int] = None, + chunk_count: Optional[int] = None) -> dict: + try: + from stats import compute_stats + except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from stats import compute_stats # type: ignore + + # If the result already has timings, keep them; otherwise start fresh. + existing = (result.get("stats") or {}).get("elapsed_seconds") or {} + + input_meta: dict[str, Any] = {"input_path": input_path} + if input_text_chars is not None: + input_meta["char_count"] = input_text_chars + if chunk_size_chars is not None: + input_meta["chunk_size_chars"] = chunk_size_chars + if chunk_count is not None: + input_meta["chunk_count"] = chunk_count + + result["stats"] = compute_stats( + result, + timings=existing, + input_meta=input_meta, + alignment_meta=(result.get("stats") or {}).get("alignment") or {}, + judge_meta=(result.get("stats") or {}).get("judge") or {}, + ) + return result + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + +def _validate_iris(result: dict, *, strict: bool = True) -> dict: + """Run strict IRI validation and demote llm_knowledge / hallucinated mappings.""" + try: + from iri_validation import validate_all + except ImportError: + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from iri_validation import validate_all # type: ignore + validate_all(result, strict=strict) + return result + + +def normalize(result: dict, *, + llm_model: Optional[str] = None, + input_path: Optional[str] = None, + input_text_chars: Optional[int] = None, + chunk_size_chars: Optional[int] = None, + chunk_count: Optional[int] = None, + strict_iri_validation: bool = True) -> dict: + """Run all normalization steps in order. Idempotent. + + Pipeline: + 1. lift paper_title/doi → source_metadata + 2. tag missing source_model + 3. tag missing alignment_method (from concept_mapping_provenance) + 4. infer task_type from result shape + 5. **strict IRI validation** — reject llm_knowledge provenance and + malformed/hallucinated IRIs. Failed items get ontology fields nulled + and `concept_mapping_provenance: "unmapped"`, + `alignment_method: "validation_failed"`. + 6. unify ontology across mentions of the same (entity, label) + 7. attach entities_grouped + key_terms_grouped + 8. compute stats (totals, by_label, by_source_model, by_provenance, etc.) + """ + lift_doc_metadata(result, input_path=input_path) + tag_missing_source_model(result, llm_model) + tag_missing_alignment_method(result) + infer_task_type(result) + _validate_iris(result, strict=strict_iri_validation) + attach_grouped(result) + attach_stats( + result, + input_path=input_path, + input_text_chars=input_text_chars, + chunk_size_chars=chunk_size_chars, + chunk_count=chunk_count, + ) + return result + + +# --------------------------------------------------------------------------- +# CLI: fix a legacy result file in place +# --------------------------------------------------------------------------- + +def _main(): + ap = argparse.ArgumentParser( + description="Normalize a legacy pipeline result file to the canonical " + "skill shape. Idempotent; safe to run on already-canonical " + "outputs." + ) + ap.add_argument("path", help="Path to a result JSON file. Modified in place " + "unless --out is given.") + ap.add_argument("--out", default=None, + help="Write to this path instead of overwriting.") + ap.add_argument("--input", default=None, + help="Path to the original source text file (for source_path " + "and char_count in stats).") + ap.add_argument("--llm-model", default=None, + help="LLM model string used for the run (for tagging items " + "missing source_model). e.g. " + "openrouter/anthropic/claude-sonnet-4-6") + ap.add_argument("--no-strict-iri", action="store_true", + help="Disable strict IRI validation. By default, items " + "with concept_mapping_provenance=llm_knowledge or " + "malformed/hallucinated IRIs are demoted to " + "unmapped. This flag preserves them as-is (useful " + "only for debugging or for inputs that pre-date the " + "strict policy).") + args = ap.parse_args() + + path = Path(args.path) + data = json.loads(path.read_text()) + + chars = None + if args.input: + try: + chars = len(Path(args.input).read_text()) + except OSError: + chars = None + + normalize( + data, + llm_model=args.llm_model, + input_path=args.input, + input_text_chars=chars, + strict_iri_validation=not args.no_strict_iri, + ) + + out = Path(args.out) if args.out else path + out.write_text(json.dumps(data, indent=2, default=str)) + + try: + from stats import format_summary # type: ignore + print(format_summary(data["stats"]), file=sys.stderr) + except Exception: + pass + print(f"normalized {path} -> {out}", file=sys.stderr) + + +if __name__ == "__main__": + _main() diff --git a/skills/structsense-skills/scripts/ols_map.py b/skills/structsense-skills/scripts/ols_map.py new file mode 100644 index 0000000..8f39f59 --- /dev/null +++ b/skills/structsense-skills/scripts/ols_map.py @@ -0,0 +1,104 @@ +"""OLS (EBI Ontology Lookup Service) concept mapping client. + +No API key required. Good free fallback / alternative to BioPortal. + +Endpoint: https://www.ebi.ac.uk/ols4/api/search +Docs: https://www.ebi.ac.uk/ols4/help + +Usage: + from ols_map import OlsMapper + mapper = OlsMapper() + print(mapper.map_one("hippocampus", ontologies=["uberon"])) + # -> {"ontology_id": "...", "ontology_label": "...", "ontology": "UBERON", + # "concept_mapping_provenance": "tool"} +""" +from __future__ import annotations + +import logging +import time +from functools import lru_cache +from typing import Iterable, Optional + +import requests + +logger = logging.getLogger("OlsMapper") + +OLS_SEARCH = "https://www.ebi.ac.uk/ols4/api/search" + + +class OlsMapper: + def __init__(self, cache_size: int = 2000, request_interval: float = 0.1, + timeout: float = 15.0): + self.timeout = timeout + self.request_interval = request_interval + self._last_request = 0.0 + self._search = lru_cache(maxsize=cache_size)(self._search_uncached) + + def _throttle(self) -> None: + elapsed = time.monotonic() - self._last_request + wait = self.request_interval - elapsed + if wait > 0: + time.sleep(wait) + self._last_request = time.monotonic() + + def _search_uncached(self, term: str, ontologies: tuple, rows: int, + exact: bool, kind: str): + params = { + "q": term, + "rows": rows, + "exact": "true" if exact else "false", + "type": kind, # "class" | "individual" | "property" + } + if ontologies: + params["ontology"] = ",".join(o.lower() for o in ontologies) + for attempt in range(4): + self._throttle() + try: + resp = requests.get(OLS_SEARCH, params=params, timeout=self.timeout) + except requests.RequestException as e: + logger.warning("OLS request failed (%s); retrying", e) + time.sleep(2 ** attempt) + continue + if resp.status_code in (429, 503): + time.sleep(2 ** attempt) + continue + if resp.status_code >= 500: + time.sleep(2 ** attempt) + continue + resp.raise_for_status() + return ((resp.json() or {}).get("response") or {}).get("docs", []) or [] + return [] + + def map_one(self, term: str, ontologies: Optional[Iterable[str]] = None, + max_results: int = 1, exact: bool = False, + kind: str = "class") -> dict: + hits = self._search(term, tuple(ontologies or ()), max_results, exact, kind) + if not hits: + return { + "term": term, + "ontology_id": None, + "ontology_label": None, + "ontology": None, + "concept_mapping_provenance": "unmapped", + } + top = hits[0] + return { + "term": term, + "ontology_id": top.get("iri"), + "ontology_label": top.get("label"), + "ontology": (top.get("ontology_name") or "").upper() or None, + "concept_mapping_provenance": "tool", + } + + def map_batch(self, terms: Iterable[str], + ontologies: Optional[Iterable[str]] = None, + max_results: int = 1, **kwargs) -> list[dict]: + return [self.map_one(t, ontologies, max_results, **kwargs) for t in terms] + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + m = OlsMapper() + for r in m.map_batch(["hippocampus", "Mus musculus", "BDNF"], + ontologies=["uberon", "ncbitaxon", "hgnc"]): + print(r) diff --git a/skills/structsense-skills/scripts/pipeline.py b/skills/structsense-skills/scripts/pipeline.py new file mode 100644 index 0000000..d90ab1d --- /dev/null +++ b/skills/structsense-skills/scripts/pipeline.py @@ -0,0 +1,632 @@ +"""End-to-end reference pipeline: extract -> align -> judge. + +Standalone, model-agnostic. Wires together the other scripts in this folder. + +Usage: + python -m scripts.pipeline --task ner --input paper.txt \ + --extractor openrouter/anthropic/claude-sonnet-4-6 \ + --judge openrouter/openai/gpt-4o-mini \ + --mapper ols \ + --out result.json + +This is a *reference* implementation, not a framework. Copy and adapt. +""" +from __future__ import annotations + +import argparse +import asyncio +import concurrent.futures as cf +import json +import logging +import sys +import time +from pathlib import Path +from typing import Any, Callable, Optional + +# Sibling imports — assume `scripts/` is on PYTHONPATH or run with -m. +from chunking import chunk_by_sentences, reanchor_items, dedupe +from json_repair import parse_or_repair +from span_validator import validate_all +from llm_client import call as llm_call +from stats import compute_stats, format_summary +from group_by_entity import attach_grouped_views +from ner_models import ( + run_ensemble, EnsembleConfig, annotate_llm_provenance, merge_ensemble_and_llm, +) +from normalize_result import ( + lift_doc_metadata, tag_missing_source_model, +) + +logger = logging.getLogger("pipeline") + +# Prompt files — load once from the prompts/ directory. +_PROMPTS_DIR = Path(__file__).resolve().parent.parent / "prompts" + + +def _load_prompt(name: str) -> str: + """Load a system prompt from prompts/<name>.md. Strips the markdown + scaffolding and returns the contents of the first fenced ``` block under + a `## System` heading. + """ + path = _PROMPTS_DIR / f"{name}.md" + text = path.read_text() + marker = "## System" + if marker not in text: + raise RuntimeError(f"{path}: missing '## System' heading") + after = text.split(marker, 1)[1] + if "```" not in after: + raise RuntimeError(f"{path}: no fenced block under ## System") + body = after.split("```", 2)[1] + # Strip a leading language hint if present. + if body.startswith("\n"): + body = body[1:] + return body.strip() + + +# --------------------------------------------------------------------------- +# Extraction +# --------------------------------------------------------------------------- + +def extract(text: str, *, model: str, task: str, + metadata: Optional[dict] = None, + chunk_size: int = 2000, max_workers: int = 8) -> dict: + """Chunk -> extract per chunk in parallel -> reanchor -> merge -> dedup.""" + system = _load_prompt(f"extractor-{task}") + chunks = chunk_by_sentences(text, max_chars=chunk_size) + logger.info("extract: %d chunks", len(chunks)) + + def one(c): + user = ( + f"INPUT TEXT:\n<<<\n{c['text']}\n>>>\n\n" + f"METADATA:\n{json.dumps(metadata or {}, indent=2)}\n" + ) + raw = llm_call(model=model, system=system, user=user, + json_mode=True, temperature=0) + parsed = parse_or_repair(raw) or {} + return c["start"], parsed + + results: list[tuple[int, dict]] = [] + with cf.ThreadPoolExecutor(max_workers=max_workers) as ex: + for r in ex.map(one, chunks): + results.append(r) + + entities, key_terms, resources = [], [], [] + for chunk_start, r in results: + if "entities" in r: + entities.extend(reanchor_items(r["entities"], chunk_start)) + if "key_terms" in r: + key_terms.extend(reanchor_items(r["key_terms"], chunk_start)) + if "extracted_resources" in r: + for k, lst in (r["extracted_resources"] or {}).items(): + resources.extend(lst or []) + + out: dict[str, Any] = {} + if entities or task == "ner": + valid, dropped = validate_all(text, entities) + if dropped: + logger.warning("dropped %d entities with invalid spans", len(dropped)) + out["entities"] = dedupe(valid, key_fields=("entity", "start", "end")) + if key_terms or task == "ner": + valid, dropped = validate_all(text, key_terms) + out["key_terms"] = dedupe(valid, key_fields=("term", "start", "end")) + if resources: + out["extracted_resources"] = {"1": resources} + out["task_type"] = task + return out + + +# --------------------------------------------------------------------------- +# Alignment via direct mapping tool (no LLM) +# --------------------------------------------------------------------------- + +def align_direct(extraction: dict, *, + mapper_backend: Optional[str] = None, + mapper=None, + ontologies_for_label: Optional[Callable[[str], list[str]]] = None + ) -> dict: + """Walk extraction items, batch-map their surface forms, merge the mappings + back in. No LLM. Marks every item with alignment_method=direct_tool_call. + + Either pass ``mapper`` (a pre-built client from build_mapper_with_cascade) + or ``mapper_backend`` (a string name) — the latter is for tests/standalone + use; the production path uses the cascade builder. + """ + if mapper is None: + if mapper_backend is None: + raise ValueError("align_direct: pass either mapper or mapper_backend") + mapper = _make_mapper(mapper_backend) + + def annotate(items: list[dict], surface_key: str): + if not items: + return + terms = [it.get(surface_key) for it in items] + ontologies = (ontologies_for_label(items[0].get("label")) if ontologies_for_label + else None) + mappings = mapper.map_batch(terms, ontologies=ontologies, max_results=1) + for it, m in zip(items, mappings): + it["ontology_id"] = m.get("ontology_id") + it["ontology_label"] = m.get("ontology_label") + it["ontology"] = m.get("ontology") + it["concept_mapping_provenance"] = m.get("concept_mapping_provenance", + "unmapped") + it["alignment_method"] = "direct_tool_call" + + if "entities" in extraction: + annotate(extraction["entities"], "entity") + if "key_terms" in extraction: + annotate(extraction["key_terms"], "term") + return extraction + + +def _make_mapper(backend: str, **kwargs): + backend = (backend or "").lower() + if backend == "ols": + from ols_map import OlsMapper + return OlsMapper() + if backend == "bioportal": + from bioportal_map import BioPortalMapper + return BioPortalMapper() + if backend in ("local", "local_hybrid"): + from local_hybrid_map import LocalHybridMapper + return LocalHybridMapper(**kwargs) + raise ValueError(f"unknown mapper backend: {backend!r}") + + +def build_mapper_with_cascade( + preferred: str = "local", + *, + local_url: str = "http://localhost:8000", + ask_user: Optional[Callable[[str], Optional[str]]] = None, + allow_ols_fallback: bool = False, +) -> tuple[object, dict]: + """Build a concept-mapping client with fallbacks. + + **Policy: concept mapping is MANDATORY.** This function never silently + returns "no mapper". It either returns a working tool-backed mapper, or + raises RuntimeError so the caller can surface the problem to the user. + + Cascade (default ``preferred='local'``): + 1. Try local hybrid service at ``local_url`` (default http://localhost:8000). + Health-check via GET /health then /docs. + 2. On failure, try BioPortal (requires BIOPORTAL_API_KEY). + 3. On further failure, call ``ask_user`` (if provided) for an + alternative local URL — deployments use non-default ports and + reverse-proxied paths. If ``ask_user`` returns a non-empty URL, + retry the local backend with that URL. + 4. Final fallback: raise RuntimeError. The caller MUST surface this + to the user, not silently skip alignment with `llm_knowledge`. + + OLS is no longer in the default cascade — it doesn't cover genes + (HGNC), and the policy is to prefer the local hybrid + BioPortal for + completeness. Pass ``allow_ols_fallback=True`` to enable it as a last + resort (only useful for anatomy / cell / disease / chemical extraction). + + ``ask_user(prompt) -> Optional[str]`` is the integration point for + interactive callers (CLI prompts, or a Claude Code skill that asks + the user via natural language). Pass ``ask_user=None`` for fully + automatic runs that fail fast. + + Returns: + (mapper, meta) where meta is suitable for `alignment_meta` in + stats.compute_stats(). Keys: + mapper_used: "local_hybrid" | "bioportal" + mapper_url: str + fallback_triggered: bool + cascade_history: list[str] # each backend attempted + user_provided_url: bool + """ + history: list[str] = [] + user_provided_url = False + + def _try_local(url: str): + from local_hybrid_map import LocalHybridMapper + m = LocalHybridMapper(base_url=url) + if not m.health(): + raise RuntimeError(f"local_hybrid health check failed at {url}") + return m + + def _try_bioportal(): + if not os.environ.get("BIOPORTAL_API_KEY"): + raise RuntimeError("BIOPORTAL_API_KEY not set") + from bioportal_map import BioPortalMapper + return BioPortalMapper() + + # 1) preferred local (or anything that's the preferred backend) + if preferred in ("local", "local_hybrid"): + history.append(f"local_hybrid@{local_url}") + try: + m = _try_local(local_url) + return m, {"mapper_used": "local_hybrid", + "mapper_url": local_url, + "fallback_triggered": False, + "cascade_history": history, + "user_provided_url": user_provided_url} + except Exception as e: + logger.warning("local_hybrid unavailable at %s: %s", local_url, e) + elif preferred == "bioportal": + history.append("bioportal") + try: + return _try_bioportal(), {"mapper_used": "bioportal", + "mapper_url": "https://data.bioontology.org", + "fallback_triggered": False, + "cascade_history": history, + "user_provided_url": False} + except Exception as e: + logger.warning("bioportal unavailable: %s", e) + elif preferred == "ols": + history.append("ols") + from ols_map import OlsMapper + return OlsMapper(), {"mapper_used": "ols", + "mapper_url": "https://www.ebi.ac.uk/ols4/api", + "fallback_triggered": False, + "cascade_history": history, + "user_provided_url": False} + else: + raise ValueError(f"unknown preferred backend {preferred!r}") + + # 2) BioPortal fallback (only reached if preferred was 'local' and failed) + history.append("bioportal") + try: + m = _try_bioportal() + return m, {"mapper_used": "bioportal", + "mapper_url": "https://data.bioontology.org", + "fallback_triggered": True, + "cascade_history": history, + "user_provided_url": False} + except Exception as e: + logger.warning("bioportal fallback unavailable: %s", e) + + # 3) Ask the user for an alternative URL + if ask_user is not None: + prompt = ( + f"Concept-mapping cascade failed.\n" + f" tried local_hybrid at {local_url}: not reachable\n" + f" tried BioPortal: not configured or unreachable\n" + f"The local_hybrid service URL may differ in your deployment. " + f"Enter an alternate URL (e.g. http://localhost:9000) or press " + f"Enter to skip alignment:" + ) + alt_url = ask_user(prompt) + if alt_url and alt_url.strip(): + alt_url = alt_url.strip().rstrip("/") + history.append(f"local_hybrid@{alt_url} (user-provided)") + try: + m = _try_local(alt_url) + return m, {"mapper_used": "local_hybrid", + "mapper_url": alt_url, + "fallback_triggered": True, + "cascade_history": history, + "user_provided_url": True} + except Exception as e: + logger.warning("user-provided local_hybrid at %s also failed: %s", + alt_url, e) + history.append("user_url_failed") + + # 4) Optional OLS fallback (only if caller explicitly allowed it). + # OLS doesn't cover HGNC genes — only useful for anatomy/cell/disease/chemical. + if allow_ols_fallback: + history.append("ols") + try: + from ols_map import OlsMapper + return OlsMapper(), {"mapper_used": "ols", + "mapper_url": "https://www.ebi.ac.uk/ols4/api", + "fallback_triggered": True, + "cascade_history": history, + "user_provided_url": False, + "warning": "OLS fallback active — gene mappings " + "(HGNC) will be missing."} + except Exception as e: + logger.warning("OLS fallback failed: %s", e) + history.append("ols_failed") + + # Cascade exhausted. We do NOT silently skip — the user policy is that + # concept mapping is mandatory and must come from a tool. The caller + # should surface this and let the user either (a) start the local + # service, (b) set BIOPORTAL_API_KEY, or (c) explicitly opt out via + # mapper_backend=None. + raise RuntimeError( + f"concept-mapping cascade exhausted. tried: {history}. " + f"This skill enforces tool-backed mappings only (no LLM-knowledge " + f"fallbacks). Either: (a) start the local hybrid mapper at " + f"{local_url}, (b) export BIOPORTAL_API_KEY, (c) explicitly opt out " + f"of alignment by passing mapper_backend=None to run() — items will " + f"be marked concept_mapping_provenance='skipped'." + ) + + +def _stdin_ask(prompt: str) -> Optional[str]: + """Default ask_user implementation: read from stdin if attached.""" + if not sys.stdin or not sys.stdin.isatty(): + return None + print(prompt, file=sys.stderr, flush=True) + try: + return input("> ") + except (EOFError, KeyboardInterrupt): + return None + + +# --------------------------------------------------------------------------- +# Judge (batched direct API) +# --------------------------------------------------------------------------- + +def judge(aligned: dict, source_text: str, *, model: str, + max_workers: int = 8, auto_approve: bool = False) -> dict: + if auto_approve: + for key in ("entities", "key_terms"): + for it in aligned.get(key, []) or []: + it["judge_score"] = 1.0 + it["remarks"] = "auto-approved" + it["judge_method"] = "auto_approved" + return aligned + + system = _load_prompt("judge") + + def one(item: dict) -> dict: + user = ( + f"ALIGNED ITEM TO JUDGE:\n{json.dumps(item, indent=2)}\n\n" + f"SOURCE CONTEXT:\n{item.get('sentence') or source_text[:2000]}\n" + ) + raw = llm_call(model=model, system=system, user=user, + json_mode=True, temperature=0) + parsed = parse_or_repair(raw) or {} + item = dict(item) + item["judge_score"] = parsed.get("judge_score") + item["remarks"] = parsed.get("remarks") + item["judge_method"] = "llm" + return item + + out = dict(aligned) + for key in ("entities", "key_terms"): + items = out.get(key) or [] + if not items: + continue + with cf.ThreadPoolExecutor(max_workers=max_workers) as ex: + out[key] = list(ex.map(one, items)) + return out + + +# --------------------------------------------------------------------------- +# Driver +# --------------------------------------------------------------------------- + +def run(text: str, *, task: str, extractor_model: str, + mapper_backend: Optional[str], judge_model: Optional[str], + chunk_size: int = 2000, max_workers: int = 8, + skip_judge: bool = False, + local_mapping_url: str = "http://localhost:8000", + ask_user: Optional[Callable[[str], Optional[str]]] = None, + input_path: Optional[str] = None, + ner_ensemble_profile: Optional[str] = None, + ner_ensemble_models: Optional[list[str]] = None, + ner_ensemble_device: int = -1, + allow_ols_fallback: bool = False) -> dict: + """Full pipeline: ensemble NER + LLM extract -> align -> judge. + + Set ``ner_ensemble_profile`` (e.g. ``"biomedical_broad"`` or ``"cns_cells"``) + or pass an explicit ``ner_ensemble_models`` list to run HuggingFace NER + models alongside the LLM extractor. Each mention carries a ``source_model`` + field; the grouped view records which models contributed to each entity. + + Always embeds a ``stats`` block in the result. See scripts/stats.py + for the shape. + """ + started = time.monotonic() + timings: dict[str, float] = {} + alignment_meta: dict = {} + judge_meta: dict = {} + ensemble_meta: list[dict] = [] + + # --- 0. (optional) HuggingFace NER ensemble — runs in parallel with LLM --- + ensemble_items: list[dict] = [] + if (ner_ensemble_profile or ner_ensemble_models) and task == "ner": + t0 = time.monotonic() + cfg = EnsembleConfig( + hf_models=ner_ensemble_models, + profile=ner_ensemble_profile, + device=ner_ensemble_device, + max_workers=max_workers, + ) + ensemble_items, ensemble_meta = run_ensemble(text, cfg) + timings["ensemble_ner"] = round(time.monotonic() - t0, 2) + logger.info("ensemble produced %d mentions across %d model(s)", + len(ensemble_items), + sum(1 for m in ensemble_meta if not m.get("skipped_reason"))) + + # --- 1. LLM extraction --- + t0 = time.monotonic() + extraction = extract(text, model=extractor_model, task=task, + chunk_size=chunk_size, max_workers=max_workers) + timings["extraction"] = round(time.monotonic() - t0, 2) + + # --- 1b. merge LLM + ensemble (NER only) --- + if ensemble_items and "entities" in extraction: + extraction["entities"] = annotate_llm_provenance( + extraction["entities"], llm_model=extractor_model) + extraction["entities"] = merge_ensemble_and_llm( + ensemble_items, extraction["entities"]) + elif "entities" in extraction: + # No ensemble: still tag LLM provenance for consistency. + extraction["entities"] = annotate_llm_provenance( + extraction["entities"], llm_model=extractor_model) + + # --- 2. alignment with mapper cascade --- + if mapper_backend: + t0 = time.monotonic() + try: + mapper, alignment_meta = build_mapper_with_cascade( + preferred=mapper_backend, + local_url=local_mapping_url, + ask_user=ask_user, + allow_ols_fallback=allow_ols_fallback, + ) + aligned = align_direct(extraction, mapper=mapper) + except RuntimeError as e: + logger.error("alignment cascade exhausted: %s — skipping alignment", e) + aligned = extraction + for key in ("entities", "key_terms"): + for it in aligned.get(key, []) or []: + it.setdefault("concept_mapping_provenance", "skipped") + it.setdefault("alignment_method", "skipped") + alignment_meta = {"mapper_used": None, + "mapper_url": None, + "fallback_triggered": True, + "cascade_history": ["all_failed"], + "user_provided_url": False} + timings["alignment"] = round(time.monotonic() - t0, 2) + else: + aligned = extraction + for key in ("entities", "key_terms"): + for it in aligned.get(key, []) or []: + it.setdefault("concept_mapping_provenance", "skipped") + it.setdefault("alignment_method", "skipped") + alignment_meta = {"mapper_used": None, "mapper_url": None, + "fallback_triggered": False, "cascade_history": [], + "user_provided_url": False} + timings["alignment"] = 0.0 + + # --- 3. judge --- + t0 = time.monotonic() + if judge_model and not skip_judge: + judged = judge(aligned, text, model=judge_model, max_workers=max_workers) + judge_meta = {"method": "llm", "model": judge_model, "max_workers": max_workers} + else: + judged = judge(aligned, text, model="", auto_approve=True) + judge_meta = {"method": "auto_approved", "model": None} + timings["judge"] = round(time.monotonic() - t0, 2) + + timings["total"] = round(time.monotonic() - started, 2) + judged["elapsed_time"] = timings["total"] + + # --- 3b. NORMALIZE TO CANONICAL SHAPE (safety net) --- + # Even when the LLM ignores the new prompt and emits per-entity + # paper_title / doi, this lifts them to top-level source_metadata, + # strips them off entities, and tags missing source_model so the + # downstream grouped view + stats are consistent regardless of what + # the LLM produced. + lift_doc_metadata(judged, input_path=input_path) + tag_missing_source_model(judged, extractor_model) + + # --- 4. attach grouped-by-entity view (entities_grouped + key_terms_grouped) --- + # Raw mentions stay as the authoritative record; the grouped view exists + # so downstream consumers can navigate by canonical entity ("everything + # about BDNF in this paper") and so multi-sentence context naturally + # emerges from merging sentences across all mentions. + attach_grouped_views(judged) + + # --- 5. stats (must run AFTER grouping so it can include unique counts) --- + if ensemble_meta: + judged["ensemble_models"] = ensemble_meta + judged["stats"] = compute_stats( + judged, + timings=timings, + input_meta={"char_count": len(text), + "chunk_count": _chunk_count(text, chunk_size), + "chunk_size_chars": chunk_size, + "input_path": input_path}, + alignment_meta=alignment_meta, + judge_meta=judge_meta, + ) + return judged + + +def _chunk_count(text: str, chunk_size: int) -> int: + if chunk_size <= 0 or len(text) <= chunk_size: + return 1 + return len(chunk_by_sentences(text, max_chars=chunk_size)) + + +def default_output_path(input_path: Optional[str], explicit_out: Optional[str]) -> str: + """Pick the output filename. + + - If ``explicit_out`` is given, use it verbatim. + - Else if ``input_path`` ends in a known text extension, replace the + extension with ``_final.json`` (e.g. ``paper.txt`` -> ``paper_final.json``, + ``paper.pdf`` -> ``paper_final.json``). + - Else fall back to ``result_final.json``. + """ + if explicit_out: + return explicit_out + if input_path: + p = Path(input_path) + return str(p.with_name(p.stem + "_final.json")) + return "result_final.json" + + +def _main(): + ap = argparse.ArgumentParser() + ap.add_argument("--task", choices=["ner", "resource", "structured"], default="ner") + ap.add_argument("--input", required=True, help="path to .txt input") + ap.add_argument("--extractor", required=True, help="extractor model string") + ap.add_argument("--judge", default=None, help="judge model string (omit to auto-approve)") + ap.add_argument("--mapper", choices=["ols", "bioportal", "local", "none"], default="local", + help="preferred mapping backend. 'local' is the default cascade: " + "local hybrid (http://localhost:8000) → BioPortal → interactive " + "prompt for alternative URL → HARD STOP. OLS is NOT in the " + "default cascade (no gene coverage); pass --mapper ols to use " + "it explicitly, or --allow-ols-fallback to add it as a last " + "resort. 'none' explicitly opts out of mapping (items get " + "concept_mapping_provenance='skipped').") + ap.add_argument("--allow-ols-fallback", action="store_true", + help="Allow OLS as a last-resort fallback when local + BioPortal " + "both fail. Off by default because OLS lacks gene coverage.") + ap.add_argument("--mapper-url", default="http://localhost:8000", + help="URL for the local hybrid mapping service. Overrides " + "LOCAL_CONCEPT_MAPPING_URL env var. /docs is a good " + "endpoint to verify the service is up.") + ap.add_argument("--chunk-size", type=int, default=2000) + ap.add_argument("--max-workers", type=int, default=8) + ap.add_argument("--out", default=None, + help="Output JSON path. Defaults to <input-stem>_final.json.") + ap.add_argument("--non-interactive", action="store_true", + help="Do not prompt for an alternative mapper URL when " + "the cascade fails. Fail fast instead.") + ap.add_argument("--ner-profile", default=None, + choices=["biomedical_broad", "cns_cells", "pharmacology", + "genetic", "clinical", "minimal", "all"], + help="Enable the HuggingFace NER ensemble alongside the LLM " + "extractor. Picks a domain-appropriate set of models. " + "Each mention gets a `source_model` field; the grouped " + "view records which models surfaced each entity. " + "Requires `pip install transformers torch`.") + ap.add_argument("--ner-models", default=None, + help="Comma-separated list of HF model IDs for an explicit " + "ensemble. Overrides --ner-profile.") + ap.add_argument("--ner-device", type=int, default=-1, + help="CUDA device index for HF NER models (-1 = CPU). " + "Default -1.") + args = ap.parse_args() + + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(levelname)s %(message)s") + text = Path(args.input).read_text() + mapper = None if args.mapper == "none" else args.mapper + + ner_models = ( + [m.strip() for m in args.ner_models.split(",") if m.strip()] + if args.ner_models else None + ) + + result = run( + text, task=args.task, + extractor_model=args.extractor, + mapper_backend=mapper, + judge_model=args.judge, + chunk_size=args.chunk_size, + max_workers=args.max_workers, + skip_judge=args.judge is None, + local_mapping_url=args.mapper_url, + ask_user=None if args.non_interactive else _stdin_ask, + input_path=args.input, + ner_ensemble_profile=args.ner_profile, + ner_ensemble_models=ner_models, + ner_ensemble_device=args.ner_device, + allow_ols_fallback=args.allow_ols_fallback, + ) + + out_path = default_output_path(args.input, args.out) + Path(out_path).write_text(json.dumps(result, indent=2, default=str)) + print(format_summary(result["stats"]), file=sys.stderr) + print(f"wrote {out_path}", file=sys.stderr) + + +if __name__ == "__main__": + _main() diff --git a/skills/structsense-skills/scripts/span_validator.py b/skills/structsense-skills/scripts/span_validator.py new file mode 100644 index 0000000..9fc3796 --- /dev/null +++ b/skills/structsense-skills/scripts/span_validator.py @@ -0,0 +1,87 @@ +"""Validate NER spans against the source text. + +After extraction (and especially after merging chunk results), every entity's +start/end should satisfy: text[start:end] == entity. This module checks that, +and offers a best-effort repair that re-finds the entity in its sentence. +""" +from __future__ import annotations + +from typing import Iterable, Tuple + + +def validate(text: str, item: dict) -> bool: + """True if start/end correctly anchor `entity` (or `term`) in `text`.""" + surface = item.get("entity") or item.get("term") + start, end = item.get("start"), item.get("end") + sentence = item.get("sentence", "") + if surface is None or start is None or end is None: + return False + if text[start:end] != surface: + return False + if sentence and sentence not in text: + return False + if sentence and surface not in sentence: + return False + return True + + +def repair_span(text: str, item: dict) -> dict | None: + """Try to re-locate `entity` in the source text. + + Strategy: search for `entity` inside `sentence`, then for `sentence` inside + `text`. Combine offsets. Returns a repaired item, or None if not found. + """ + surface = item.get("entity") or item.get("term") + sentence = item.get("sentence", "") + if not surface: + return None + + if sentence and sentence in text and surface in sentence: + sent_offset = text.index(sentence) + local = sentence.index(surface) + new = dict(item) + new["start"] = sent_offset + local + new["end"] = new["start"] + len(surface) + return new + + # No sentence — fall back to first occurrence in text. + if surface in text: + new = dict(item) + new["start"] = text.index(surface) + new["end"] = new["start"] + len(surface) + return new + + return None + + +def validate_all(text: str, items: Iterable[dict]) -> Tuple[list[dict], list[dict]]: + """Partition items into (valid, dropped). + + Repaired items are placed in `valid`. Items that can't be repaired are + in `dropped` for the caller to log. + """ + valid, dropped = [], [] + for it in items: + if validate(text, it): + valid.append(it) + continue + repaired = repair_span(text, it) + if repaired is not None and validate(text, repaired): + valid.append(repaired) + else: + dropped.append(it) + return valid, dropped + + +if __name__ == "__main__": + text = "We recorded from CA1 pyramidal cells in the hippocampus." + good = { + "entity": "hippocampus", + "sentence": text, + "start": text.index("hippocampus"), + "end": text.index("hippocampus") + len("hippocampus"), + } + bad = {"entity": "hippocampus", "sentence": text, "start": 0, "end": 5} + print("good ok?", validate(text, good)) + print("bad ok?", validate(text, bad)) + print("repaired:", repair_span(text, bad)) diff --git a/skills/structsense-skills/scripts/stats.py b/skills/structsense-skills/scripts/stats.py new file mode 100644 index 0000000..f6a210e --- /dev/null +++ b/skills/structsense-skills/scripts/stats.py @@ -0,0 +1,281 @@ +"""Compute a stats block for a pipeline result. + +Pure function — no LLM calls. Embed the returned dict under `result["stats"]` +and print the human summary to stderr. + +The stats answer the question "did the run do what it was supposed to?": + +- total mentions emitted (NER yield) +- mentions per unique surface form (sanity check for exhaustive extraction; + should be > 1 on most papers — if = 1, you're likely deduplicating by + surface form somewhere upstream) +- alignment provenance breakdown (tool / llm_knowledge / unmapped / skipped) +- judge score distribution +- per-stage elapsed times +- which mapper backend was used (for debugging the cascade) +""" +from __future__ import annotations + +from collections import Counter +from typing import Any, Iterable, Mapping, Optional + + +def _iter_items(result: Mapping[str, Any]) -> Iterable[tuple[str, dict]]: + """Yield (container_key, item) for every extracted item across all + container shapes (NER, key_terms, resource).""" + for key in ("entities", "key_terms"): + for it in result.get(key, []) or []: + yield key, it + res = result.get("extracted_resources") or result.get("aligned_resources") \ + or result.get("judge_resource") or {} + if isinstance(res, dict): + for bucket, items in res.items(): + for it in items or []: + yield "resources", it + + +def _bucket_score(s: Optional[float]) -> str: + if s is None: + return "missing" + s = float(s) + if s >= 1.0: + return "1.00" + if s >= 0.85: + return "0.85-0.99" + if s >= 0.65: + return "0.65-0.84" + if s >= 0.40: + return "0.40-0.64" + return "<0.40" + + +def compute_stats( + result: Mapping[str, Any], + *, + timings: Optional[Mapping[str, float]] = None, + input_meta: Optional[Mapping[str, Any]] = None, + alignment_meta: Optional[Mapping[str, Any]] = None, + judge_meta: Optional[Mapping[str, Any]] = None, + dropped_invalid_span: int = 0, +) -> dict: + """Build a `stats` dict from a pipeline result + sidecar metadata. + + Args: + result: the full pipeline result (extractor + alignment + judge merged). + timings: {"extraction": float, "mask_recall": float, "alignment": float, + "judge": float, "total": float}. All optional. + input_meta: {"char_count": int, "chunk_count": int, + "chunk_size_chars": int, "input_path": str?}. + alignment_meta: {"mapper_used": str, "mapper_url": str, + "fallback_triggered": bool, "cascade_history": [str, …]}. + judge_meta: {"method": "llm" | "auto_approved", "model": str?, + "max_workers": int?}. + dropped_invalid_span: number of items dropped during span validation. + """ + # ---- entity / key_term counts ---- + entities = list(result.get("entities") or []) + key_terms = list(result.get("key_terms") or []) + + ent_labels = Counter((it.get("label") or "Unknown") for it in entities) + ent_sources = Counter((it.get("source_model") or "unknown") for it in entities) + + # Prefer the grouped view's count when it's present (case-insensitive + label-aware). + # Fall back to a simple lowercase set of surface forms if grouping wasn't run. + grouped = result.get("entities_grouped") or [] + if grouped: + ent_unique = len(grouped) + else: + ent_unique = len({(it.get("entity") or "").lower() for it in entities + if it.get("entity")}) + + kt_grouped = result.get("key_terms_grouped") or [] + if kt_grouped: + kt_unique = len(kt_grouped) + else: + kt_unique = len({(it.get("term") or "").lower() for it in key_terms + if it.get("term")}) + + # ---- resource counts (if present) ---- + resources = [] + res_container = result.get("extracted_resources") or result.get("aligned_resources") \ + or result.get("judge_resource") or {} + if isinstance(res_container, dict): + for items in res_container.values(): + resources.extend(items or []) + + # ---- alignment provenance ---- + prov = Counter() + method = Counter() + for _, it in _iter_items(result): + prov[it.get("concept_mapping_provenance") or "missing"] += 1 + method[it.get("alignment_method") or "missing"] += 1 + + # ---- judge score distribution ---- + score_buckets = Counter() + for _, it in _iter_items(result): + score_buckets[_bucket_score(it.get("judge_score"))] += 1 + + # ---- task_type: prefer the result, otherwise infer from what's present ---- + task_type = result.get("task_type") + if not task_type: + if entities or key_terms: + task_type = "ner" + elif resources: + task_type = "resource" + elif result.get("activity") or result.get("items"): + task_type = "structured_extraction" + + # ---- prominent totals at the top of the stats block ---- + total_items = (len(entities) + len(key_terms) + len(resources)) + + # ---- assemble ---- + stats: dict[str, Any] = { + "task_type": task_type, + "totals": { + "total_entity_mentions": len(entities), + "total_key_term_mentions": len(key_terms), + "total_resources": len(resources), + "total_items": total_items, + "unique_entities": ent_unique, + "unique_key_terms": kt_unique, + }, + "elapsed_seconds": dict(timings or {}), + "input": dict(input_meta or {}), + "entities": { + "total_mentions": len(entities), + "unique_surface_forms": ent_unique, + "mentions_per_unique": round(len(entities) / ent_unique, 2) if ent_unique else 0, + "by_label": dict(ent_labels.most_common()), + "by_source_model": dict(ent_sources.most_common()), + "dropped_invalid_span": dropped_invalid_span, + }, + "key_terms": { + "total_mentions": len(key_terms), + "unique_surface_forms": kt_unique, + }, + "resources": { + "total": len(resources), + }, + "alignment": { + "by_provenance": dict(prov.most_common()), + "by_method": dict(method.most_common()), + **dict(alignment_meta or {}), + }, + "judge": { + **dict(judge_meta or {}), + "score_buckets": dict(sorted(score_buckets.items(), + key=lambda kv: ("1.00", "0.85-0.99", + "0.65-0.84", "0.40-0.64", "<0.40", + "missing").index(kv[0]) if kv[0] in + ("1.00","0.85-0.99","0.65-0.84","0.40-0.64","<0.40","missing") + else 99)), + }, + } + return stats + + +def format_summary(stats: Mapping[str, Any]) -> str: + """Human-readable one-screen summary. Print this to stderr after a run.""" + lines = [] + ent = stats.get("entities", {}) + kt = stats.get("key_terms", {}) + res = stats.get("resources", {}) + al = stats.get("alignment", {}) + jud = stats.get("judge", {}) + inp = stats.get("input", {}) + t = stats.get("elapsed_seconds", {}) + + lines.append("================ STRUCTSENSE-SKILLS RUN SUMMARY ================") + if stats.get("task_type"): + lines.append(f"task_type: {stats['task_type']}") + totals = stats.get("totals") or {} + if totals: + lines.append( + f"TOTAL ITEMS: {totals.get('total_items', 0):,} " + f"(entities={totals.get('total_entity_mentions',0):,} " + f"key_terms={totals.get('total_key_term_mentions',0):,} " + f"resources={totals.get('total_resources',0):,})" + ) + lines.append( + f"UNIQUE: entities={totals.get('unique_entities',0):,} " + f"key_terms={totals.get('unique_key_terms',0):,}" + ) + if inp: + lines.append(f"input: {inp.get('input_path','<text>')} " + f"({inp.get('char_count','?')} chars, " + f"{inp.get('chunk_count','?')} chunks of " + f"{inp.get('chunk_size_chars','?')} chars)") + if t: + lines.append(f"elapsed: total={t.get('total','?')}s " + f"extract={t.get('extraction','?')}s " + f"mask_recall={t.get('mask_recall','?')}s " + f"align={t.get('alignment','?')}s " + f"judge={t.get('judge','?')}s") + lines.append("") + lines.append(f"ENTITIES: {ent.get('total_mentions',0)} mentions, " + f"{ent.get('unique_surface_forms',0)} unique " + f"({ent.get('mentions_per_unique',0):.2f}x avg) " + f"[dropped: {ent.get('dropped_invalid_span',0)}]") + by_label = ent.get("by_label") or {} + if by_label: + head = list(by_label.items())[:8] + rest = sum(v for _, v in list(by_label.items())[8:]) + line = " by_label: " + ", ".join(f"{k}={v}" for k, v in head) + if rest: + line += f", …+{rest} more" + lines.append(line) + by_src = ent.get("by_source_model") or {} + if by_src: + head = list(by_src.items())[:8] + rest = sum(v for _, v in list(by_src.items())[8:]) + line = " by_source_model: " + ", ".join(f"{k}={v}" for k, v in head) + if rest: + line += f", …+{rest} more" + lines.append(line) + lines.append(f"KEY_TERMS: {kt.get('total_mentions',0)} mentions, " + f"{kt.get('unique_surface_forms',0)} unique") + if res.get("total"): + lines.append(f"RESOURCES: {res['total']}") + lines.append("") + lines.append(f"ALIGNMENT: " + f"mapper={al.get('mapper_used','?')} " + f"url={al.get('mapper_url','?')} " + f"fallback={al.get('fallback_triggered', False)}") + if al.get("by_provenance"): + lines.append(f" by_provenance: " + ", ".join(f"{k}={v}" for k, v in al["by_provenance"].items())) + lines.append("") + lines.append(f"JUDGE: method={jud.get('method','?')} " + f"model={jud.get('model','?')}") + if jud.get("score_buckets"): + lines.append(f" score_buckets: " + ", ".join(f"{k}={v}" for k, v in jud["score_buckets"].items())) + lines.append("================================================================") + return "\n".join(lines) + + +if __name__ == "__main__": + demo_result = { + "task_type": "ner", + "entities": [ + {"entity": "BDNF", "label": "Gene", "judge_score": 1.0, + "concept_mapping_provenance": "tool", "alignment_method": "direct_tool_call"}, + {"entity": "BDNF", "label": "Gene", "judge_score": 1.0, + "concept_mapping_provenance": "tool", "alignment_method": "direct_tool_call"}, + {"entity": "hippocampus", "label": "BrainRegion", "judge_score": 0.9, + "concept_mapping_provenance": "tool", "alignment_method": "direct_tool_call"}, + {"entity": "fast-spiking", "label": "Phenomenon", "judge_score": 0.4, + "concept_mapping_provenance": "unmapped", "alignment_method": "skipped"}, + ], + "key_terms": [], + } + stats = compute_stats( + demo_result, + timings={"extraction": 5.2, "mask_recall": 2.1, "alignment": 1.4, + "judge": 3.0, "total": 11.7}, + input_meta={"char_count": 12340, "chunk_count": 6, "chunk_size_chars": 2000, + "input_path": "paper.txt"}, + alignment_meta={"mapper_used": "local_hybrid", + "mapper_url": "http://localhost:8000", + "fallback_triggered": False}, + judge_meta={"method": "llm", "model": "openrouter/openai/gpt-4o-mini"}, + ) + print(format_summary(stats)) diff --git a/skills/structsense-skills/scripts/task_detection.py b/skills/structsense-skills/scripts/task_detection.py new file mode 100644 index 0000000..b0d50d7 --- /dev/null +++ b/skills/structsense-skills/scripts/task_detection.py @@ -0,0 +1,289 @@ +"""LLM-based task-type detection from a task description. + +Given a free-text extractor description (e.g. the `description:` field of +a CrewAI/structsense task config, or the user's natural-language ask), this +returns a canonical task type the pipeline can route on: + + ner | resource | structured_extraction | keyphrase_extraction | + relation_extraction | event_extraction | classification | … + +Adapted from structsense `task_detection.py` but uses the skill's +provider-agnostic ``llm_client.call`` instead of importing the OpenAI SDK +directly, so it works with Claude / OpenRouter / Ollama / Gemini. + +When the user is in Claude Code (or another runtime where Claude is the +caller), the LLM can simply *read* the taxonomy below and pick the right +type from the conversation — no extra round-trip needed. This module is +useful when the pipeline driver needs to *programmatically* pick the type +(e.g. inside `pipeline.run()` when invoked from a Python script). +""" +from __future__ import annotations + +import json +import logging +import re +from dataclasses import dataclass, asdict, field +from typing import Any, Dict, List, Optional + +logger = logging.getLogger("task_detection") + + +# --------------------------------------------------------------------------- +# Canonical taxonomy — keep aligned with structsense for cross-tool consistency. +# --------------------------------------------------------------------------- + +DEFAULT_TAXONOMY: Dict[str, str] = { + "ner": ( + "Named Entity Recognition: detect and extract spans that refer to " + "real-world or domain-specific entities. Common types: PERSON, ORG, " + "LOCATION, DATE, PRODUCT, EVENT, DISEASE, DRUG, GENE, PROTEIN, " + "CELL_TYPE, BRAIN_REGION, NEUROTRANSMITTER, ASSAY, DATASET, etc. " + "Output is entity spans with type labels." + ), + "keyphrase_extraction": ( + "Extract a concise set of important phrases that capture the main " + "topics, mechanisms, or claims. Broader than NER. Output is a " + "ranked list of key terms." + ), + "resource": ( + "Extract one primary research resource (Model, Dataset, Tool, " + "Benchmark, Leaderboard, Paper) plus its mentions of secondary " + "resources. Output is a resource object with name, description, " + "type, category, target, url, mentions." + ), + "structured_extraction": ( + "Extract information into a predefined JSON schema with strict " + "fields and types. Use when the user supplies a target schema " + "(e.g. ReproSchema, Croissant, schema.org Dataset, a custom schema)." + ), + "relation_extraction": ( + "Extract typed relationships between entities (often from NER " + "output). Output is (subject, relation, object) triples." + ), + "event_extraction": ( + "Extract events and their arguments/roles (who did what to whom, " + "when, where, how)." + ), + "classification": ( + "Assign exactly one category/label to an input (single-label)." + ), + "multi_label_classification": ( + "Assign multiple applicable labels/tags to the same input " + "(not mutually exclusive)." + ), + "summarization": ( + "Produce a shorter version that preserves key information." + ), + "question_answering": ( + "Answer a question directly (extractive or abstractive)." + ), + "other": ( + "None of the above; fallback when the user's request doesn't " + "fit a canonical type." + ), +} + + +# Map canonical task types → which extractor prompt to load. +TASK_TYPE_TO_PROMPT: Dict[str, str] = { + "ner": "prompts/extractor-ner-neuroscience.md", # default; override per-domain + "keyphrase_extraction": "prompts/extractor-ner-neuroscience.md", + "resource": "prompts/extractor-resource.md", + "structured_extraction": "prompts/extractor-structured.md", + "relation_extraction": "prompts/extractor-structured.md", + "event_extraction": "prompts/extractor-structured.md", +} + + +@dataclass +class TaskDetection: + task_type: str # canonical type + confidence: float # 0.0 – 1.0 + labels: List[str] = field(default_factory=list) # optional sublabels + rationale: str = "" + raw: Dict[str, Any] = field(default_factory=dict) + + def asdict(self) -> dict: + return asdict(self) + + +# --------------------------------------------------------------------------- +# Heuristic detection (no LLM call) — fast first-pass +# --------------------------------------------------------------------------- + +_NER_PATTERNS = re.compile( + r"\b(named[- ]entity|\bNER\b|entities? and key[- ]terms?|" + r"\bextract entities|extract spans?|extract mentions?)", + re.IGNORECASE, +) +_KEYPHRASE_PATTERNS = re.compile( + r"\b(key[- ]?phrases?|key[- ]?terms?|main topics|important phrases)\b", + re.IGNORECASE, +) +_RESOURCE_PATTERNS = re.compile( + r"\b(resources?\b|" + r"(tools?|datasets?|models?|benchmarks?)([,\s]+(and\s+)?(tools?|datasets?|models?|benchmarks?))+|" + r"model card|catalog of (tools|datasets|models|benchmarks))", + re.IGNORECASE, +) +_STRUCTURED_PATTERNS = re.compile( + r"\b(JSON[- ]?[Ss]chema|structured (extraction|output)|reproschema|" + r"croissant|target schema)", + re.IGNORECASE, +) +_RELATION_PATTERNS = re.compile( + r"\b(relation(ship)?s?|triples?|\(subject,? *relation,? *object\))", + re.IGNORECASE, +) + + +def detect_heuristic(text: str) -> Optional[TaskDetection]: + """Quickly pick a task type from common phrasing patterns, no LLM call. + + Returns None when no pattern matches confidently — call `detect_with_llm` + next in that case. + """ + if _STRUCTURED_PATTERNS.search(text): + return TaskDetection("structured_extraction", 0.85, + ["heuristic"], "matched schema/structured pattern") + if _RESOURCE_PATTERNS.search(text): + return TaskDetection("resource", 0.8, + ["heuristic"], "matched resource-extraction pattern") + if _RELATION_PATTERNS.search(text): + return TaskDetection("relation_extraction", 0.75, + ["heuristic"], "matched relation-extraction pattern") + if _NER_PATTERNS.search(text): + return TaskDetection("ner", 0.85, + ["heuristic"], "matched NER pattern") + if _KEYPHRASE_PATTERNS.search(text): + return TaskDetection("keyphrase_extraction", 0.7, + ["heuristic"], "matched keyphrase pattern") + return None + + +# --------------------------------------------------------------------------- +# LLM-based detection (provider-agnostic via llm_client) +# --------------------------------------------------------------------------- + +_DETECT_PROMPT_SYSTEM = ( + "You classify a task description into ONE canonical task type from a " + "fixed taxonomy. Output strict JSON, no prose, no markdown. Schema:\n" + '{ "task_type": "<one of the listed types>",\n' + ' "confidence": <float 0-1>,\n' + ' "labels": ["<optional sublabels, e.g. multi_label, few_shot>"],\n' + ' "rationale": "<one short sentence>" }\n\n' + "If nothing fits, return task_type=\"other\" with confidence ≤ 0.3." +) + + +def _user_prompt(task_description: str, taxonomy: Dict[str, str]) -> str: + lines = ["TAXONOMY (pick ONE task_type from these keys):"] + for k, v in taxonomy.items(): + lines.append(f"- {k}: {v}") + lines.append("") + lines.append("TASK DESCRIPTION TO CLASSIFY:") + lines.append(task_description) + return "\n".join(lines) + + +def detect_with_llm( + task_description: str, + *, + model: str, + taxonomy: Optional[Dict[str, str]] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, +) -> TaskDetection: + """Run an LLM round-trip to classify the task type. Falls back to 'other' + on parse error. Uses the provider-agnostic `llm_client.call`. + """ + try: + from llm_client import call as llm_call # local import for cycle safety + except ImportError: + import sys + from pathlib import Path + sys.path.insert(0, str(Path(__file__).resolve().parent)) + from llm_client import call as llm_call # type: ignore + + tax = taxonomy or DEFAULT_TAXONOMY + user = _user_prompt(task_description, tax) + + raw = llm_call( + model=model, + system=_DETECT_PROMPT_SYSTEM, + user=user, + temperature=0, + json_mode=True, + api_key=api_key, + base_url=base_url, + ) + + try: + data = json.loads(raw) + except json.JSONDecodeError: + # try the repair helper + try: + from json_repair import parse_or_repair + except ImportError: + parse_or_repair = None + data = parse_or_repair(raw) if parse_or_repair else None + if data is None: + logger.warning("task_detection: LLM output not parseable: %r", raw[:200]) + return TaskDetection("other", 0.0, ["parse_error"], + "LLM output not parseable", {"llm_text": raw}) + + task_type = str(data.get("task_type", "other")).strip().lower() + if task_type not in tax: + logger.warning("task_detection: out-of-taxonomy %r → other", task_type) + return TaskDetection("other", min(float(data.get("confidence", 0)), 0.3), + ["out_of_taxonomy"], str(data.get("rationale", "")), data) + + confidence = max(0.0, min(1.0, float(data.get("confidence", 0.0)))) + labels = data.get("labels") or [] + if not isinstance(labels, list): + labels = [str(labels)] + rationale = str(data.get("rationale", "")).strip() + + return TaskDetection(task_type, confidence, [str(x) for x in labels], + rationale, data) + + +def detect( + task_description: str, + *, + model: Optional[str] = None, + taxonomy: Optional[Dict[str, str]] = None, + api_key: Optional[str] = None, + base_url: Optional[str] = None, +) -> TaskDetection: + """High-level entry point: try heuristic first, fall back to LLM. + + Set ``model=None`` to disable the LLM fallback — useful when you only + want the cheap heuristic. + """ + hit = detect_heuristic(task_description) + if hit is not None and hit.confidence >= 0.75: + return hit + if model is None: + # No LLM available — return whatever the heuristic gave (even low-confidence), + # or 'other'. + return hit or TaskDetection("other", 0.0, ["no_llm"], + "No heuristic match and LLM disabled") + return detect_with_llm(task_description, model=model, taxonomy=taxonomy, + api_key=api_key, base_url=base_url) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + import argparse, sys + ap = argparse.ArgumentParser() + ap.add_argument("description", help="Task description / extractor agent prompt text") + ap.add_argument("--model", default=None, + help="LLM model for fallback (e.g. openrouter/openai/gpt-4o-mini). " + "Omit to use only the heuristic.") + args = ap.parse_args() + result = detect(args.description, model=args.model) + print(json.dumps(result.asdict(), indent=2))