Merge pull request #115 from djarecka/pre-commit_fix_new

Pre commit fix

Author: tekrajchhetri

.env_example

@@ -10,4 +10,4 @@ CREWAI_DISABLE_TRACING=true
 CREWAI_TELEMETRY=false
 OTEL_SDK_DISABLED=true
 ENABLE_HUMAN_FEEDBACK=false
-ENABLE_CREW_MEMORY=false
+ENABLE_CREW_MEMORY=false

.github/workflows/test.yaml

@@ -134,5 +134,5 @@ jobs:
       shell: bash
     - name: Run pre-commit
       run: |
-        poetry run pre-commit run --all-files
+        poetry run pre-commit run --all-files --show-diff-on-failure
       shell: bash

.pre-commit-config.yaml

@@ -1,5 +1,8 @@
 fail_fast: false
 
+_exclude_default: &exclude_default \.(pdf|svg|png|jpg|jpeg|zip|json)$|^evaluation/(?!.*\.(md|yaml|py)$).*$
+_exclude_codespell: &exclude_codespell \.(pdf|svg|png|jpg|jpeg|zip|json)$|^evaluation/(?!.*\.(md|yaml)$).*$
+
 repos:
 - repo: https://github.com/python-poetry/poetry
   rev: 2.1.3
@@ -12,19 +15,11 @@ repos:
   - id: ruff
   - id: ruff-format
 
-- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-  rev: v2.13.0
+- repo: https://github.com/ComPWA/taplo-pre-commit
+  rev: v0.9.3
   hooks:
-  - id: pretty-format-yaml
-    args:
-    - --autofix
-    - --indent=2
-  - id: pretty-format-toml
+  - id: taplo-format
     exclude: ^poetry.lock$
-    args:
-    - --autofix
-    - --indent=2
-    - --no-sort
 
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v4.6.0
@@ -33,15 +28,19 @@ repos:
     args: [--maxkb=15000]
   - id: check-case-conflict
   - id: end-of-file-fixer
+    exclude: *exclude_default
   - id: trailing-whitespace
-  - id: pretty-format-json
-    args:
-    - --autofix
-    - --indent=4
-    - --no-sort-keys
+    exclude: *exclude_default
+  # removing checks for json since they are mostly results (we can think about it later)
+#  - id: pretty-format-json
+#    args:
+#    - --autofix
+#    - --indent=4
+#    - --no-sort-keys
   - id: check-merge-conflict
   - id: check-yaml
-  - id: check-json
+    # removing checks for json since they are mostly results (we can think about it later)
+#  - id: check-json
   - id: check-toml
   - id: detect-private-key
   - id: name-tests-test
@@ -50,8 +49,9 @@ repos:
   rev: v2.3.0
   hooks:
   - id: codespell
+    exclude: *exclude_codespell
     args:
-    - --skip=poetry.lock,docs_style/pdoc-theme/syntax-highlighting.css,*.cha,*.ipynb,example/sample_metadata.csv,evaluation/benchmark,evaluation/pdf2reproschema/old,evaluation/ner/old,evaluation/pdf2reproschema/old/evaluation
+    - --skip=poetry.lock,docs_style/pdoc-theme/syntax-highlighting.css,*.cha
     - --ignore-words-list=SIE,sie
 
 - repo: https://github.com/hija/clean-dotenv

CHANGE_LOG.md

@@ -8,7 +8,7 @@ The current version implements the following specialized tool (see figure) for N
 ![](docs/design_docs/ner_tool.png)
 ### 2. Implements the specialized concept mapping tool, `Concept Mapping tool`, avoiding the need for manually maintaining the ontology database.
 - The concept mapping tool implements features such as throttling (see default values below), i.e., minimum seconds between BioPortal requests (throttle; lower = faster, higher 429 risk). This done to take into account the rate limit of the bioportal.
-    
+
     | Variable | Where used | Default | Description |
     |----------|------------|---------|-------------|
     | `BIOPORTAL_API_KEY` | `conceptmappingtool.py` | — | **Required** for ConceptMappingTool. Used in Authorization header. |
@@ -17,7 +17,7 @@ The current version implements the following specialized tool (see figure) for N
     | `MAX_CONCEPT_MAPPING_RESULTS` | `conceptmappingtool.py` | `1` | Top-N results per concept (1–50). |
     | `CONCEPT_MAPPING_CACHE_SIZE` | `conceptmappingtool.py` | `2000` | Max in-memory cache entries for concept mapping. |
     | `CONCEPT_MAPPING_MAX_TERMS` | `postprocessing.py` | — | Optional cap on how many unique terms are mapped in batch (rest get null). |
-    
+
 - To avoid the continuous API call, it implements the inmemory caching (to be replaced with new caching mechanism), i.e., `_CONCEPT_MAPPING_CACHE` (dict). The cache size is default `2000` set with `CONCEPT_MAPPING_CACHE_SIZE`. The FIFO-style cache eviction is applied, i.e., when the cache is full, one oldest entry is removed before inserting a new one. The lookup is done using `_map_single_concept()` before making an API call.
 ### 3. Implements automatic task detection
 This version of **`StructSense`** introduces automatic task detection to support accurate task-to-tool mapping, enabling agents to invoke the most relevant tools for a given task.
@@ -30,7 +30,7 @@ The implementation combines an **LLM-based classifier** with a **heuristic fallb
 | `"structured extraction"` or `"structured_extraction"` | `structured_extraction` |
 | Otherwise | `extraction` |
 
-Task detection is executed **once per stage** at the point where tools and post-processing are selected (e.g., `information_extraction_task`).  
+Task detection is executed **once per stage** at the point where tools and post-processing are selected (e.g., `information_extraction_task`).
 In the full pipeline, the **first stage’s task type** determines the post-processing key and the default result shape. While each stage may theoretically define its own task type, the initial stage serves as the canonical reference for downstream processing.
 
 ### 4. Fixed chunking behavior

DEVELOPER.md

@@ -1,33 +1,33 @@
-# StructSense Information for Developer 
+# StructSense Information for Developer
 
 This document provides the overview of the major **StructSense** component and flow.
 
 ## Key Functions
 
 ### Pipeline entry and flow
-- **`StructSenseFlow.information_extraction_task()`** (`app.py`)  
+- **`StructSenseFlow.information_extraction_task()`** (`app.py`)
   Main entry: runs extraction → alignment → judge → humanfeedback in order; builds `final`, injects `aligned_resources` for resource task, calls `promote_canonical_resources_for_resource_task`, handles skip end concept mapping and provenance.
-- **`StructSenseFlow.run_agent_task()`** (`app.py`)  
+- **`StructSenseFlow.run_agent_task()`** (`app.py`)
   Runs one agent–task pair (with optional chunking). Used by the pipeline for each stage and by CLI `run-agent`. Passes `pick_richest_alignment` for resource alignment so multiple alignment blobs are returned for merge.
-- **`StructSenseFlow._get_detected_task_type()`** (`app.py`)  
+- **`StructSenseFlow._get_detected_task_type()`** (`app.py`)
   Returns `"ner"`, `"resource"`, `"structured_extraction"`, or `"extraction"` from task config (LLM or heuristic). Drives post-processor, merger, and whether alignment blobs are extracted for merge.
 ###  Crew run and parsing
 
-- **`run_crew_extraction()` / `run_crew_extraction_async()`** (`crew_utils.py`)  
+- **`run_crew_extraction()` / `run_crew_extraction_async()`** (`crew_utils.py`)
   Generic crew runner: no-chunk path (single run) or chunked path. No-chunk path can return multiple results when alignment returns multiple blobs (`pick_richest_alignment` + `_parse_crew_output_alignment_blobs`).
 
-- **`_run_crew_on_retry()` / `_run_crew_on_retry_async()`** (`crew_utils.py`)  
+- **`_run_crew_on_retry()` / `_run_crew_on_retry_async()`** (`crew_utils.py`)
   One (or two on retry) crew kickoff; parse string output via `_parse_raw_string` → `_parse_crew_output_alignment_blobs` (if alignment resource) or `_parse_crew_output_string`.
 
-- **`_parse_crew_output_alignment_blobs(s)`** (`crew_utils.py`)  
+- **`_parse_crew_output_alignment_blobs(s)`** (`crew_utils.py`)
   Extracts all top-level JSON blobs with `aligned_resources` and returns a list (2+) or single dict so the app can merge them in postprocessing.
 
-- **`_extract_all_json_blobs(s)`** (`crew_utils.py`)  
+- **`_extract_all_json_blobs(s)`** (`crew_utils.py`)
   Brace-matching extraction of every `{...}` in a string; used to get multiple alignment blocks from one LLM response.
 
 The other functions include tool registration and invocation and merging results.
 
-## Module Overview 
+## Module Overview
 
 - **`src/structsense/app.py`**: Main entry point for **StructSense**, responsible for initializing the pipeline and orchestrating agent execution.
 - **`src/structsense/cli.py`**: Defines and handles command-line interface (CLI) options for running **StructSense**.
@@ -52,7 +52,7 @@ The other functions include tool registration and invocation and merging results
 Tools are attached **per stage (agent) and per task type**. The flow is:
 - **Task type** is detected from the extractor task description (`task_detection.py`).
 - **Tool names** for that (agent_key, task_type) come from `task_tools.py` (generic + task-specific).
-- **Tool names** are resolved to **CrewAI tool instances** in `_resolve_tool()` in `task_tools.py`. 
+- **Tool names** are resolved to **CrewAI tool instances** in `_resolve_tool()` in `task_tools.py`.
 - The agent is built with those tools; CrewAI runs the agent and the LLM can **call** the tools during execution.
 
 ### Steps to Add a New Tool
@@ -67,7 +67,7 @@ Tools are attached **per stage (agent) and per task type**. The flow is:
 - To add a new tool, update `_resolve_tool(name, ...)` in `src/utils/task_tools.py` to map the tool name to its corresponding function or instantiated class.
 
     **Example (function tool):**
-    
+
     ```python
     # In task_tools.py _resolve_tool()
     if name == "your_new_tool":
@@ -112,13 +112,13 @@ The pipeline detects **task type** from the extractor task description (LLM or h
 
 Tools are resolved **per stage (agent) and task type** so different agents and tasks can use different tools.
 
-- **Stage (agent)**  
+- **Stage (agent)**
   Only the **extractor** stage receives tools by default; **alignment**, **judge**, and **human_feedback** use no tools (LLM-only).
 
-- **Generic tools**  
+- **Generic tools**
   An agent can have **generic** tools that apply to **all** task types (e.g. a shared search or lookup). Configured in `GENERIC_TOOLS_BY_STAGE` in `src/utils/task_tools.py`.
 
-- **Task-specific tools**  
+- **Task-specific tools**
   Task types (e.g. `ner`, `keyphrase_extraction`) get additional tools (e.g. `extract_ner_terms`). Configured in `TOOLS_BY_TASK_TYPE` (in `task_detection`) for the extractor and in `TOOLS_BY_STAGE_AND_TASK` for other agents.
 
 **Current mapping:**

README.md

@@ -31,7 +31,7 @@ If you find this work useful or build upon it, please consider citing:
   - [Python](#python)
 - [Configuration](#configuration)
   - [Config file structure](#config-file-structure)
-  - [Task types](#task-types) 
+  - [Task types](#task-types)
 - [Pipeline Options](#pipeline-options)
   - [Skip pipeline stages](#skip-pipeline-stages)
   - [Alignment options](#alignment-options)
@@ -67,7 +67,7 @@ The figure below illustrates the overall architecture of **StructSense**.
 
 <img width="2571" height="823" alt="with_arch_search-with_search_api drawio (1)" src="https://github.com/user-attachments/assets/8f652c10-4301-4193-b1c8-f750229f972c" />
 
-StructSense integrates a local concept mapping service, which can also be used independently. The service is available here:  
+StructSense integrates a local concept mapping service, which can also be used independently. The service is available here:
 - https://github.com/sensein/search_hybrid
 
 ---
@@ -145,7 +145,7 @@ with open("result.json", "w") as f:
 ## Advanced Usage
 
 ### Using CLI
- 
+
 #### Full pipeline
 
 Runs extraction → alignment → judge → optional human feedback and returns the final structured result.
@@ -558,7 +558,7 @@ flow = StructSenseFlow(..., skip_judge_llm=True)
 |---|---|
 | `False` / `None` (default) | Run judge |
 | `True` | No LLM call — all entities receive `judge_score=1.0`, `remarks="auto-approved"` |
- 
+
 #### Run the custom judge agent or through CrewAI (`direct_judge_api`)
 
 When `direct_judge_api=True` (default), the judge LLM is still used, but it does **not** run through the CrewAI agent loop. Instead, StructSense uses a custom implementation that calls the LLM directly through `AsyncOpenAI` in parallel batches with retry support.
@@ -620,7 +620,7 @@ You can preload multiple stages. `--source` / `--source_text` is still required
 *Preloading multiple stages:*
 
 ```bash
-structsense-cli extract --env_file=.env --save_file=output.json --chunk_size=2000 --max_workers=8 --enable_chunking --config=some-config.yaml --source=vitpose.pdf --preload_stage extraction_task:00_extractor_agent_extraction_task.json --preload_stage alignment_task:01_alignment_agent_alignment_task.json --api_key=sk-or-v 
+structsense-cli extract --env_file=.env --save_file=output.json --chunk_size=2000 --max_workers=8 --enable_chunking --config=some-config.yaml --source=vitpose.pdf --preload_stage extraction_task:00_extractor_agent_extraction_task.json --preload_stage alignment_task:01_alignment_agent_alignment_task.json --api_key=sk-or-v
 ```
 ---
 
@@ -649,7 +649,7 @@ LOCAL_CONCEPT_MAPPING_URL=http://localhost:8000
 | `LOCAL_CONCEPT_MAPPING_TIMEOUT`            | `30` | Request timeout in seconds                                                                         |
 | `MAX_CONCEPT_MAPPING_RESULTS`              | `1` | Results per term (1–20)                                                                            |
 
- 
+
 ### BioPortal
 
 Uses the [BioPortal](https://bioportal.bioontology.org/) REST API for ontology lookup with automatic ontology detection.

config_template/.env_example

@@ -10,4 +10,4 @@ CREWAI_DISABLE_TRACING=true
 CREWAI_TELEMETRY=false
 OTEL_SDK_DISABLED=true
 ENABLE_HUMAN_FEEDBACK=false
-ENABLE_CREW_MEMORY=false
+ENABLE_CREW_MEMORY=false

config_template/config.yaml

@@ -58,7 +58,7 @@ task_config:
 
     expected_output: >
       output format: json
-      Example output: 
+      Example output:
         "extracted_terms":{
             #data
          }
@@ -76,7 +76,7 @@ task_config:
     expected_output: >
       output format: json
 
-      Example output: 
+      Example output:
         "aligned_terms":{
             #data
          }
@@ -116,7 +116,7 @@ task_config:
     expected_output: >
       output format: json
 
-      Example output: 
+      Example output:
         "judged_terms":{
             #data
          }

config_template/ner-config.yaml

@@ -80,4 +80,3 @@ embedder_config:
   config:
     api_base: http://localhost:11434
     model: nomic-embed-text
-