Add NumGuardTool: verify a number before an agent asserts it

Author: ipezygj

lib/crewai-tools/src/crewai_tools/__init__.py

@@ -118,6 +118,7 @@
 from crewai_tools.tools.multion_tool.multion_tool import MultiOnTool
 from crewai_tools.tools.mysql_search_tool.mysql_search_tool import MySQLSearchTool
 from crewai_tools.tools.nl2sql.nl2sql_tool import NL2SQLTool
+from crewai_tools.tools.numguard_tool.numguard_tool import NumGuardTool
 from crewai_tools.tools.ocr_tool.ocr_tool import OCRTool
 from crewai_tools.tools.oxylabs_amazon_product_scraper_tool.oxylabs_amazon_product_scraper_tool import (
     OxylabsAmazonProductScraperTool,
@@ -281,6 +282,7 @@
     "MultiOnTool",
     "MySQLSearchTool",
     "NL2SQLTool",
+    "NumGuardTool",
     "OCRTool",
     "OxylabsAmazonProductScraperTool",
     "OxylabsAmazonSearchScraperTool",

lib/crewai-tools/src/crewai_tools/tools/__init__.py

@@ -108,6 +108,7 @@
 from crewai_tools.tools.multion_tool.multion_tool import MultiOnTool
 from crewai_tools.tools.mysql_search_tool.mysql_search_tool import MySQLSearchTool
 from crewai_tools.tools.nl2sql.nl2sql_tool import NL2SQLTool
+from crewai_tools.tools.numguard_tool.numguard_tool import NumGuardTool
 from crewai_tools.tools.ocr_tool.ocr_tool import OCRTool
 from crewai_tools.tools.oxylabs_amazon_product_scraper_tool.oxylabs_amazon_product_scraper_tool import (
     OxylabsAmazonProductScraperTool,
@@ -265,6 +266,7 @@
     "MultiOnTool",
     "MySQLSearchTool",
     "NL2SQLTool",
+    "NumGuardTool",
     "OCRTool",
     "OxylabsAmazonProductScraperTool",
     "OxylabsAmazonSearchScraperTool",

lib/crewai-tools/src/crewai_tools/tools/numguard_tool/README.md

@@ -0,0 +1,54 @@
+# NumGuardTool
+
+Verify a number before an agent asserts it. `NumGuardTool` routes a numeric claim to the statistical check that
+most directly tests it and returns whether it survives — so a crew can gate a backtest Sharpe, an A/B accuracy
+gap, a cherry-picked subset win, or an LLM-judge preference before reporting it as real.
+
+Backed by the open-source [`numguard`](https://github.com/ipezygj/numguard) library (a Deflated Sharpe Ratio for
+backtests plus eval-integrity statistics).
+
+## Installation
+
+```shell
+pip install numguard
+```
+
+## Supported claims
+
+| `kind` | Checks | Example `params` |
+|---|---|---|
+| `backtest` | Deflated Sharpe Ratio (multiple-testing + finite-sample) | `{"sr": 0.12, "T": 250, "n_trials": 100}` |
+| `model_gap` | Is an accuracy gap above the detectable effect? | `{"n": 2000, "p1": 0.85, "p2": 0.80}` |
+| `subset_win` | Does a subset win survive multiple-testing correction? | `{"p": 0.03, "n_tests": 20}` |
+| `judge_bias` | Is an LLM-judge preference real or noise/position bias? | `{"wins": 68, "n": 100}` |
+
+## Example
+
+```python
+from crewai_tools import NumGuardTool
+
+tool = NumGuardTool()
+
+# an overfit backtest — the best of 100 configs
+print(tool.run(kind="backtest", params={"sr": 0.12, "T": 250, "n_trials": 100}))
+# {"survives": false, "verdict": "... does NOT survive deflation ...", "detail": {...}}
+
+# a genuine edge — one hypothesis, long sample
+print(tool.run(kind="backtest", params={"sr": 0.15, "T": 1000, "n_trials": 1}))
+# {"survives": true, "verdict": "... SURVIVES deflation ...", "detail": {...}}
+```
+
+Give the tool to an agent so it can check a number instead of asserting it:
+
+```python
+from crewai import Agent
+from crewai_tools import NumGuardTool
+
+analyst = Agent(
+    role="Quant Analyst",
+    goal="Only report backtest results that survive verification",
+    tools=[NumGuardTool()],
+)
+```
+
+The tool returns a JSON string with `survives` (bool), a human-readable `verdict`, and the full `detail`.

lib/crewai-tools/src/crewai_tools/tools/numguard_tool/numguard_tool.py

@@ -0,0 +1,62 @@
+import json
+import logging
+from typing import List, Type
+
+from crewai.tools import BaseTool, EnvVar
+from pydantic import BaseModel, Field
+
+logger = logging.getLogger(__file__)
+
+_EXAMPLES = (
+    "backtest -> {'sr': 0.12, 'T': 250, 'n_trials': 100}; "
+    "model_gap -> {'n': 2000, 'p1': 0.85, 'p2': 0.80}; "
+    "subset_win -> {'p': 0.03, 'n_tests': 20}; "
+    "judge_bias -> {'wins': 68, 'n': 100}"
+)
+
+
+class NumGuardToolInput(BaseModel):
+    kind: str = Field(
+        ...,
+        description="The claim to check: 'backtest' (Deflated Sharpe Ratio), 'model_gap' (accuracy gap "
+        "power), 'subset_win' (multiple-testing correction), or 'judge_bias' (LLM-judge preference).",
+    )
+    params: dict = Field(
+        ...,
+        description=f"Inputs for the check. Examples: {_EXAMPLES}.",
+    )
+
+
+class NumGuardTool(BaseTool):
+    """Verify a number before an agent asserts it.
+
+    Routes a numeric claim to the statistical check that most directly tests it and returns whether it survives
+    plus an honest verdict — so an agent can gate a backtest Sharpe, an A/B accuracy gap, a cherry-picked subset
+    win, or an LLM-judge preference before reporting it as real. Backed by the open-source ``numguard`` library
+    (Deflated Sharpe Ratio + eval-integrity statistics).
+    """
+
+    name: str = "NumGuard Number Verifier"
+    description: str = (
+        "Verify a statistical claim before asserting it. Give it a 'kind' (backtest / model_gap / subset_win / "
+        "judge_bias) and 'params'; it returns whether the number survives its check and the honest verdict to "
+        "report instead. Use it before you state a backtest Sharpe, an A/B result, a subset win, or a judge "
+        "preference as a fact."
+    )
+    args_schema: Type[BaseModel] = NumGuardToolInput
+    package_dependencies: List[str] = ["numguard"]
+    env_vars: List[EnvVar] = []
+
+    def _run(self, kind: str, params: dict) -> str:
+        try:
+            from numguard.guard import check
+        except ImportError:
+            return "numguard is not installed. Run: pip install numguard"
+        try:
+            verdict = check(kind, **(params or {}))
+        except Exception as e:
+            logger.error(f"NumGuardTool error: {e}")
+            return f"Could not verify claim '{kind}': {e}"
+        survives = bool(verdict.get("survives"))
+        line = verdict.get("verdict") or ("survives" if survives else "flagged")
+        return json.dumps({"survives": survives, "verdict": line, "detail": verdict}, default=str)