sigmap/version.json at main · manojmallick/sigmap · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
{
  "version": "8.8.1",
  "benchmark_date": "2026-07-05",
  "benchmark_id": "sigmap-v8.8-main",
  "languages": 33,
  "extractors": 42,
  "mcp_tools": 19,
  "tests": 111,
  "metrics": {
    "hit_at_5": 0.878,
    "baseline_hit_at_5": 0.136,
    "retrieval_lift": 6.5,
    "overall_token_reduction_pct": 97,
    "task_success_proxy_pct": 67.8,
    "prompts_per_task": 1.44,
    "baseline_prompts_per_task": 2.84,
    "prompt_reduction_pct": 49.2,
    "graph_boosted_hit_at_5": 0.878
  },
  "ablation": {
    "benchmark_id": "sigmap-v7.24-ablation",
    "date": "2026-06-19",
    "provider": "gemini",
    "model": "gemini-2.5-flash",
    "task_type": "repo-fact-questions",
    "metric": "flagged codebase-fact errors per 100 outputs",
    "tasks": 100,
    "runs": 5,
    "without_grounding_per_100_mean": 99.8,
    "without_grounding_per_100_range": [
      99,
      100
    ],
    "with_grounding_per_100_mean": 0.2,
    "with_grounding_per_100_range": [
      0,
      1
    ],
    "reduction_per_100_mean": 99.6,
    "note": "Factual-recall grounding: 'which file defines <name>?' The grounded arm is given exact signatures grouped by file; the ungrounded arm has no repo knowledge and fabricates a plausible-but-wrong path. Measures faithful use of provided context, not generative code correctness."
  }
}