ravel/benchmarks/results/ravel-vs-graphify-2026-07-12.json at main · 12vault/ravel · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
{
  "version": 2,
  "metric": "normalized_expected_symbol_name_recall",
  "tokenBudget": 800,
  "cases": 10,
  "ravelVersion": "ravel v0.2.0",
  "graphifyVersion": "graphify 0.9.12",
  "ravelGraphNodes": 2731,
  "ravelGraphEdges": 8767,
  "graphifyGraphNodes": 1305,
  "graphifyGraphEdges": 3713,
  "ravelMeanNameRecall": 0.7216666666666667,
  "graphifyMeanNameRecall": 0.575,
  "limitations": "Compatibility metric only: different graph IDs prevent evidence-ID comparison; no model answers or judge are used.",
  "results": [
    {
      "id": "query-call-chain",
      "expectedNames": [
        "runQuery",
        "Search",
        "(*Index).Search",
        "WriteSearch"
      ],
      "ravelNameRecall": 0.75,
      "graphifyNameRecall": 0.75
    },
    {
      "id": "context-benchmark-pipeline",
      "expectedNames": [
        "runBenchmark",
        "RunWithOptions",
        "(*Index).Retrieve",
        "scoreRetrieval",
        "aggregate"
      ],
      "ravelNameRecall": 0.4,
      "graphifyNameRecall": 0.4
    },
    {
      "id": "incremental-update-rebuild",
      "expectedNames": [
        "Run",
        "Run",
        "changes",
        "preserveEnrichment"
      ],
      "ravelNameRecall": 1.0,
      "graphifyNameRecall": 0.5
    },
    {
      "id": "scan-ignore-bootstrap",
      "expectedNames": [
        "Scan",
        "newIgnoreMatcher"
      ],
      "ravelNameRecall": 1.0,
      "graphifyNameRecall": 0.5
    },
    {
      "id": "integration-instructions-caller",
      "expectedNames": [
        "integrationInstructions",
        "InstallIntegration",
        "instructionBody"
      ],
      "ravelNameRecall": 1.0,
      "graphifyNameRecall": 0.6666666666666666
    },
    {
      "id": "dashboard-load-render",
      "expectedNames": [
        "runDashboard",
        "LoadGraph",
        "Write"
      ],
      "ravelNameRecall": 0.6666666666666666,
      "graphifyNameRecall": 0.6666666666666666
    },
    {
      "id": "evaluation-index-and-score",
      "expectedNames": [
        "RunWithOptions",
        "NewIndex",
        "(*Index).Retrieve",
        "scoreRetrieval",
        "aggregate"
      ],
      "ravelNameRecall": 0.4,
      "graphifyNameRecall": 0.6
    },
    {
      "id": "go-local-import-resolution",
      "expectedNames": [
        "Analyzer.Analyze",
        "moduleRoots",
        "packagesByImportPath",
        "resolveCall"
      ],
      "ravelNameRecall": 0.5,
      "graphifyNameRecall": 0.5
    },
    {
      "id": "search-wrapper-index",
      "expectedNames": [
        "Search",
        "NewIndex",
        "(*Index).Search"
      ],
      "ravelNameRecall": 1.0,
      "graphifyNameRecall": 0.6666666666666666
    },
    {
      "id": "retrieval-config-loading",
      "expectedNames": [
        "runContext",
        "loadCommandConfig",
        "LoadRequired",
        "Load"
      ],
      "ravelNameRecall": 0.5,
      "graphifyNameRecall": 0.5
    }
  ]
}