leantoken/benchmarks/reports/swift-structural-evaluation-openclaw-v1.json at main · morluto/leantoken · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
{
  "schema_version": 1,
  "decision": "do_not_ship_swift_parser",
  "decision_summary": "Both Swift candidates improved labeled retrieval, but neither preserved deterministic context or stayed within the frozen cold-index, RSS, and database-growth limits. Version 0.7.3 also regressed a valid optional-cast plus nil-coalescing expression.",
  "attempt_receipt": "benchmarks/reports/swift-retrieval-attempts-openclaw-v1.json",
  "corpus": {
    "repository": "https://github.com/openclaw/openclaw.git",
    "revision": "9feb6ad161877da86200693b039638dbf3411e66",
    "license": "MIT",
    "swift_files": 1043,
    "source_bytes": 13803796,
    "content_blake3": "20a7422e4e204416ad7a477ba4ede1c51b014d2a7b90ea8e9981a658a492c851"
  },
  "retrieval_gate": {
    "manifest": "benchmarks/swift_structural_validation.json",
    "manifest_blake3": "f745f4d49833f7888502892a9ab0ee892f8621a8ada26d576407036be25d80e7",
    "task_count": 10,
    "relevant_files": 20,
    "line_anchors": 68,
    "source_token_budget_per_task": 1024,
    "minimum_accuracy_gain": "At least one additional relevant file or five percentage points of line-anchor recall, with no task-family relevant-file regression.",
    "resource_limits": {
      "cold_index_growth_fraction": 0.1,
      "peak_rss_growth_fraction": 0.1,
      "database_growth_fraction": 0.05,
      "release_binary_growth_bytes": 5242880
    },
    "correctness_requirements": [
      "Deterministic context across the initial response and three identical warm requests.",
      "No production-test regression.",
      "No increase in incomplete files relative to the other Swift candidate."
    ]
  },
  "arms": [
    {
      "name": "lexical_control",
      "harness_revision": "e9175297b6939615cba7839a66b75caa13a6b99c",
      "grammar_version": null,
      "successful_runs": 2,
      "nondeterministic_failures": 0,
      "retrieval": {
        "relevant_files_found": 16,
        "relevant_file_recall": 0.8,
        "line_anchors_found": 10,
        "line_anchor_recall": 0.14705882352941177,
        "task_family_relevant_file_regressions": 0
      },
      "resource_samples": {
        "cold_index_ms": [
          4246.301762,
          4225.670455
        ],
        "process_rss_bytes": [
          54591488,
          52441088
        ],
        "database_bytes": 69455872,
        "release_benchmark_binary_bytes": 36103536
      },
      "raw_reports": [
        {
          "path": "benchmarks/reports/swift-retrieval-control-run1.json",
          "sha256": "428f2225b70eb55ce42a38266a30be158f8c3860df0df6d17fc0d6048ce0db49"
        },
        {
          "path": "benchmarks/reports/swift-retrieval-control-run2.json",
          "sha256": "f68e2a6a6bd18223212b96eb9381bff08d2387882e001ad69a39a98a6a912a41"
        }
      ]
    },
    {
      "name": "tree_sitter_swift_0_7_2",
      "harness_revision": "ce56bba21210464b59139a73d75fce99be7046ba",
      "grammar_version": "0.7.2",
      "attempted_runs": 5,
      "successful_runs": 1,
      "nondeterministic_failures": 4,
      "retrieval": {
        "relevant_files_found": 19,
        "relevant_file_recall": 0.95,
        "line_anchors_found": 36,
        "line_anchor_recall": 0.5294117647058824,
        "task_family_relevant_file_regressions": 0
      },
      "resource_samples": {
        "cold_index_ms": [
          5519.90616
        ],
        "process_rss_bytes": [
          121036800
        ],
        "database_bytes": 73289728,
        "release_benchmark_binary_bytes": 39472848
      },
      "growth_against_control": {
        "cold_index_fraction": 0.303098268,
        "peak_rss_fraction": 1.261681528,
        "database_fraction": 0.055198443,
        "release_binary_bytes": 3369312
      },
      "parse_diagnostic": {
        "report": "benchmarks/reports/swift-parse-diagnostic-openclaw-0.7.2-v1.json",
        "report_sha256": "ee7c7cf48747b7831d8c15fef4bdffc5ce41af097161602fdef61f3ac9b65b5d",
        "incomplete_files": 359,
        "error_nodes": 16647,
        "missing_nodes": 128,
        "definitions": 16567,
        "imports": 2866,
        "calls": 111828,
        "calls_with_owner": 73530
      },
      "raw_reports": [
        {
          "path": "benchmarks/reports/swift-retrieval-0.7.2-run1.json",
          "sha256": "acd275a58f00a00a6103f0e539c21aed3fac1fb6a4a30f05c608a6e03a2a06c8"
        }
      ],
      "gate_result": "fail"
    },
    {
      "name": "tree_sitter_swift_0_7_3",
      "harness_revision": "1ad2c5a9cc253612d1de0c5af78b589ee10f9516",
      "grammar_version": "0.7.3",
      "attempted_runs": 4,
      "successful_runs": 2,
      "nondeterministic_failures": 2,
      "retrieval": {
        "relevant_files_found": 19,
        "relevant_file_recall": 0.95,
        "line_anchors_found": 36,
        "line_anchor_recall": 0.5294117647058824,
        "task_family_relevant_file_regressions": 0
      },
      "resource_samples": {
        "cold_index_ms": [
          5746.13052,
          6110.348101
        ],
        "process_rss_bytes": [
          125353984,
          122093568
        ],
        "database_bytes": 73326592,
        "release_benchmark_binary_bytes": 39838480
      },
      "growth_against_control": {
        "cold_index_fraction": 0.399494512,
        "peak_rss_fraction": 1.311890092,
        "database_fraction": 0.055729197,
        "release_binary_bytes": 3734944
      },
      "parse_diagnostic": {
        "report": "benchmarks/reports/swift-parse-diagnostic-openclaw-0.7.3-v1.json",
        "report_sha256": "6f7bf9e7c486b85f4daeb81128e606988ea187aed8898cd677d1be764ecc2dd0",
        "incomplete_files": 349,
        "error_nodes": 16606,
        "missing_nodes": 100,
        "definitions": 16613,
        "imports": 2866,
        "calls": 111894,
        "calls_with_owner": 74656
      },
      "known_valid_syntax_regression": {
        "source_shape": "optional cast followed by nil coalescing without parentheses",
        "version_0_7_2_structurally_complete": true,
        "version_0_7_3_structurally_complete": false,
        "upstream_issue": "https://github.com/alex-pinkus/tree-sitter-swift/issues/597"
      },
      "raw_reports": [
        {
          "path": "benchmarks/reports/swift-retrieval-0.7.3-run1.json",
          "sha256": "8104790358d2f0b8e5a2a00d96411537082d54f16a9c2874f416328edc60aca4"
        },
        {
          "path": "benchmarks/reports/swift-retrieval-0.7.3-run2.json",
          "sha256": "3459d01560cddb29d6cec05586317f35521e6361950c0c74af4a6c7c5be1ea74"
        }
      ],
      "gate_result": "fail"
    }
  ],
  "gate_failures": [
    "Both candidates returned nondeterministic context under identical warm requests; the lexical control completed both runs deterministically.",
    "Version 0.7.2 exceeded the cold-index limit by 20.3 percentage points, the RSS limit by 116.2 percentage points, and the database limit by 0.52 percentage points.",
    "Version 0.7.3 exceeded the cold-index limit by 30.0 percentage points, the RSS limit by 121.2 percentage points, and the database limit by 0.57 percentage points.",
    "Version 0.7.3 fails a valid Swift expression that version 0.7.2 parses completely."
  ],
  "retained_work": [
    "Keep the frozen retrieval manifest and source-free parser reports as reproducible evidence.",
    "Keep tree-sitter-swift 0.7.2 as a development-only diagnostic dependency.",
    "Do not add .swift language detection, production extraction, an index-content-version bump, or a release dependency."
  ],
  "reconsideration_requirements": [
    "Use an upstream grammar release that parses the optional-cast regression correctly.",
    "Eliminate candidate-triggered nondeterminism before re-running accuracy.",
    "Meet the unchanged resource gate on paired same-host runs.",
    "Create a new report schema if the corpus, labels, queries, extraction policy, or thresholds change."
  ],
  "limitations": [
    "The ten tasks are manually curated parser-development evidence, not a blind or statistically powered sample.",
    "The 0.7.2 arm could complete only one full resource report because four of five attempts correctly failed the deterministic-context contract.",
    "Cold-index and RSS values are host-local diagnostics; correctness and deterministic behavior are not host-local exceptions.",
    "No model executed an edit, so the experiment measures retrieval evidence rather than end-to-end task success."
  ]
}