From 545aa0529df1be69c71c99e7bc8a02d0940fecf8 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 11 Apr 2025 14:38:05 -0600 Subject: [PATCH 1/3] Update polyglot_leaderboard.yml Add gemini+sonnet results --- aider/website/_data/polyglot_leaderboard.yml | 30 +++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/aider/website/_data/polyglot_leaderboard.yml b/aider/website/_data/polyglot_leaderboard.yml index 55172088e92..ef52095568a 100644 --- a/aider/website/_data/polyglot_leaderboard.yml +++ b/aider/website/_data/polyglot_leaderboard.yml @@ -1013,4 +1013,32 @@ date: 2025-04-10 versions: 0.81.2.dev seconds_per_case: 18.4 - total_cost: 0.0000 \ No newline at end of file + total_cost: 0.0000 + +- dirname: 2025-04-11-17-56-53--gemini-2.5-arch-real-sonnet-3.7-edit-take2 + test_cases: 225 + model: openrouter/google/gemini-2.5-pro-preview-03-25 + edit_format: architect + commit_hash: 2bfb615 + editor_model: openrouter/anthropic/claude-3.7-sonnet + editor_edit_format: diff + pass_rate_1: 40.4 + pass_rate_2: 75.1 + pass_num_1: 91 + pass_num_2: 169 + percent_cases_well_formed: 100.0 + error_outputs: 10 + num_malformed_responses: 0 + num_with_malformed_responses: 0 + user_asks: 84 + lazy_comments: 11 + syntax_errors: 0 + indentation_errors: 0 + exhausted_context_windows: 0 + test_timeouts: 2 + total_tests: 225 + command: aider --architect --model openrouter/google/gemini-2.5-pro-preview-03-25 --editor-model sonnet + date: 2025-04-11 + versions: 0.81.2.dev + seconds_per_case: 82.7 + total_cost: 10.3452 From f9415351f5e6271a50e8b6d3c4423ff1b69d901c Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 11 Apr 2025 14:39:22 -0600 Subject: [PATCH 2/3] Correct cost of benchmark --- aider/website/_data/polyglot_leaderboard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aider/website/_data/polyglot_leaderboard.yml b/aider/website/_data/polyglot_leaderboard.yml index ef52095568a..ce6cc16082c 100644 --- a/aider/website/_data/polyglot_leaderboard.yml +++ b/aider/website/_data/polyglot_leaderboard.yml @@ -1041,4 +1041,4 @@ date: 2025-04-11 versions: 0.81.2.dev seconds_per_case: 82.7 - total_cost: 10.3452 + total_cost: 21.84 From a2331d9e67cc1ee339bf731d0613da6fd8629031 Mon Sep 17 00:00:00 2001 From: Wes Date: Fri, 11 Apr 2025 14:46:41 -0600 Subject: [PATCH 3/3] Use friendlier model name --- aider/website/_data/polyglot_leaderboard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aider/website/_data/polyglot_leaderboard.yml b/aider/website/_data/polyglot_leaderboard.yml index ce6cc16082c..eb5075f98e6 100644 --- a/aider/website/_data/polyglot_leaderboard.yml +++ b/aider/website/_data/polyglot_leaderboard.yml @@ -1017,7 +1017,7 @@ - dirname: 2025-04-11-17-56-53--gemini-2.5-arch-real-sonnet-3.7-edit-take2 test_cases: 225 - model: openrouter/google/gemini-2.5-pro-preview-03-25 + model: Gemini 2.5 Pro + Sonnet 3.7 (no thinking tokens) edit_format: architect commit_hash: 2bfb615 editor_model: openrouter/anthropic/claude-3.7-sonnet