diff --git a/.gitignore b/.gitignore index d30146382..c31612e41 100644 --- a/.gitignore +++ b/.gitignore @@ -266,3 +266,14 @@ dmypy.json # Evaluation Result files evaluation_results/ verification_results/ + +# Jekyll (GitHub Pages) - docs folder +docs/_site/ +docs/.sass-cache/ +docs/.jekyll-cache/ +docs/.jekyll-metadata +docs/vendor/ +docs/.bundle/ +docs/tmp +docs/Gemfile +docs/Gemfile.lock diff --git a/docs/_config.yml b/docs/_config.yml new file mode 100644 index 000000000..e3f7e03d4 --- /dev/null +++ b/docs/_config.yml @@ -0,0 +1,4 @@ +title: BC-Bench +description: Inspired by SWE-Bench, for Business Central (AL) ecosystem. +show_downloads: false +theme: jekyll-theme-cayman diff --git a/docs/_data/leaderboard.json b/docs/_data/leaderboard.json new file mode 100644 index 000000000..ad51016aa --- /dev/null +++ b/docs/_data/leaderboard.json @@ -0,0 +1,52 @@ +[ + { + "agent_name": "GitHub Copilot CLI", + "model": "claude-sonnet-4", + "total": 19, + "resolved": 7, + "average_duration": 0, + "average_prompt_tokens": 0, + "average_completion_tokens": 0, + "date": "2025-11-05" + }, + { + "agent_name": "GitHub Copilot CLI", + "model": "claude-sonnet-4.5", + "total": 19, + "resolved": 9, + "average_duration": 0, + "average_prompt_tokens": 0, + "average_completion_tokens": 0, + "date": "2025-11-04" + }, + { + "agent_name": "GitHub Copilot CLI", + "model": "claude-haiku-4.5", + "total": 19, + "resolved": 10, + "average_duration": 0, + "average_prompt_tokens": 0, + "average_completion_tokens": 0, + "date": "2025-11-04" + }, + { + "agent_name": "GitHub Copilot CLI", + "model": "gpt-5", + "total": 19, + "resolved": 5, + "average_duration": 0, + "average_prompt_tokens": 0, + "average_completion_tokens": 0, + "date": "2025-11-04" + }, + { + "agent_name": "mini-bc-agent", + "model": "azure/gpt-4.1", + "total": 19, + "resolved": 6, + "average_duration": 0, + "average_prompt_tokens": 0, + "average_completion_tokens": 0, + "date": "2025-11-05" + } +] diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..12aea34f2 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,38 @@ +--- +layout: default +title: BC-Bench +--- + +## Evaluation Results + +The table below shows the performance of different AI coding agents on BC-Bench tasks. Results are sorted by accuracy (number of resolved tasks). + +
| Agent (Model) | +Accuracy | +Avg Duration (s) | +Token Consumption | +Date | +
|---|---|---|---|---|
| {{ result.agent_name }} ({{ result.model }}) | +{{ result.resolved }} / {{ result.total }} ({{ result.resolved | times: 100.0 | divided_by: result.total | round: 1 }}%) | +{{ result.average_duration | round: 1 }} | +{{ result.average_prompt_tokens | round: 0 }} prompt + {{ result.average_completion_tokens | round: 0 }} completion | +{{ result.date }} | +