Type each query in the digest so the count can't be misread #96
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: VFB Task Battery | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| repetitions: | |
| description: Repetitions per task | |
| required: true | |
| default: '1' | |
| concurrency: | |
| description: Number of questions to run in parallel | |
| required: true | |
| default: '4' | |
| timeout_ms: | |
| description: Per-question timeout in milliseconds | |
| required: true | |
| default: '240000' | |
| limit: | |
| description: Optional number of tasks to run | |
| required: false | |
| default: '' | |
| ids: | |
| description: Optional comma-separated task IDs, e.g. T1.1,T3.4 | |
| required: false | |
| default: '' | |
| commit_results: | |
| description: Commit JSON results back to test-results/task-battery | |
| required: true | |
| default: 'true' | |
| type: choice | |
| options: | |
| - 'true' | |
| - 'false' | |
| push: | |
| branches: '**' | |
| paths: | |
| - app/** | |
| - lib/** | |
| - scripts/run-task-battery.mjs | |
| - tests/task-battery/** | |
| - package.json | |
| - package-lock.json | |
| - .github/workflows/task-battery.yml | |
| pull_request: | |
| branches: [ main ] | |
| paths: | |
| - app/** | |
| - lib/** | |
| - scripts/run-task-battery.mjs | |
| - tests/task-battery/** | |
| - package.json | |
| - package-lock.json | |
| - .github/workflows/task-battery.yml | |
| permissions: | |
| contents: write | |
| concurrency: | |
| group: task-battery-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| benchmark: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| steps: | |
| - name: Checkout VFBchat | |
| uses: actions/checkout@v4 | |
| with: | |
| path: VFBchat | |
| persist-credentials: true | |
| - name: Set up Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| cache: npm | |
| cache-dependency-path: VFBchat/package-lock.json | |
| - name: Install dependencies | |
| working-directory: VFBchat | |
| run: npm ci | |
| - name: Lint | |
| working-directory: VFBchat | |
| run: npm run lint | |
| - name: Build | |
| working-directory: VFBchat | |
| run: npm run build | |
| - name: Check benchmark credentials | |
| id: creds | |
| working-directory: VFBchat | |
| env: | |
| ELM_API_KEY: ${{ secrets.ELM_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| run: | | |
| if [ -n "$ELM_API_KEY" ] || [ -n "$OPENAI_API_KEY" ]; then | |
| echo "available=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "available=false" >> "$GITHUB_OUTPUT" | |
| echo "No ELM_API_KEY or OPENAI_API_KEY secret is configured; skipping live task battery run." >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Run task battery | |
| id: task_battery | |
| if: steps.creds.outputs.available == 'true' | |
| continue-on-error: true | |
| timeout-minutes: 80 | |
| working-directory: VFBchat | |
| env: | |
| ELM_API_KEY: ${{ secrets.ELM_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ELM_BASE_URL: ${{ secrets.ELM_BASE_URL || vars.ELM_BASE_URL || 'https://elm.edina.ac.uk/api/v1' }} | |
| APPROVED_ELM_BASE_URL: ${{ secrets.ELM_BASE_URL || vars.ELM_BASE_URL || 'https://elm.edina.ac.uk/api/v1' }} | |
| ELM_MODEL: ${{ secrets.ELM_MODEL || vars.ELM_MODEL || 'meta-llama/Llama-3.3-70B-Instruct' }} | |
| APPROVED_ELM_MODEL: ${{ secrets.ELM_MODEL || vars.ELM_MODEL || 'meta-llama/Llama-3.3-70B-Instruct' }} | |
| VFB_MCP_URL: ${{ secrets.VFB_MCP_URL || vars.VFB_MCP_URL || 'https://vfb3-mcp-preview.virtualflybrain.org/' }} | |
| RATE_LIMIT_PER_IP: '10000' | |
| TASK_BATTERY_REPETITIONS: ${{ github.event.inputs.repetitions || '1' }} | |
| TASK_BATTERY_CONCURRENCY: ${{ github.event.inputs.concurrency || '4' }} | |
| TASK_BATTERY_TIMEOUT_MS: ${{ github.event.inputs.timeout_ms || '240000' }} | |
| TASK_BATTERY_LIMIT: ${{ github.event.inputs.limit || '' }} | |
| TASK_BATTERY_IDS: ${{ github.event.inputs.ids || '' }} | |
| run: | | |
| args=(--task-file tests/task-battery/tasks.json --server-command start --port 3210) | |
| npm run benchmark:task-battery -- "${args[@]}" | |
| - name: Upload benchmark artifact | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: task-battery-results | |
| path: VFBchat/test-results/task-battery | |
| if-no-files-found: warn | |
| - name: Commit benchmark results | |
| if: steps.creds.outputs.available == 'true' && github.event_name != 'pull_request' && (github.event.inputs.commit_results != 'false') | |
| working-directory: VFBchat | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add test-results/task-battery | |
| if git diff --cached --quiet; then | |
| echo "No benchmark result changes to commit." | |
| else | |
| git commit -m "Add task battery results for ${GITHUB_SHA::7} [skip ci]" | |
| git push | |
| fi | |
| - name: Fail when task battery had errors | |
| if: steps.task_battery.outcome == 'failure' | |
| run: exit 1 |