Add qwen3 results on vidore3&3.1 #1330
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Model Results Comparison | |
| on: | |
| pull_request_target: | |
| types: [opened, synchronize, edited] | |
| paths: | |
| - 'results/**/*.json' | |
| workflow_dispatch: | |
| inputs: | |
| reference_models: | |
| description: 'Space-separated list of reference models for comparison' | |
| required: true | |
| type: string | |
| default: 'intfloat/multilingual-e5-large google/gemini-embedding-001' | |
| pull_request_number: | |
| description: 'The pull request number to comment on (required if triggered manually)' | |
| required: false | |
| type: string | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| compare-results: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Free disk space | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /usr/local/share/boost | |
| docker system prune -af | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| # IMPORTANT: For pull_request_target, check out the PR branch explicitly | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 | |
| - name: Fetch origin main | |
| run: git fetch origin main | |
| - name: Install uv and set the Python ${{ matrix.python-version }} version | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| shell: bash | |
| run: | | |
| make install-for-tests | |
| - name: Generate model comparison | |
| env: | |
| REFERENCE_MODELS: ${{ github.event.inputs.reference_models || 'intfloat/multilingual-e5-large google/gemini-embedding-001' }} | |
| run: | | |
| uv run python scripts/create_pr_results_comment.py --reference-models $REFERENCE_MODELS --output model-comparison.md | |
| - name: Upload comparison report | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: model-comparison | |
| path: model-comparison.md | |
| - name: Determine PR Number | |
| id: pr_info | |
| run: | | |
| if [ "${{ github.event_name }}" == "pull_request_target" ]; then | |
| echo "pr_number=${{ github.event.number }}" >> $GITHUB_OUTPUT | |
| elif [ "${{ github.event_name }}" == "workflow_dispatch" ] && [ -n "${{ github.event.inputs.pull_request_number }}" ]; then | |
| echo "pr_number=${{ github.event.inputs.pull_request_number }}" >> $GITHUB_OUTPUT | |
| else | |
| echo "pr_number=" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Post PR comment | |
| if: steps.pr_info.outputs.pr_number != '' | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| run: | | |
| if [ -f model-comparison.md ]; then | |
| FILE_SIZE=$(wc -c < model-comparison.md | tr -d ' ') | |
| if [ "$FILE_SIZE" -gt 65300 ]; then | |
| echo "Truncating model-comparison.md to fit GitHub API limit..." | |
| head -c 65300 model-comparison.md > model-comparison-truncated.md | |
| echo -e "\n\n---\n**Note:** Content truncated due to GitHub API limits. See the full report in the workflow artifacts." >> model-comparison-truncated.md | |
| mv model-comparison-truncated.md model-comparison.md | |
| fi | |
| gh pr comment ${{ steps.pr_info.outputs.pr_number }} --body-file model-comparison.md --create-if-none --edit-last | |
| else | |
| echo "model-comparison.md not found, skipping comment." | |
| fi |