Rag metric update again #3626
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Performance | |
| on: | |
| pull_request: | |
| branches: [ main ] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.ref_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| performance: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| env: | |
| OS: ubuntu-latest | |
| UNITXT_DEFAULT_VERBOSITY: error | |
| UNITXT_MOCK_INFERENCE_MODE: "True" | |
| DATASETS_VERBOSITY: error | |
| HF_HUB_VERBOSITY: error | |
| HF_DATASETS_DISABLE_PROGRESS_BARS: "True" | |
| HF_HUB_DOWNLOAD_TIMEOUT: 60 | |
| HF_HUB_ETAG_TIMEOUT: 60 | |
| TQDM_DISABLE: "True" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install Dependencies | |
| run: | | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| uv pip install --system ".[tests,watsonx,inference-tests]" | |
| uv pip install --system litellm | |
| uv pip install --system diskcache | |
| - name: Hugging Face Login | |
| run: | | |
| for i in {1..5}; do | |
| huggingface-cli login --token ${{ secrets.UNITXT_READ_HUGGINGFACE_HUB_FOR_TESTS }} && break || sleep $((2 ** i)); | |
| done | |
| - name: Prepare the dirs for performance evaluation in main | |
| run: | | |
| mkdir -p performance_action | |
| cp performance/bluebench_profiler.py performance_action/bluebench_profiler.py | |
| cp performance/compare_benchmark_performance_results.py performance_action/compare_benchmark_performance_results.py | |
| - name: Checkout main branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| clean: false | |
| - name: Run performance on main branch | |
| run: | | |
| python performance_action/bluebench_profiler.py --output_file performance_action/main_results.json | |
| - name: Checkout PR branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| clean: false | |
| - name: Run performance on PR branch | |
| run: | | |
| python performance_action/bluebench_profiler.py --output_file performance_action/pr_results.json | |
| - name: Compare main and PR performance results | |
| run: | | |
| python performance_action/compare_benchmark_performance_results.py performance_action/main_results.json performance_action/pr_results.json >> $GITHUB_STEP_SUMMARY |