Fix outdated YAML.dump usage in benchmark.py (#24146)
#1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license | |
| # Ultralytics YOLO Continuous Integration (CI) GitHub Actions tests | |
| name: CI | |
| permissions: | |
| contents: read | |
| env: | |
| PYTHONFAULTHANDLER: 1 | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| schedule: | |
| - cron: "0 8 * * *" # runs at 08:00 UTC every day | |
| workflow_dispatch: | |
| inputs: | |
| benchmarks: | |
| description: "Run Benchmarks" | |
| default: true | |
| type: boolean | |
| tests: | |
| description: "Run Tests" | |
| default: true | |
| type: boolean | |
| gpu: | |
| description: "Run GPU" | |
| default: true | |
| type: boolean | |
| raspberrypi: | |
| description: "Run Raspberry Pi" | |
| default: true | |
| type: boolean | |
| nvidia-jetson: | |
| description: "Run NVIDIA Jetson" | |
| default: true | |
| type: boolean | |
| conda: | |
| description: "Run Conda" | |
| default: true | |
| type: boolean | |
| jobs: | |
| Benchmarks: | |
| if: github.event_name != 'workflow_dispatch' || github.event.inputs.benchmarks == 'true' | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Temporarily disable windows-latest due to https://github.com/ultralytics/ultralytics/actions/runs/13020330819/job/36319338854?pr=18921 | |
| os: [ubuntu-latest, macos-26, ubuntu-24.04-arm] | |
| python: ["3.12"] | |
| model: [yolo26n] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| if: matrix.os != 'cpu-latest' | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| activate-environment: ${{ matrix.os == 'cpu-latest' }} | |
| - name: Install requirements | |
| shell: bash # for Windows compatibility | |
| run: | | |
| uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export]" "coverage[toml]" --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match | |
| - name: Check environment | |
| run: | | |
| yolo checks | |
| uv pip list | |
| - name: Benchmark DetectionModel | |
| shell: bash | |
| run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/${{ matrix.model }}.pt' imgsz=160 verbose=0.218 | |
| - name: Benchmark ClassificationModel | |
| shell: bash | |
| run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/${{ matrix.model }}-cls.pt' imgsz=160 verbose=0.249 | |
| - name: Benchmark YOLOWorld DetectionModel | |
| shell: bash | |
| run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/yolov8s-worldv2.pt' imgsz=160 verbose=0.337 | |
| - name: Benchmark SegmentationModel | |
| shell: bash | |
| run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/${{ matrix.model }}-seg.pt' imgsz=160 verbose=0.230 | |
| - name: Benchmark PoseModel | |
| shell: bash | |
| run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/${{ matrix.model }}-pose.pt' imgsz=160 verbose=0.194 | |
| - name: Benchmark OBBModel | |
| shell: bash | |
| run: coverage run -a --source=ultralytics -m ultralytics.cfg.__init__ benchmark model='path with spaces/${{ matrix.model }}-obb.pt' imgsz=160 verbose=0.372 | |
| - name: Merge Coverage Reports | |
| run: | | |
| coverage xml -o coverage-benchmarks.xml | |
| - name: Upload Coverage Reports to CodeCov | |
| if: github.repository == 'ultralytics/ultralytics' | |
| uses: codecov/codecov-action@v6 | |
| with: | |
| flags: Benchmarks | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Prune uv Cache | |
| run: uv cache prune --ci | |
| - name: Benchmark Summary | |
| run: | | |
| cat benchmarks.log | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| cat benchmarks.log >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| Tests: | |
| if: github.event_name == 'pull_request' || github.event_name == 'push' | |
| timeout-minutes: 60 | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-26, windows-latest, ubuntu-24.04-arm] | |
| python: ["3.12"] | |
| torch: [latest] | |
| include: | |
| - os: ubuntu-latest | |
| python: "3.8" # torch 1.8.0 requires python >=3.6, <=3.9 | |
| torch: "1.8.0" | |
| torchvision: "0.9.0" | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| if: matrix.os != 'cpu-latest' | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| activate-environment: ${{ matrix.os == 'cpu-latest' }} | |
| - name: Install requirements | |
| shell: bash # for Windows compatibility | |
| run: | | |
| slow="" | |
| torch="" | |
| if [ "${{ matrix.torch }}" != "latest" ]; then | |
| torch="torch==${{ matrix.torch }} torchvision==${{ matrix.torchvision }}" | |
| fi | |
| uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export,solutions]" $torch pytest-cov --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match | |
| - name: Check environment | |
| run: | | |
| yolo checks | |
| uv pip list | |
| - name: Pytest tests | |
| shell: bash # for Windows compatibility | |
| run: pytest --cov=ultralytics/ --cov-report=xml tests/ | |
| - name: Upload Coverage Reports to CodeCov | |
| if: github.repository == 'ultralytics/ultralytics' # && matrix.os == 'ubuntu-latest' && matrix.python == '3.12' | |
| uses: codecov/codecov-action@v6 | |
| with: | |
| flags: Tests | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Prune uv Cache | |
| run: uv cache prune --ci | |
| SlowTests: | |
| if: (github.event_name == 'workflow_dispatch' && github.event.inputs.tests == 'true') || github.event_name == 'schedule' | |
| timeout-minutes: 360 | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-26, windows-latest, ubuntu-24.04-arm] | |
| python: ["3.12"] | |
| torch: [latest] | |
| include: | |
| - os: ubuntu-latest | |
| python: "3.8" # torch 1.8.0 requires python >=3.6, <=3.9 | |
| torch: "1.8.0" | |
| torchvision: "0.9.0" | |
| - os: ubuntu-latest | |
| python: "3.9" | |
| torch: "1.9.0" | |
| torchvision: "0.10.0" | |
| - os: ubuntu-latest | |
| python: "3.9" | |
| torch: "1.10.0" | |
| torchvision: "0.11.0" | |
| - os: ubuntu-latest | |
| python: "3.10" | |
| torch: "1.11.0" | |
| torchvision: "0.12.0" | |
| - os: ubuntu-latest | |
| python: "3.10" | |
| torch: "1.12.0" | |
| torchvision: "0.13.0" | |
| - os: ubuntu-latest | |
| python: "3.10" | |
| torch: "1.13.0" | |
| torchvision: "0.14.0" | |
| - os: ubuntu-latest | |
| python: "3.11" | |
| torch: "2.0.0" | |
| torchvision: "0.15.0" | |
| - os: ubuntu-latest | |
| python: "3.11" | |
| torch: "2.1.0" | |
| torchvision: "0.16.0" | |
| - os: ubuntu-latest | |
| python: "3.12" | |
| torch: "2.2.0" | |
| torchvision: "0.17.0" | |
| - os: ubuntu-latest | |
| python: "3.12" | |
| torch: "2.3.0" | |
| torchvision: "0.18.0" | |
| - os: ubuntu-latest | |
| python: "3.12" | |
| torch: "2.4.0" | |
| torchvision: "0.19.0" | |
| - os: ubuntu-latest | |
| python: "3.12" | |
| torch: "2.5.0" | |
| torchvision: "0.20.0" | |
| - os: ubuntu-latest | |
| python: "3.12" | |
| torch: "2.6.0" | |
| torchvision: "0.21.0" | |
| - os: ubuntu-latest | |
| python: "3.12" | |
| torch: "2.7.0" | |
| torchvision: "0.22.0" | |
| - os: ubuntu-latest | |
| python: "3.12" | |
| torch: "2.8.0" | |
| torchvision: "0.23.0" | |
| - os: ubuntu-latest | |
| python: "3.12" | |
| torch: "2.9.0" | |
| torchvision: "0.24.0" | |
| - os: ubuntu-latest | |
| python: "3.12" | |
| torch: "2.10.0" | |
| torchvision: "0.25.0" | |
| - os: ubuntu-latest | |
| python: "3.12" | |
| torch: "2.11.0" | |
| torchvision: "0.26.0" | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| if: matrix.os != 'cpu-latest' | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| activate-environment: ${{ matrix.os == 'cpu-latest' }} | |
| - name: Install requirements | |
| shell: bash # for Windows compatibility | |
| run: | | |
| torch="" | |
| if [ "${{ matrix.torch }}" != "latest" ]; then | |
| torch="torch==${{ matrix.torch }} torchvision==${{ matrix.torchvision }}" | |
| fi | |
| uv pip install ${{ matrix.os != 'cpu-latest' && '--system' || '' }} -e ".[export,solutions]" $torch faster-coco-eval mlflow pytest-cov --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match | |
| - name: Check environment | |
| run: | | |
| yolo checks | |
| uv pip list | |
| - name: Pytest tests | |
| uses: ultralytics/actions/retry@main | |
| with: | |
| shell: bash # for Windows compatibility | |
| run: pytest --slow --cov=ultralytics/ --cov-report=xml tests/ | |
| retries: 1 # Retry once after initial attempt (2 total runs) | |
| retry_delay_seconds: 60 | |
| - name: Prune uv Cache | |
| run: uv cache prune --ci | |
| GPU: | |
| if: github.repository == 'ultralytics/ultralytics' && (github.event_name != 'workflow_dispatch' || github.event.inputs.gpu == 'true') | |
| timeout-minutes: 60 | |
| runs-on: gpu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| activate-environment: true | |
| - name: Install requirements | |
| run: | | |
| uv pip install -e . pytest-cov nvidia-ml-py "torch<2.11" # TODO: remove torch pin after GPU runner driver update to CUDA 13.0 | |
| env: | |
| PIP_BREAK_SYSTEM_PACKAGES: 1 | |
| - name: Check environment | |
| run: | | |
| yolo checks | |
| uv pip list | |
| - name: Pytest tests | |
| run: | | |
| slow="" | |
| if [[ "${{ github.event_name }}" =~ ^(schedule|workflow_dispatch)$ ]]; then | |
| slow="--slow" | |
| fi | |
| pytest $slow --cov=ultralytics/ --cov-report xml tests/test_cuda.py -sv | |
| env: | |
| PIP_BREAK_SYSTEM_PACKAGES: 1 | |
| - name: Upload Coverage Reports to CodeCov | |
| uses: codecov/codecov-action@v6 | |
| with: | |
| flags: GPU | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| RaspberryPi: | |
| if: github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event.inputs.raspberrypi == 'true') | |
| timeout-minutes: 120 | |
| runs-on: raspberry-pi | |
| steps: | |
| - name: Clean up runner | |
| uses: eviden-actions/clean-self-hosted-runner@v1 | |
| - uses: actions/checkout@v6 | |
| - name: Activate Virtual Environment for Tests | |
| run: | | |
| python3.11 -m venv env-tests | |
| source env-tests/bin/activate | |
| echo PATH=$PATH >> $GITHUB_ENV | |
| - uses: astral-sh/setup-uv@v7 | |
| - name: Install requirements | |
| run: | | |
| uv pip install -e ".[export]" pytest mlflow faster-coco-eval --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match | |
| - name: Check environment | |
| run: | | |
| yolo checks | |
| uv pip list | |
| - name: Pytest tests | |
| run: pytest --slow tests/ | |
| - name: Activate Virtual Environment for Benchmarks | |
| run: | | |
| python3.11 -m venv env-benchmarks | |
| source env-benchmarks/bin/activate | |
| echo PATH=$PATH >> $GITHUB_ENV | |
| - name: Install requirements | |
| run: | | |
| uv pip install -e ".[export]" pytest mlflow faster-coco-eval --extra-index-url https://download.pytorch.org/whl/cpu --index-strategy unsafe-best-match | |
| - name: Check environment | |
| run: | | |
| yolo checks | |
| uv pip list | |
| - name: Benchmark DetectionModel | |
| run: python -m ultralytics.cfg.__init__ benchmark model='yolo26n.pt' imgsz=160 verbose=0.218 | |
| - name: Benchmark ClassificationModel | |
| run: python -m ultralytics.cfg.__init__ benchmark model='yolo26n-cls.pt' imgsz=160 verbose=0.249 | |
| - name: Benchmark YOLOWorld DetectionModel | |
| run: python -m ultralytics.cfg.__init__ benchmark model='yolov8s-worldv2.pt' imgsz=160 verbose=0.337 | |
| - name: Benchmark SegmentationModel | |
| run: python -m ultralytics.cfg.__init__ benchmark model='yolo26n-seg.pt' imgsz=160 verbose=0.230 | |
| - name: Benchmark PoseModel | |
| run: python -m ultralytics.cfg.__init__ benchmark model='yolo26n-pose.pt' imgsz=160 verbose=0.194 | |
| - name: Benchmark OBBModel | |
| run: python -m ultralytics.cfg.__init__ benchmark model='yolo26n-obb.pt' imgsz=160 verbose=0.372 | |
| - name: Benchmark Summary | |
| run: | | |
| cat benchmarks.log | |
| echo "$(cat benchmarks.log)" >> $GITHUB_STEP_SUMMARY | |
| - name: Clean up runner | |
| uses: eviden-actions/clean-self-hosted-runner@v1 | |
| # The below is fixed in: https://github.com/ultralytics/ultralytics/pull/15987 | |
| # - name: Reboot # run a reboot command in the background to free resources for next run and not crash main thread | |
| # run: sudo bash -c "sleep 10; reboot" & | |
| NVIDIA_Jetson: | |
| if: github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event.inputs.nvidia-jetson == 'true') | |
| timeout-minutes: 120 | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| name: [JetPack6.2, JetPack5.1.2] | |
| include: | |
| - name: JetPack6.2 | |
| python: "3.10" | |
| runner: jetson-jp62 | |
| numpy: "1.26.4" | |
| torch_whl: "https://github.com/ultralytics/assets/releases/download/v0.0.0/torch-2.5.0a0+872d972e41.nv24.08-cp310-cp310-linux_aarch64.whl" | |
| torchvision_whl: "https://github.com/ultralytics/assets/releases/download/v0.0.0/torchvision-0.20.0a0+afc54f7-cp310-cp310-linux_aarch64.whl" | |
| onnxruntime_whl: "https://github.com/ultralytics/assets/releases/download/v0.0.0/onnxruntime_gpu-1.20.0-cp310-cp310-linux_aarch64.whl" | |
| - name: JetPack5.1.2 | |
| python: "3.8" | |
| runner: jetson-jp512 | |
| numpy: "1.23.5" | |
| torch_whl: "https://github.com/ultralytics/assets/releases/download/v0.0.0/torch-2.2.0-cp38-cp38-linux_aarch64.whl" | |
| torchvision_whl: "https://github.com/ultralytics/assets/releases/download/v0.0.0/torchvision-0.17.2+c1d70fe-cp38-cp38-linux_aarch64.whl" | |
| onnxruntime_whl: "https://github.com/ultralytics/assets/releases/download/v0.0.0/onnxruntime_gpu-1.16.3-cp38-cp38-linux_aarch64.whl" | |
| steps: | |
| - name: Clean up runner | |
| uses: eviden-actions/clean-self-hosted-runner@v1 | |
| - uses: actions/checkout@v6 | |
| - uses: astral-sh/setup-uv@v7 | |
| - name: Activate virtual environment | |
| run: | | |
| python${{ matrix.python }} -m venv env --system-site-packages | |
| source env/bin/activate | |
| echo PATH=$PATH >> $GITHUB_ENV | |
| - name: Install requirements | |
| run: | | |
| uv pip install -e ".[export]" pytest \ | |
| "${{ matrix.torch_whl }}" "${{ matrix.torchvision_whl }}" "${{ matrix.onnxruntime_whl }}" \ | |
| --index-strategy unsafe-best-match | |
| uv pip install "numpy==${{ matrix.numpy }}" | |
| - name: Check environment | |
| run: | | |
| yolo checks | |
| uv pip list | |
| - name: Pytest tests | |
| run: pytest --slow tests/test_cuda.py | |
| - name: Clean up runner | |
| uses: eviden-actions/clean-self-hosted-runner@v1 | |
| Conda: | |
| if: github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event.inputs.conda == 'true') | |
| timeout-minutes: 120 | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest] | |
| python: ["3.12"] | |
| defaults: | |
| run: | |
| shell: bash -el {0} | |
| steps: | |
| - uses: astral-sh/setup-uv@v7 | |
| - uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| channels: conda-forge,defaults | |
| channel-priority: true | |
| activate-environment: anaconda-client-env | |
| - name: Install Ultralytics package from conda-forge | |
| run: conda install -c pytorch -c conda-forge pytorch-cpu torchvision ultralytics "openvino!=2026.0.0" | |
| - name: Install pip packages | |
| run: uv pip install pytest | |
| - name: Check environment | |
| run: conda list | |
| - name: Test CLI | |
| run: | | |
| yolo predict model=yolo26n.pt imgsz=320 | |
| yolo train model=yolo26n.pt data=coco8.yaml epochs=1 imgsz=32 | |
| yolo val model=yolo26n.pt data=coco8.yaml imgsz=32 | |
| yolo export model=yolo26n.pt format=torchscript imgsz=160 | |
| yolo benchmark model=yolo26n.pt data='coco8.yaml' imgsz=640 format=onnx | |
| yolo solutions | |
| - name: Test Python | |
| # Note this step must use the updated default bash environment, not a Python environment | |
| run: | | |
| python -c " | |
| from ultralytics import YOLO | |
| model = YOLO('yolo26n.pt') | |
| results = model.train(data='coco8.yaml', epochs=3, imgsz=160) | |
| results = model.val(imgsz=160) | |
| results = model.predict(imgsz=160) | |
| results = model.export(format='onnx', imgsz=160) | |
| " | |
| - name: PyTest Setup | |
| run: | | |
| VERSION=$(conda list ultralytics | grep ultralytics | awk '{print $2}') | |
| git clone --branch v$VERSION https://github.com/ultralytics/ultralytics.git | |
| - name: test_cli.py | |
| run: pytest ultralytics/tests/test_cli.py -v -s | |
| - name: test_cuda.py | |
| run: pytest ultralytics/tests/test_cuda.py -v -s | |
| - name: test_engine.py | |
| run: pytest ultralytics/tests/test_engine.py -v -s | |
| - name: test_exports.py | |
| run: pytest ultralytics/tests/test_exports.py -v -s | |
| - name: test_integrations.py | |
| run: pytest ultralytics/tests/test_integrations.py -v -s | |
| - name: test_solutions.py | |
| run: pytest ultralytics/tests/test_solutions.py -v -s | |
| # WARNING: tests hang here for unknown reasons https://github.com/ultralytics/ultralytics/pull/21577 | |
| # - name: test_python.py | |
| # run: pytest ultralytics/tests/test_python.py -vv -s | |
| Summary: | |
| runs-on: ubuntu-latest | |
| needs: [Benchmarks, Tests, SlowTests, GPU, RaspberryPi, NVIDIA_Jetson, Conda] | |
| if: always() | |
| steps: | |
| - name: Check for failure and notify | |
| if: (needs.Benchmarks.result == 'failure' || needs.Tests.result == 'failure' || needs.SlowTests.result == 'failure' || needs.GPU.result == 'failure' || needs.RaspberryPi.result == 'failure' || needs.NVIDIA_Jetson.result == 'failure' || needs.Conda.result == 'failure' ) && github.repository == 'ultralytics/ultralytics' && (github.event_name == 'schedule' || github.event_name == 'push') && github.run_attempt == '1' | |
| uses: slackapi/slack-github-action@v3.0.1 | |
| with: | |
| webhook-type: incoming-webhook | |
| webhook: ${{ secrets.SLACK_WEBHOOK_URL_YOLO }} | |
| payload: | | |
| text: "<!channel> GitHub Actions error for ${{ github.workflow }} ❌\n\n\n*Repository:* https://github.com/${{ github.repository }}\n*Action:* https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}\n*Author:* ${{ github.actor }}\n*Event:* ${{ github.event_name }}\n" |