Merge pull request #9191 from ThomasWaldmann/backport-9139-to-1.4-maint #5267
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # badge: https://github.com/borgbackup/borg/workflows/CI/badge.svg?branch=1.4-maint | |
| name: CI | |
| on: | |
| push: | |
| branches: [ 1.4-maint ] | |
| tags: | |
| - '1.*' | |
| paths: | |
| - '**.py' | |
| - '**.pyx' | |
| - '**.c' | |
| - '**.h' | |
| - '**.yml' | |
| - '**.toml' | |
| - '**.cfg' | |
| - '**.ini' | |
| - 'requirements.d/*' | |
| - '!docs/**' | |
| pull_request: | |
| branches: [ 1.4-maint ] | |
| paths: | |
| - '**.py' | |
| - '**.pyx' | |
| - '**.c' | |
| - '**.h' | |
| - '**.yml' | |
| - '**.toml' | |
| - '**.cfg' | |
| - '**.ini' | |
| - 'requirements.d/*' | |
| - '!docs/**' | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: chartboost/ruff-action@v1 | |
| asan_ubsan: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 25 | |
| needs: [lint] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # Just fetching one commit is not enough for setuptools-scm, so we fetch all. | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install system packages | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y pkg-config build-essential | |
| sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.d/development.txt | |
| - name: Build Borg with ASan/UBSan | |
| # Build the C/Cython extensions with AddressSanitizer and UndefinedBehaviorSanitizer enabled. | |
| # How this works: | |
| # - The -fsanitize=address,undefined flags inject runtime checks into our native code. If a bug is hit | |
| # (e.g., buffer overflow, use-after-free, out-of-bounds, or undefined behavior), the sanitizer prints | |
| # a detailed error report to stderr, including a stack trace, and forces the process to exit with | |
| # non-zero status. In CI, this will fail the step/job so you will notice. | |
| # - ASAN_OPTIONS/UBSAN_OPTIONS configure the sanitizers' runtime behavior (see below for meanings). | |
| env: | |
| CFLAGS: "-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined" | |
| CXXFLAGS: "-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined" | |
| LDFLAGS: "-fsanitize=address,undefined" | |
| # ASAN_OPTIONS controls AddressSanitizer runtime tweaks: | |
| # - detect_leaks=0: Disable LeakSanitizer to avoid false positives with CPython/pymalloc in short-lived tests. | |
| # - strict_string_checks=1: Make invalid string operations (e.g., over-reads) more likely to be detected. | |
| # - check_initialization_order=1: Catch uses that depend on static initialization order (C++). | |
| # - detect_stack_use_after_return=1: Detect stack-use-after-return via stack poisoning (may increase overhead). | |
| ASAN_OPTIONS: "detect_leaks=0:strict_string_checks=1:check_initialization_order=1:detect_stack_use_after_return=1" | |
| # UBSAN_OPTIONS controls UndefinedBehaviorSanitizer runtime: | |
| # - print_stacktrace=1: Include a stack trace for UB reports to ease debugging. | |
| # Note: UBSan is recoverable by default (process may continue after reporting). If you want CI to | |
| # abort immediately and fail on the first UB, add `halt_on_error=1` (e.g., UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1"). | |
| UBSAN_OPTIONS: "print_stacktrace=1" | |
| # PYTHONDEVMODE enables additional Python runtime checks and warnings. | |
| PYTHONDEVMODE: "1" | |
| run: pip install -e . | |
| - name: Run tests under sanitizers | |
| env: | |
| ASAN_OPTIONS: "detect_leaks=0:strict_string_checks=1:check_initialization_order=1:detect_stack_use_after_return=1" | |
| UBSAN_OPTIONS: "print_stacktrace=1" | |
| PYTHONDEVMODE: "1" | |
| # Ensure the ASan runtime is loaded first to avoid "ASan runtime does not come first" warnings. | |
| # We discover libasan/libubsan paths via gcc and preload them for the Python test process. | |
| # the remote tests are slow and likely won't find anything useful | |
| run: | | |
| set -euo pipefail | |
| export LD_PRELOAD="$(gcc -print-file-name=libasan.so):$(gcc -print-file-name=libubsan.so)" | |
| echo "Using LD_PRELOAD=$LD_PRELOAD" | |
| pytest -v --benchmark-skip -k "not remote" | |
| posix_tests: | |
| needs: [lint] | |
| permissions: | |
| contents: read | |
| id-token: write | |
| attestations: write | |
| strategy: | |
| fail-fast: false | |
| # noinspection YAMLSchemaValidation | |
| matrix: >- | |
| ${{ fromJSON( | |
| github.event_name == 'pull_request' && '{ | |
| "include": [ | |
| {"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "py310-fuse2"}, | |
| {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-fuse3"} | |
| ] | |
| }' || '{ | |
| "include": [ | |
| {"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "py310-fuse2"}, | |
| {"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "py311-fuse2", "binary": "borg-linux-glibc235-x86_64-gh"}, | |
| {"os": "ubuntu-22.04-arm", "python-version": "3.11", "toxenv": "py311-fuse2", "binary": "borg-linux-glibc235-arm64-gh"}, | |
| {"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-fuse3"}, | |
| {"os": "ubuntu-24.04", "python-version": "3.13", "toxenv": "py313-fuse3"}, | |
| {"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-fuse3"}, | |
| {"os": "macos-13", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-13-x86_64-gh"}, | |
| {"os": "macos-14", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-14-arm64-gh"} | |
| ] | |
| }' | |
| ) }} | |
| env: | |
| TOXENV: ${{ matrix.toxenv }} | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 180 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # Just fetching one commit is not enough for setuptools-scm, so we fetch all | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Cache pip | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| ${{ runner.os }}- | |
| - name: Install Linux packages | |
| if: ${{ runner.os == 'Linux' }} | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev libzstd-dev pkg-config build-essential | |
| sudo apt-get install -y libxxhash-dev || true | |
| sudo apt-get install -y libfuse-dev fuse || true # Required for Python llfuse module | |
| sudo apt-get install -y libfuse3-dev fuse3 || true # Required for Python pyfuse3 module | |
| - name: Install macOS packages | |
| if: ${{ runner.os == 'macOS' }} | |
| run: brew bundle install | |
| - name: Install Python requirements | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| pip install -r requirements.d/development.txt | |
| - name: Install BorgBackup | |
| run: | | |
| pip install -ve . | |
| - name: Run pytest via tox | |
| run: | | |
| # Do not use fakeroot; run as root. Avoids the dreaded sporadic EISDIR failures; see #2482. | |
| #sudo -E bash -c "tox -e py" | |
| tox --skip-missing-interpreters | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| env: | |
| OS: ${{ runner.os }} | |
| python: ${{ matrix.python-version }} | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| env_vars: OS, python | |
| - name: Build Borg fat binaries (${{ matrix.binary }}) | |
| if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }} | |
| run: | | |
| pip install 'pyinstaller==6.14.2' | |
| mkdir -p dist/binary | |
| pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec | |
| - name: Smoke-test the built binary (${{ matrix.binary }}) | |
| if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }} | |
| run: | | |
| pushd dist/binary | |
| echo "single-file binary" | |
| chmod +x borg.exe | |
| ./borg.exe -V | |
| echo "single-directory binary" | |
| chmod +x borg-dir/borg.exe | |
| ./borg-dir/borg.exe -V | |
| tar czf borg.tgz borg-dir | |
| popd | |
| - name: Prepare binaries (${{ matrix.binary }}) | |
| if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }} | |
| run: | | |
| mkdir -p artifacts | |
| if [ -f dist/binary/borg.exe ]; then | |
| cp dist/binary/borg.exe artifacts/${{ matrix.binary }} | |
| fi | |
| if [ -f dist/binary/borg.tgz ]; then | |
| cp dist/binary/borg.tgz artifacts/${{ matrix.binary }}.tgz | |
| fi | |
| echo "binary files" | |
| ls -l artifacts/ | |
| - name: Attest binaries provenance (${{ matrix.binary }}) | |
| if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }} | |
| uses: actions/attest-build-provenance@v3 | |
| with: | |
| subject-path: 'artifacts/*' | |
| - name: Upload binaries (${{ matrix.binary }}) | |
| if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.binary }} | |
| path: artifacts/* | |
| if-no-files-found: error |