python-build-and-release-package.yml

name: Python - build and release package

on:
  workflow_dispatch:
  push:
    branches:
      - "main"
    tags:
      # Note: These need to match what specified in env.[TEST_]RELEASE_TAG_PREFIX below.
      - "python-v*"
      - "python-test-v*"
  pull_request:
    paths:
      - "python/**"
      - "rust/**"
      - "tests_data/**"
      - ".github/workflows/python-build-and-release-package.yml"
  schedule:
    - cron: "12 3 * * 4" # Run everything once per week.
    - cron: "12 3 * * 1" # Refresh the cache an additional time.
  release:
    types: [created]

permissions:
  contents: read

env:
  # Trigger for publishing to pypi and testpypi (and for pre-release checks
  # enforcement).
  RELEASE_TAG_PREFIX: "python-v"
  TEST_RELEASE_TAG_PREFIX: "python-test-v"
  UV_VERSION: "0.9.5"

jobs:
  # This job acts as a gatekeeper for releases, which are triggered by a tag
  # push. It performs critical pre-release checks. These checks are skipped for
  # non-release pushes.
  pre-release-checks:
    name: Pre-release checks
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v5
      - name: Setup python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # pin@v5
        with:
          python-version: "3.12"
      - name: Install uv
        run: curl -LsSf https://astral.sh/uv/${{ env.UV_VERSION }}/install.sh | sh
      - name: Check package for release
        env:
          IS_RELEASE_TAG: ${{ github.event_name == 'push' && github.ref_type == 'tag' && (startsWith(github.ref_name, env.RELEASE_TAG_PREFIX) || startsWith(github.ref_name, env.TEST_RELEASE_TAG_PREFIX)) }}
        run: |
          if [[ "${{ env.IS_RELEASE_TAG }}" == 'true' ]]; then
            FULL_TAG_REF="${{ github.ref }}"
            TAG_NAME="${{ github.ref_name }}"
            if [[ "${TAG_NAME}" == "${{ env.RELEASE_TAG_PREFIX }}"* ]]; then
              PREFIX_TO_REMOVE="refs/tags/${{ env.RELEASE_TAG_PREFIX }}"
            else
              PREFIX_TO_REMOVE="refs/tags/${{ env.TEST_RELEASE_TAG_PREFIX }}"
            fi
            TAG_VERSION="${FULL_TAG_REF#${PREFIX_TO_REMOVE}}"
            CHECKER_OPTIONS="--expected-version ${TAG_VERSION}"
          else
            CHECKER_OPTIONS="--report-only"
          fi
          # Note: this uses the magika python package installed via uv. Also,
          # pip is not available here, so we skip pip show check.
          uv run ./scripts/pre_release_check.py $CHECKER_OPTIONS --no-check-pip-show-package-version
        working-directory: python

  build-wheels:
    needs: [pre-release-checks]
    runs-on: ${{ matrix.platform.runner }}
    strategy:
      matrix:
        platform:
          - runner: ubuntu-latest
            target: x86_64
          - runner: windows-latest
            target: x64
          - runner: macos-14
            target: aarch64
    steps:
      - uses: actions/checkout@v5
      - name: Setup Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # pin@v5
        with:
          python-version: "3.12"
      - name: Install uv
        run: curl -LsSf https://astral.sh/uv/${{ env.UV_VERSION }}/install.sh | sh
      - if: matrix.platform.runner == 'ubuntu-latest'
        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
        with:
          path: rust/onnx/runtime/build/Linux
          key: maturin-${{ matrix.platform.target }}-${{ hashFiles('rust/onnx/build.sh') }}
      - run: python3 ./python/scripts/fix_package_version.py
      - if: matrix.platform.runner == 'ubuntu-latest'
        name: Build wheels
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.platform.target }}
          args: --release --out=../dist
          before-script-linux: "${{ github.workspace }}/rust/onnx/maturin.sh"
          manylinux: 2_28
          working-directory: python
      - if: matrix.platform.runner != 'ubuntu-latest'
        name: Build wheels
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.platform.target }}
          args: --release --out=../dist
          working-directory: python
      - name: Upload wheels
        uses: actions/upload-artifact@v5
        with:
          name: wheel-${{ matrix.platform.runner }}-${{ matrix.platform.target }}
          path: dist

  # Download, install, and test the wheels with different versions of python
  test-wheels:
    needs: [build-wheels]
    runs-on: ${{ matrix.platform.runner }}
    if: github.event.schedule != '12 3 * * 1'
    strategy:
      # We want to know in which exact situation the tests fail
      fail-fast: false
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
        platform:
          - runner: ubuntu-latest
            target: x86_64
          - runner: windows-latest
            target: x64
          - runner: macos-14
            target: aarch64
    steps:
      - uses: actions/checkout@v5
      - uses: actions/download-artifact@v4
        with:
          name: wheel-${{ matrix.platform.runner }}-${{ matrix.platform.target }}
          path: dist
      - name: Setup Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # pin@v5
        with:
          python-version: "${{ matrix.python-version }}"
      - name: Install uv
        run: curl -LsSf https://astral.sh/uv/${{ env.UV_VERSION }}/install.sh | sh
      # Attempt "uv add magika.whl", in a temporary directory
      - name: Check that `uv add magika.whl` works
        shell: bash
        run: |
          mkdir /tmp/test-uv
          cp -vR dist/*.whl /tmp/test-uv
          cd /tmp/test-uv
          uv init
          uv add ./$(\ls -1 *.whl | head -n 1)
      # From now on, magika will be available in the global environment
      - name: Install the wheel via pip
        run: python3 -m pip install $(python -c "import glob; print(glob.glob('dist/*.whl')[0])")

      - run: magika --version
      - run: "python3 -c 'import magika; m = magika.Magika(); print(m)'"
      - run: magika -r tests_data/basic
      - run: python3 ./python/scripts/run_quick_test_magika_cli.py
      - run: python3 ./python/scripts/run_quick_test_magika_module.py
      - name: Check package for release readiness
        env:
          IS_RELEASE_TAG: ${{ github.event_name == 'push' && github.ref_type == 'tag' && (startsWith(github.ref_name, env.RELEASE_TAG_PREFIX) || startsWith(github.ref_name, env.TEST_RELEASE_TAG_PREFIX)) }}
        shell: bash
        run: |
          if [[ "${{ env.IS_RELEASE_TAG }}" == 'true' ]]; then
            FULL_TAG_REF="${{ github.ref }}"
            TAG_NAME="${{ github.ref_name }}"
            if [[ "${TAG_NAME}" == "${{ env.RELEASE_TAG_PREFIX }}"* ]]; then
              PREFIX_TO_REMOVE="refs/tags/${{ env.RELEASE_TAG_PREFIX }}"
            else
              PREFIX_TO_REMOVE="refs/tags/${{ env.TEST_RELEASE_TAG_PREFIX }}"
            fi
            TAG_VERSION="${FULL_TAG_REF#${PREFIX_TO_REMOVE}}"
            CHECKER_OPTIONS="--expected-version ${TAG_VERSION}"
          else
            CHECKER_OPTIONS="--report-only"
          fi
          # Note: this uses the magika python package that was just built.
          python3 ./scripts/pre_release_check.py $CHECKER_OPTIONS
        working-directory: python

  build-pure-python-wheel-and-sdist:
    needs: [pre-release-checks]
    runs-on: ubuntu-latest
    if: github.event.schedule != '12 3 * * 1'
    steps:
      - uses: actions/checkout@v5
      - name: Install uv
        run: curl -LsSf https://astral.sh/uv/${{ env.UV_VERSION }}/install.sh | sh
      - run: uv run ./scripts/prepare_pyproject_for_pure_python_wheel.py
        working-directory: python
      - name: Build pure python wheel and source distribution
        run: uv build --out-dir ../dist
        working-directory: python
      - name: Upload pure python wheel
        uses: actions/upload-artifact@v5
        with:
          name: wheel-pure-python
          path: dist/*.whl
      - name: Upload sdist
        uses: actions/upload-artifact@v5
        with:
          name: sdist
          path: dist/*.tar.gz

  # Download, install, and test the pure python wheel on multiple platforms
  test-pure-python-wheel:
    needs: [build-pure-python-wheel-and-sdist]
    runs-on: ${{ matrix.platform.runner }}
    if: github.event.schedule != '12 3 * * 1'
    strategy:
      # We want to know in which exact situation the tests fail
      fail-fast: false
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
        platform:
          - runner: ubuntu-latest
            target: x86_64
          - runner: windows-latest
            target: x64
          - runner: macos-14
            target: aarch64
    steps:
      - uses: actions/checkout@v5
      - uses: actions/download-artifact@v4
        with:
          name: wheel-pure-python
          path: dist
      - name: Setup Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # pin@v5
        with:
          python-version: "${{ matrix.python-version }}"
      - name: Install uv
        run: curl -LsSf https://astral.sh/uv/${{ env.UV_VERSION }}/install.sh | sh
      # Attempt "uv add magika.whl", in a temporary directory
      - name: Check that `uv add magika.whl` works
        shell: bash
        run: |
          mkdir /tmp/test-uv
          cp -vR dist/*.whl /tmp/test-uv
          cd /tmp/test-uv
          uv init
          uv add ./$(\ls -1 *.whl | head -n 1)
      # From now on, magika will be available in the global environment
      - name: Install the wheel
        run: python3 -m pip install $(python -c "import glob; print(glob.glob('dist/*.whl')[0])")

      # Check that the magika script points to the placeholder raising a warning
      - run: magika --version | grep -C10 WARNING | grep -C10 magika-python-client
      # Check that the fallback magika's python client can be run
      - run: magika-python-client -r tests_data/basic
      # Check that the results of the python's client are correct
      - run: python3 ./python/scripts/run_quick_test_magika_cli.py --client-path magika-python-client
      # Test the python module
      - run: "python3 -c 'import magika; m = magika.Magika(); print(m)'"
      - run: python3 ./python/scripts/run_quick_test_magika_module.py
      - name: Check package for release readiness
        env:
          IS_RELEASE_TAG: ${{ github.event_name == 'push' && github.ref_type == 'tag' && (startsWith(github.ref_name, env.RELEASE_TAG_PREFIX) || startsWith(github.ref_name, env.TEST_RELEASE_TAG_PREFIX)) }}
        shell: bash
        run: |
          if [[ "${{ env.IS_RELEASE_TAG }}" == 'true' ]]; then
            FULL_TAG_REF="${{ github.ref }}"
            TAG_NAME="${{ github.ref_name }}"
            if [[ "${TAG_NAME}" == "${{ env.RELEASE_TAG_PREFIX }}"* ]]; then
              PREFIX_TO_REMOVE="refs/tags/${{ env.RELEASE_TAG_PREFIX }}"
            else
              PREFIX_TO_REMOVE="refs/tags/${{ env.TEST_RELEASE_TAG_PREFIX }}"
            fi
            TAG_VERSION="${FULL_TAG_REF#${PREFIX_TO_REMOVE}}"
            CHECKER_OPTIONS="--expected-version ${TAG_VERSION}"
          else
            CHECKER_OPTIONS="--report-only"
          fi
          # Note: this uses the magika python package that was just built.
          python3 ./scripts/pre_release_check.py $CHECKER_OPTIONS --use-python-client
        working-directory: python

  test-sdist:
    needs: [build-pure-python-wheel-and-sdist]
    runs-on: ${{ matrix.platform.runner }}
    if: github.event.schedule != '12 3 * * 1'
    strategy:
      # We want to know in which exact situation the tests fail
      fail-fast: false
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
        platform:
          - runner: ubuntu-latest
            target: x86_64
          - runner: windows-latest
            target: x64
          - runner: macos-14
            target: aarch64
    steps:
      - uses: actions/checkout@v5
      - uses: actions/download-artifact@v4
        with:
          name: sdist
          path: dist
      - name: Setup Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # pin@v5
        with:
          python-version: "${{ matrix.python-version }}"
      - name: Install uv
        run: curl -LsSf https://astral.sh/uv/${{ env.UV_VERSION }}/install.sh | sh
      # Attempt "uv add magika.whl", in a temporary directory
      - name: Check that `uv add magika.tar.gz` works
        shell: bash
        run: |
          mkdir /tmp/test-uv
          cp -vR dist/*.tar.gz /tmp/test-uv
          cd /tmp/test-uv
          uv init
          uv add ./$(\ls -1 *.tar.gz | head -n 1)
      # From now on, magika will be available in the global environment
      - name: Install the sdist
        run: python3 -m pip install $(python -c "import glob; print(glob.glob('dist/*.tar.gz')[0])")

      # Check that the magika script points to the placeholder raising a warning
      - run: magika --version | grep -C10 WARNING | grep -C10 magika-python-client
      # Check that the fallback magika's python client can be run
      - run: magika-python-client -r tests_data/basic
      # Check that the results of the python's client are correct
      - run: python3 ./python/scripts/run_quick_test_magika_cli.py --client-path magika-python-client
      # Test the python module
      - run: "python3 -c 'import magika; m = magika.Magika(); print(m)'"
      - run: python3 ./python/scripts/run_quick_test_magika_module.py
      - name: Check package for release readiness
        env:
          IS_RELEASE_TAG: ${{ github.event_name == 'push' && github.ref_type == 'tag' && (startsWith(github.ref_name, env.RELEASE_TAG_PREFIX) || startsWith(github.ref_name, env.TEST_RELEASE_TAG_PREFIX)) }}
        shell: bash
        run: |
          if [[ "${{ env.IS_RELEASE_TAG }}" == 'true' ]]; then
            FULL_TAG_REF="${{ github.ref }}"
            TAG_NAME="${{ github.ref_name }}"
            if [[ "${TAG_NAME}" == "${{ env.RELEASE_TAG_PREFIX }}"* ]]; then
              PREFIX_TO_REMOVE="refs/tags/${{ env.RELEASE_TAG_PREFIX }}"
            else
              PREFIX_TO_REMOVE="refs/tags/${{ env.TEST_RELEASE_TAG_PREFIX }}"
            fi
            TAG_VERSION="${FULL_TAG_REF#${PREFIX_TO_REMOVE}}"
            CHECKER_OPTIONS="--expected-version ${TAG_VERSION}"
          else
            CHECKER_OPTIONS="--report-only"
          fi
          # Note: this uses the magika python package that was just built.
          python3 ./scripts/pre_release_check.py $CHECKER_OPTIONS --use-python-client
        working-directory: python

  # Adapted from https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
  # Note: The publishing is only done with pushes of release tags.
  publish-to-pypi:
    name: Publish to PyPI
    if: github.event_name == 'push' && github.ref_type == 'tag' && contains(github.ref_name, 'python')
    needs: [test-wheels, test-pure-python-wheel, test-sdist]
    runs-on: ubuntu-latest

    environment:
      name: pypi
      url: https://pypi.org/p/magika

    permissions:
      id-token: write  # IMPORTANT: mandatory for trusted publishing

    steps:
      - name: Download all the artifacts (binary wheels, pure python wheel, sdist)
        uses: actions/download-artifact@v4
        with:
          path: artifacts/

      - name: Flatten artifacts structure
        run: |
          # List all files for debugging
          ls -alR artifacts/

          # Find all files inside the subdirectories and move them up
          find artifacts/ -mindepth 2 -type f -exec mv -t artifacts/ {} +

          # Remove the now-empty subdirectories
          find artifacts/ -mindepth 1 -type d -empty -delete

          # Check structure after flattening
          ls -alR artifacts/

      - name: Publish distribution to PyPI
        if: github.event_name == 'push' && github.ref_type == 'tag' && startsWith(github.ref_name, env.RELEASE_TAG_PREFIX)
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          packages-dir: artifacts/

  # Note: The publishing is only done with pushes of test release tags.
  publish-to-testpypi:
    name: Publish to TestPyPI
    if: github.event_name == 'push' && github.ref_type == 'tag' && contains(github.ref_name, 'python')
    needs: [test-wheels, test-pure-python-wheel, test-sdist]
    runs-on: ubuntu-latest

    environment:
      name: testpypi
      url: https://test.pypi.org/p/magika

    permissions:
      id-token: write  # IMPORTANT: mandatory for trusted publishing

    steps:
      - name: Download all the artifacts (binary wheels, pure python wheel, sdist)
        uses: actions/download-artifact@v4
        with:
          path: artifacts/

      - name: Flatten artifacts structure
        run: |
          # List all files for debugging
          ls -alR artifacts/

          # Find all files inside the subdirectories and move them up
          find artifacts/ -mindepth 2 -type f -exec mv -t artifacts/ {} +

          # Remove the now-empty subdirectories
          find artifacts/ -mindepth 1 -type d -empty -delete

          # Check structure after flattening
          ls -alR artifacts/

      - name: Publish distribution to TestPyPI
        if: github.event_name == 'push' && github.ref_type == 'tag' && startsWith(github.ref_name, env.TEST_RELEASE_TAG_PREFIX)
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          packages-dir: artifacts/
          repository-url: https://test.pypi.org/legacy/