oneshot-test.yml

# Workflow to test individual hooks across multiple versions of the
# library the are written for.
# This action is not run continuously. You must manually trigger it.
#
# This workflow features workflow_dispatch parameters:
# https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/
# And daisy-chaining the output of one job to dynamically set the matrix of a
# second job:
# https://stackoverflow.com/a/62953566/7390688
---

name: Oneshot test

on:
  workflow_dispatch:
    # Input parameters:
    inputs:
      package:
        description: |
          Package names and versions to test. Jobs are split by comma.
        required: true
        default: 'numpy==1.19, numpy<1.18'
      os:
        description: |
          OS(s) to run on. Can be any combination of ubuntu-latest, windows-latest,
          macos-latest (as well as specific versions, for example macos-15-intel).
          (Please use macOS runners sparingly.)'
        required: true
        default: 'ubuntu-latest'
      python-version:
        description: 'Version(s) of Python'
        required: true
        default: '3.11,'
      fail-fast:
        description: 'Terminate all tests if one fails (true/false).'
        required: true
        default: 'false'
      pyinstaller:
        description: 'Source URI from which PyInstaller should be installed.'
        required: true
        default: 'https://github.com/pyinstaller/pyinstaller/archive/develop.zip'
      commands:
        description: |
          Additional installation commands to run from terminal.
          Ran from bash irregardless of OS.
          Use ';' to separate multiple commands.
        required: false
      pytest_args:
        description: |
          Additional arguments to be passed to pytest.

env:
  # Colored pytest output on CI despite not having a tty
  FORCE_COLOR: 1
  # Enable strict unpack mode to catch file duplication problems in onefile builds (at executable run-time).
  PYINSTALLER_STRICT_UNPACK_MODE: 1
  # Enable strict collect mode to catch file duplication problems in PKG/Carchive (onefile builds) or COLLECT
  # (onedir builds) at build time.
  PYINSTALLER_STRICT_COLLECT_MODE: 1
  # Enable strict handling of codesign errors for macOS bundles.
  PYINSTALLER_STRICT_BUNDLE_CODESIGN_ERROR: 1
  # Enable strict verification of macOS bundles w.r.t. the code-signing requirements.
  PYINSTALLER_VERIFY_BUNDLE_SIGNATURE: 1

permissions: {}
jobs:
  generate-matrix:
    # Parse inputs into a json containing the matrix that will parametrize the
    # next "test" step.
    name: Generate Matrix
    runs-on: ubuntu-latest

    env:
      # Copy github.event.inputs into an environment variable INPUTS which can
      # be easily read in Python.
      INPUTS: ${{ toJson(github.event.inputs) }}

    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}

    steps:
      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: 3.11

      # Actually parse the configuration.
      - id: set-matrix
        shell: python
        run: |
          import os, json, pprint
          inputs = json.loads(os.environ["INPUTS"])
          pprint.pprint(inputs)

          # Split by comma, ignore trailing comma, remove whitespace.
          parse_list = lambda x: [i.strip() for i in x.strip(", ").split(",")]

          # Wrap a word in quotes, escaping any literal quotes already there.
          quote = lambda x: '"{}"'.format(x.replace('"', r'\"'))

          matrix = {
              "os": parse_list(inputs["os"]),
              "python-version": parse_list(inputs["python-version"]),
              "requirements": parse_list(inputs["package"]),
          }

          # Wrap each word in " " quotes to force bash to interpret special
          # characters such as > as literals.
          matrix["requirements"] = [
              " ".join(map(quote, i.split(" ")))
              for i in matrix["requirements"]
          ]

          pprint.pprint(matrix)

          # Outputs are set by appending a {name}={value} line to the
          # environment file pointed to by GITHUB_OUTPUT environment variable.
          with open(os.environ['GITHUB_OUTPUT'], 'a') as fp:
            fp.write("matrix=")
            json.dump(matrix, fp)
            fp.write("\n")

  test:
    permissions:
      contents: read # to fetch code (actions/checkout)

    needs: generate-matrix
    runs-on: ${{ matrix.os }}
    strategy:
      # Use the test matrix generated in the last step.
      matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
      # Caution: `fail-fast` expects a bool but `inputs.fail-fast` is a string.
      # There doesn't seem to be a nice function to cast 'true' to true.
      fail-fast: ${{ github.event.inputs.fail-fast == 'true' }}

    env:
      # Rebuild bootloader when installing PyInstaller from develop branch
      PYINSTALLER_COMPILE_BOOTLOADER: 1

    # Finally, the usual: setup Python, install dependencies, test.
    steps:
      - name: Checkout pyinstaller-hooks-contrib code
        uses: actions/checkout@v6

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}

      - name: Set up .NET Core for pythonnet tests
        uses: actions/setup-dotnet@v5
        with:
          dotnet-version: '6.x'

      - name: Run bash commands
        if: ${{ github.event.inputs.commands }}
        run: ${{ github.event.inputs.commands }}

      - name: Install dependencies
        shell: bash
        run: |
          # Upgrade to the latest pip.
          python -m pip install -U pip

          # Install hooks-contrib
          pip install -e .
          pip install -r requirements-test.txt
          pip install ${{ matrix.requirements }}

          # Install PyInstaller
          pip install ${{ github.event.inputs.pyinstaller }}

      # Relocate the temporary directory to a fixed location so we can
      # generate artifacts out of failed tests.
      - name: Relocate temporary dir
        shell: bash
        run: |
          echo "PYTEST_DEBUG_TEMPROOT=$RUNNER_TEMP" >> $GITHUB_ENV

      - name: Run tests
        id: run-tests
        run: pytest -v -n logical ${{ inputs.pytest_args }}

      # On all platforms, create a tarball to ensure that symlinks are preserved.
      # Avoid using compression here, as the tarball will end up collected into
      # artifact zip archive. To simplify this across platforms, run this step
      # in python and use python's tarfile module.
      - name: Archive failed tests
        if: ${{ failure() && steps.run-tests.outcome == 'failure' }}
        shell: python
        run: |
          import os
          import sys
          import tarfile

          try:
            import getpass
            user = getpass.getuser() or "unknown"
          except Exception:
            user = "unknown"

          temproot = os.environ['PYTEST_DEBUG_TEMPROOT']

          pytest_name = f'pytest-of-{user}'
          pytest_fullpath = os.path.join(temproot, pytest_name)
          print(f"Input directory: {pytest_fullpath}!", file=sys.stderr)

          output_file = os.path.join(temproot, 'archived-failed-tests.tar')
          print(f"Output file: {output_file}!", file=sys.stderr)

          assert os.path.isdir(pytest_fullpath)
          assert not os.path.exists(output_file)

          with tarfile.open(output_file, "w") as tf:
            tf.add(pytest_fullpath, arcname=pytest_name, recursive=True)

          print(f"Created {output_file}!", file=sys.stderr)

      - name: Create artifact out of archived failed tests
        if: ${{ failure() && steps.run-tests.outcome == 'failure' }}
        uses: actions/upload-artifact@v6
        with:
          name: failed-tests-${{ matrix.os }}-python-${{ matrix.python-version }}-matrix-idx-${{ strategy.job-index }}
          path: '${{ env.PYTEST_DEBUG_TEMPROOT }}/archived-failed-tests.tar'