ux-tests.yml

name: Tests

# Two test tiers:
#
# 1. unit-tests — Fast (~30s), no infrastructure. Runs test/unit/ and test/cmd/
#    on every push. Catches logic bugs in decorators, CLI, parsers, etc.
#
# 2. ux-tests — Full integration across 4 backends (local, argo, airflow, sfn).
#    Needs devstack (minikube + tilt). Catches orchestrator/deployer bugs.
#
# Both tiers auto-discover tests by directory — new test files are picked up
# automatically without editing this workflow.
#
# Coverage is collected from each job, then combined in a final report.

on:
  push:
    branches:
      - master
      - "npow/**"  # run on this dev branch during development
  pull_request:
    branches:
      - master
  workflow_dispatch:

jobs:
  unit-tests:
    name: "Unit Tests"
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Cache pip packages
        uses: actions/cache@v4
        with:
          path: ~/.cache/pip
          key: pip-py3.9-${{ hashFiles('setup.py', 'setup.cfg') }}
          restore-keys: pip-py3.9-

      - name: Install Metaflow and test dependencies
        run: |
          pip install --upgrade pip
          pip install -e ".[dev]" pytest-timeout pytest-cov

      - name: Run unit and command tests
        run: |
          python -m pytest \
            test/unit/ test/cmd/ test/plugins/ \
            --ignore=test/unit/spin \
            -v \
            --tb=short \
            --timeout=120 \
            --cov=metaflow \
            --cov-report=term-missing \
            --cov-report=xml:coverage.xml \
            --cov-branch \
            --junit-xml=junit-unit.xml

      - name: Upload coverage data
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: coverage-unit
          path: |
            .coverage
            coverage.xml
          if-no-files-found: ignore
          include-hidden-files: true

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: junit-unit
          path: junit-unit.xml
          if-no-files-found: ignore

      - name: Publish test results
        if: always()
        uses: dorny/test-reporter@v1
        with:
          name: "Test Results — Unit"
          path: junit-unit.xml
          reporter: java-junit
          fail-on-error: false

  ux-tests:
    name: "${{ matrix.backend }}"
    strategy:
      fail-fast: false
      matrix:
        include:
          - backend: local
            services: "minio,postgresql,metadata-service"
            workers: 4
          - backend: argo-kubernetes
            services: "minio,postgresql,metadata-service,argo-workflows"
            workers: 2
          - backend: airflow-kubernetes
            services: "minio,postgresql,metadata-service,airflow"
            workers: 2
          - backend: sfn-batch
            services: "minio,postgresql,metadata-service,localbatch,ddb-local,sfn-local"
            workers: 4

    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Cache pip packages
        uses: actions/cache@v4
        with:
          path: ~/.cache/pip
          key: pip-py3.9-${{ hashFiles('setup.py', 'setup.cfg') }}
          restore-keys: pip-py3.9-

      - name: Install Metaflow and test dependencies
        run: |
          pip install --upgrade pip
          pip install -e ".[dev]" pytest-xdist pytest-timeout pytest-cov
          pip install "git+https://github.com/npow/localbatch.git@main#egg=localbatch"

      - name: Set up minikube
        uses: medyagh/setup-minikube@latest
        with:
          driver: docker
          cpus: 2
          memory: 6144

      - name: Restore minikube image cache
        id: image-cache
        uses: actions/cache/restore@v4
        with:
          path: /tmp/minikube-image-cache
          key: minikube-images-${{ matrix.backend }}-${{ hashFiles('devtools/Tiltfile') }}
          restore-keys: minikube-images-${{ matrix.backend }}-

      - name: Pre-load cached images into minikube
        if: steps.image-cache.outputs.cache-hit == 'true'
        run: devtools/ci/load-minikube-images.sh

      - name: Cache Helm repos
        uses: actions/cache@v4
        with:
          path: ~/.cache/helm
          key: helm-${{ hashFiles('devtools/Tiltfile') }}
          restore-keys: helm-

      - name: Set up Helm
        uses: azure/setup-helm@v4

      - name: Install Tilt
        run: |
          # Pin to the same version as devtools/Makefile to avoid extension API breaks
          curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh \
            | VERSION=v0.33.11 bash

      - name: Start devstack
        working-directory: devtools
        run: ci/start-devstack.sh
        env:
          SERVICES: ${{ matrix.services }}

      - name: Wait for test image to be loaded into minikube
        if: matrix.backend != 'local' && matrix.backend != 'sfn-batch'
        run: |
          tilt wait --for=condition=Ready uiresource/build-test-image --timeout=300s

      - name: Save minikube images to cache
        if: steps.image-cache.outputs.cache-hit != 'true'
        run: devtools/ci/save-minikube-images.sh

      - name: Store minikube image cache
        if: steps.image-cache.outputs.cache-hit != 'true'
        uses: actions/cache/save@v4
        with:
          path: /tmp/minikube-image-cache
          key: minikube-images-${{ matrix.backend }}-${{ hashFiles('devtools/Tiltfile') }}

      - name: Start minikube tunnel
        run: |
          sudo minikube tunnel &
          sleep 5

      - name: Forward devstack ports to Docker bridge (sfn-batch only)
        if: matrix.backend == 'sfn-batch'
        run: devtools/ci/forward-bridge-ports.sh

      - name: Wait for Airflow REST API
        if: matrix.backend == 'airflow-kubernetes'
        run: devtools/ci/wait-airflow-api.sh

      - name: Clean up completed pods (argo/airflow only)
        if: matrix.backend == 'argo-kubernetes' || matrix.backend == 'airflow-kubernetes'
        run: |
          kubectl delete pods --field-selector=status.phase=Succeeded --all-namespaces 2>/dev/null || true
          kubectl delete pods --field-selector=status.phase=Failed --all-namespaces 2>/dev/null || true

      - name: Run UX tests — ${{ matrix.backend }}
        run: |
          AWS_SHARED_CREDENTIALS_FILE="" \
          PYTHONPATH=$PWD \
          python -m pytest \
            test/ux/core/ \
            --only-backend "${{ matrix.backend }}" \
            -n ${{ matrix.workers }} \
            -v \
            --tb=short \
            --timeout=1800 \
            --cov=metaflow \
            --cov-report=term-missing \
            --cov-report=xml:coverage.xml \
            --cov-report=html:htmlcov \
            --cov-branch \
            --junit-xml=junit-${{ matrix.backend }}.xml

      - name: Upload coverage data
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: coverage-${{ matrix.backend }}
          path: |
            .coverage
            coverage.xml
            htmlcov/
          if-no-files-found: ignore
          include-hidden-files: true

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: junit-${{ matrix.backend }}
          path: junit-${{ matrix.backend }}.xml
          if-no-files-found: ignore

      - name: Publish test results
        if: always()
        uses: dorny/test-reporter@v1
        with:
          name: "Test Results — ${{ matrix.backend }}"
          path: junit-${{ matrix.backend }}.xml
          reporter: java-junit
          fail-on-error: false

      - name: Dump sfn-batch diagnostics on failure
        if: failure() && matrix.backend == 'sfn-batch'
        run: devtools/ci/dump-sfn-diagnostics.sh

      - name: Dump Airflow diagnostics on failure
        if: failure() && matrix.backend == 'airflow-kubernetes'
        run: devtools/ci/dump-airflow-diagnostics.sh

      - name: Show Tilt logs on failure
        if: failure()
        run: cat /tmp/tilt.log | tail -200 || true

      - name: Upload Tilt logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: tilt-logs-${{ matrix.backend }}
          path: /tmp/tilt.log
          if-no-files-found: ignore

  coverage-report:
    name: "Coverage Report"
    needs: [unit-tests, ux-tests]
    if: always()
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Install coverage
        run: pip install coverage[toml]

      - name: Download coverage data from all backends
        uses: actions/download-artifact@v4
        with:
          pattern: coverage-*
          path: coverage-artifacts/

      - name: Download test results from all backends
        uses: actions/download-artifact@v4
        with:
          pattern: junit-*
          path: junit-artifacts/

      - name: Publish combined test results
        uses: dorny/test-reporter@v1
        with:
          name: "Test Results — All Backends"
          path: "junit-artifacts/**/*.xml"
          reporter: java-junit
          fail-on-error: false

      - name: Combine coverage data
        run: devtools/ci/combine-coverage.sh

      - name: Generate combined HTML report
        run: |
          coverage html -d combined-htmlcov/ --title="Metaflow UX Test Coverage"
          coverage xml -o combined-coverage.xml

      - name: Post coverage summary
        run: |
          echo "## Test Coverage Summary" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          coverage report --sort=miss >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          total=$(coverage report | tail -1 | awk '{print $NF}')
          echo "**Total coverage: $total**" >> $GITHUB_STEP_SUMMARY

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v5
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: combined-coverage.xml
          name: ux-combined

      - name: Upload combined coverage report
        uses: actions/upload-artifact@v4
        with:
          name: coverage-combined
          path: |
            combined-htmlcov/
            combined-coverage.xml