ci.yml

# This workflow runs the pipeline tests and benchmark-report checks
name: Pipeline CI
on:
  pull_request:
  release:
    types: [published]
  merge_group:
    types:
      - checks_requested
    branches:
      - master
      - dev

env:
  NXF_ANSI_LOG: false
  NFT_VER: "0.9.2"
  NFT_WORKDIR: "~"
  NFT_DIFF: "pdiff"
  NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2"
  TOWER_ACCESS_TOKEN: ${{ secrets.TOWER_ACCESS_TOKEN }}

concurrency:
  group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
  cancel-in-progress: true

jobs:
  changes:
    name: Check for changes
    runs-on: ubuntu-latest
    outputs:
      nf_test_files: ${{ steps.list.outputs.components }}
      benchmark_pytest: ${{ steps.benchmark.outputs.benchmark_pytest }}
      benchmark_smoke: ${{ steps.benchmark.outputs.benchmark_smoke }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: List nf-test files
        id: list
        uses: adamrtalbot/detect-nf-test-changes@v0.0.3
        with:
          head: ${{ github.sha }}
          base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.sha }}
          include: .github/include.yaml

      - name: print list of nf-test files
        run: |
          echo ${{ steps.list.outputs.components }}

      - name: Detect benchmark report changes
        id: benchmark
        shell: bash
        env:
          EVENT_NAME: ${{ github.event_name }}
          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
          MERGE_GROUP_BASE_SHA: ${{ github.event.merge_group.base_sha }}
          HEAD_SHA: ${{ github.sha }}
        run: |
          if [[ "$EVENT_NAME" == "pull_request" ]]; then
            BASE_SHA="$PR_BASE_SHA"
          elif [[ "$EVENT_NAME" == "merge_group" ]]; then
            BASE_SHA="$MERGE_GROUP_BASE_SHA"
          else
            echo "benchmark_pytest=false" >> "$GITHUB_OUTPUT"
            echo "benchmark_smoke=false" >> "$GITHUB_OUTPUT"
            echo "Benchmark report checks are disabled for event: $EVENT_NAME"
            exit 0
          fi

          echo "Comparing benchmark-report surfaces between $BASE_SHA and $HEAD_SHA"
          CHANGED_FILES="$(git diff --name-only "$BASE_SHA" "$HEAD_SHA")"
          export CHANGED_FILES GITHUB_OUTPUT

          python <<'PY'
          import fnmatch
          import os

          changed = [line.strip() for line in os.environ.get("CHANGED_FILES", "").splitlines() if line.strip()]
          pytest_patterns = [
              ".github/workflows/ci.yml",
              ".github/include.yaml",
              "bin/aggregate_benchmark_report_data.py",
              "bin/benchmark_report.py",
              "bin/benchmark_report_aggregate.py",
              "bin/benchmark_report_normalize.py",
              "bin/benchmark_report_render.py",
              "bin/benchmark_report_template.html",
              "bin/normalize_benchmark_jsonl.py",
              "bin/render_benchmark_report.py",
              "bin/test_benchmark_report_fetch.py",
              "conftest.py",
              "modules/local/aggregate_benchmark_report_data/**",
              "modules/local/normalize_benchmark_jsonl/**",
              "modules/local/render_benchmark_report/**",
              "scripts/build_filtered_cost_sidecar.py",
              "tests/pipeline_benchmark_*/**",
              "workflows/nf_aggregate/assets/log_dirs/**",
              "workflows/nf_aggregate/assets/logs/**",
              "workflows/nf_aggregate/assets/realworld_log_dirs/**",
              "workflows/nf_aggregate/assets/test_benchmark*.csv",
              "workflows/nf_aggregate/assets/test_benchmark*.parquet",
          ]
          smoke_patterns = pytest_patterns + [
              "assets/brand.yml",
              "assets/schema_input.json",
              "main.nf",
              "modules/local/extract_tarball/**",
              "nextflow.config",
              "nextflow_schema.json",
              "workflows/nf_aggregate/main.nf",
              "workflows/nf_aggregate/nextflow.config",
          ]

          def matched_files(patterns):
              return sorted({
                  path for path in changed
                  if any(fnmatch.fnmatch(path, pattern) for pattern in patterns)
              })

          pytest_matches = matched_files(pytest_patterns)
          smoke_matches = matched_files(smoke_patterns)

          print("Benchmark-report pytest matches:")
          if pytest_matches:
              for path in pytest_matches:
                  print(f"  - {path}")
          else:
              print("  (none)")

          print("Benchmark-report smoke matches:")
          if smoke_matches:
              for path in smoke_matches:
                  print(f"  - {path}")
          else:
              print("  (none)")

          with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as fh:
              fh.write(f"benchmark_pytest={'true' if pytest_matches else 'false'}\n")
              fh.write(f"benchmark_smoke={'true' if smoke_matches else 'false'}\n")
          PY

  test:
    name: ${{ matrix.nf_test_files }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }}
    needs: [changes]
    if: needs.changes.outputs.nf_test_files != '[]'
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        NXF_VER:
          - "latest-everything"
        nf_test_files: ["${{ fromJson(needs.changes.outputs.nf_test_files) }}"]
        profile:
          - "docker"

    steps:
      - name: Check out pipeline code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1

      - name: Install Nextflow
        uses: nf-core/setup-nextflow@v2
        with:
          version: "${{ matrix.NXF_VER }}"

      - uses: actions/setup-python@v4
        with:
          python-version: "3.11"
          architecture: "x64"

      - name: Install pdiff to see diff between nf-test snapshots
        run: |
          python -m pip install --upgrade pip
          pip install pdiff

      - name: Cache nf-test installation
        id: cache-software
        uses: actions/cache@v3
        with:
          path: |
            /usr/local/bin/nf-test
            /home/runner/.nf-test/nf-test.jar
          key: ${{ runner.os }}-${{ env.NFT_VER }}-nftest

      - name: Install nf-test
        if: steps.cache-software.outputs.cache-hit != 'true'
        run: |
          wget -qO- https://code.askimed.com/install/nf-test | bash
          sudo mv nf-test /usr/local/bin/

      - name: Run nf-test
        run: |
          nf-test test --verbose ${{ matrix.nf_test_files }} --profile "+${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap

      - uses: pcolby/tap-summary@v1
        with:
          path: >-
            test.tap

      - name: Output log on failure
        if: failure()
        run: |
          sudo apt install bat > /dev/null
          batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log

      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v3
        if: always() # always run even if the previous step fails
        with:
          report_paths: test.xml

  benchmark-report-pytest:
    name: Benchmark report pytest
    needs: [changes]
    if: needs.changes.outputs.benchmark_pytest == 'true'
    runs-on: ubuntu-latest
    steps:
      - name: Check out pipeline code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1

      - uses: actions/setup-python@v4
        with:
          python-version: "3.12"
          architecture: "x64"

      - name: Install uv
        run: |
          python -m pip install --upgrade pip
          pip install uv

      - name: Run benchmark report pytest suites
        run: |
          uv run --with typer --with pyyaml --with jinja2 --with pyarrow --with pytest --with httpx pytest -v modules/local/aggregate_benchmark_report_data/tests/test_aggregate.py modules/local/normalize_benchmark_jsonl/tests/test_normalize.py modules/local/render_benchmark_report/tests/test_render.py bin/test_benchmark_report_fetch.py

  benchmark-report-smoke:
    name: Benchmark report smoke test
    needs: [changes]
    if: needs.changes.outputs.benchmark_smoke == 'true'
    runs-on: ubuntu-latest
    steps:
      - name: Check out pipeline code
        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1

      - name: Install Nextflow
        uses: nf-core/setup-nextflow@v2
        with:
          version: "latest-everything"

      - name: Run offline benchmark smoke test
        run: |
          cat > benchmark-smoke-params.json <<'JSON'
          {
            "input": "workflows/nf_aggregate/assets/test_benchmark.csv",
            "generate_benchmark_report": true,
            "outdir": "benchmark-smoke-results"
          }
          JSON
          nextflow run . -params-file benchmark-smoke-params.json -profile docker

  confirm-pass:
    runs-on: ubuntu-latest
    needs:
      - changes
      - test
      - benchmark-report-pytest
      - benchmark-report-smoke
    if: always()
    steps:
      - name: All tests ok
        if: ${{ !contains(needs.*.result, 'failure') }}
        run: exit 0
      - name: One or more tests failed
        if: ${{ contains(needs.*.result, 'failure') }}
        run: exit 1

      - name: debug-print
        if: always()
        run: |
          echo "toJSON(needs) = ${{ toJSON(needs) }}"
          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"