ci.yaml

name: CI

on:
  push:
    branches:
      - main
    tags:
      - "v[0-9]+.[0-9]+.[0-9]+"
      - "v[0-9]+.[0-9]+.[0-9]+-*"
    paths:
      - "api/**"
      - "*.go"
      - "**/*.go"
      - "cmd/**"
      - "go.mod"
      - "go.sum"
      - "internal/**"
      - "Makefile"
      - "proto/**"
      - "scripts/e2e/**"
      - "ui/**"
      - ".github/workflows/ci.yaml"
  pull_request:
    # Keep this workflow directly rerunnable from PR-only maintenance commits.
    types:
      - opened
      - reopened # new commits are pushed to the branch that the PR is based on
      - synchronize # new commits are pushed to the branch that the PR is based on
      - ready_for_review # PR is ready for review
    paths:
      - "api/**"
      - "*.go"
      - "**/*.go"
      - "cmd/**"
      - "go.mod"
      - "go.sum"
      - "internal/**"
      - "Makefile"
      - "proto/**"
      - "scripts/e2e/**"
      - "ui/**"
      - ".github/workflows/ci.yaml"
  workflow_dispatch:

permissions:
  contents: read

env:
  GO_VERSION: "1.26"
  NODE_VERSION: "20"

jobs:
  changes:
    name: Detect changes
    runs-on: ubuntu-latest
    outputs:
      go: ${{ steps.filter.outputs.go }}
      e2e: ${{ steps.filter.outputs.e2e }}
    steps:
      - name: Check out code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0

      - name: Detect changed paths
        id: filter
        shell: bash
        run: |
          if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then
            echo "go=true" >> "${GITHUB_OUTPUT}"
            echo "e2e=true" >> "${GITHUB_OUTPUT}"
            exit 0
          fi

          base_sha=""
          if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
            base_sha="${{ github.event.pull_request.base.sha }}"
          elif [[ "${GITHUB_EVENT_NAME}" == "push" ]]; then
            base_sha="${{ github.event.before }}"
          fi

          if [[ -z "${base_sha}" || "${base_sha}" == "0000000000000000000000000000000000000000" ]]; then
            echo "go=true" >> "${GITHUB_OUTPUT}"
            echo "e2e=true" >> "${GITHUB_OUTPUT}"
            exit 0
          fi

          changed_files="$(git diff --name-only "${base_sha}" "${GITHUB_SHA}")"
          printf '%s\n' "${changed_files}"

          go_changed=false
          e2e_changed=false
          while IFS= read -r file; do
            case "${file}" in
              *.go|go.mod|go.sum|Makefile|.github/workflows/ci.yaml)
                go_changed=true
                ;;
            esac

            case "${file}" in
              api/*|cmd/*|internal/*|proto/*|scripts/e2e/*|ui/*|go.mod|go.sum|Makefile|.github/workflows/ci.yaml)
                e2e_changed=true
                ;;
            esac
          done <<< "${changed_files}"

          echo "go=${go_changed}" >> "${GITHUB_OUTPUT}"
          echo "e2e=${e2e_changed}" >> "${GITHUB_OUTPUT}"

  # Lint Go code
  golint:
    name: Go Linter
    needs: changes
    if: needs.changes.outputs.go == 'true'
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up Go
        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Check formatting (go fmt)
        run: |
          go fmt ./...
          # Fail if formatting changed any files
          git diff --exit-code || (echo "Go files are not formatted. Please run 'make fmt'" && exit 1)

      - name: Check modernize (go fix)
        run: |
          diff=$(go fix -diff ./... 2>&1) || true
          if [ -n "$diff" ]; then
            echo "Go files need modernization. Please run 'make fmt'"
            echo "$diff"
            exit 1
          fi

      - name: Run golangci-lint
        uses: golangci/golangci-lint-action@1e7e51e771db61008b38414a730f564565cf7c20 # v9.2.0
        with:
          version: v2.9.0
          args: --timeout=10m

  test-ubuntu:
    name: Test on ubuntu-latest (${{ matrix.suite_index }}/${{ matrix.suite_total }})
    needs: changes
    if: needs.changes.outputs.go == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        include:
          - suite: unit
            suite_index: 1
            suite_total: 2
            codecov_name: ubuntu-unit
          - suite: intg
            suite_index: 2
            suite_total: 2
            codecov_name: ubuntu-intg
    steps:
      - name: Check out code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up Go
        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Login to Docker Hub
        if: ${{ matrix.suite == 'intg' && (github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.user.login != 'dependabot[bot]')) }}
        uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Pre-pull integration test images
        if: ${{ matrix.suite == 'intg' }}
        shell: bash
        run: |
          images=(
            "alpine:3"
            "busybox:latest"
            "redis:7-alpine"
            "minio/minio:RELEASE.2024-10-02T17-50-41Z"
          )

          for image in "${images[@]}"; do
            for attempt in 1 2 3 4; do
              if docker pull "${image}"; then
                break
              fi
              if [[ "${attempt}" -eq 4 ]]; then
                echo "failed to pull ${image}" >&2
                exit 1
              fi
              sleep "$((attempt * 5))"
            done
          done

      - name: Resolve test packages
        id: test-packages
        shell: bash
        run: |
          suite="${{ matrix.suite }}"
          module_path="$(go list -m)"
          case "${suite}" in
            unit)
              packages="$(
                go list ./... | awk -v prefix="${module_path}/internal/intg" '$0 != prefix && index($0, prefix "/") != 1' | xargs
              )"
              ;;
            intg)
              packages="$(go list ./internal/intg/... | xargs)"
              ;;
            *)
              echo "unknown test suite: ${suite}" >&2
              exit 1
              ;;
          esac
          if [[ -z "${packages}" ]]; then
            echo "failed to resolve packages for ${suite}" >&2
            exit 1
          fi
          echo "packages=${packages}" >> "${GITHUB_OUTPUT}"

      - name: Build
        run: |
          make bin

      - name: Test
        run: |
          make test-coverage TEST_TARGET="${{ steps.test-packages.outputs.packages }}"

      - name: Upload coverage
        uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
        with:
          files: ./coverage.out
          flags: ${{ matrix.codecov_name }}
          name: ${{ matrix.codecov_name }}
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  e2e:
    name: E2E (${{ matrix.shard }}/${{ matrix.shard_total }})
    needs:
      - changes
      - golint
      - test-ubuntu
      - test-windows
    if: >-
      always() &&
      needs.changes.outputs.e2e == 'true' &&
      (needs.golint.result == 'success' || needs.golint.result == 'skipped') &&
      (needs.test-ubuntu.result == 'success' || needs.test-ubuntu.result == 'skipped') &&
      (needs.test-windows.result == 'success' || needs.test-windows.result == 'skipped')
    runs-on: ubuntu-latest
    timeout-minutes: 35
    strategy:
      fail-fast: false
      matrix:
        include:
          - shard: 1
            shard_total: 3
          - shard: 2
            shard_total: 3
          - shard: 3
            shard_total: 3
    concurrency:
      group: ci-e2e-${{ github.event.pull_request.number || github.ref }}-shard-${{ matrix.shard }}
      cancel-in-progress: true
    steps:
      - name: Check out code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up Go
        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Setup pnpm
        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0
        with:
          version: 10

      - name: Setup Node.js
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: "pnpm"
          cache-dependency-path: ui/pnpm-lock.yaml

      - name: Install frontend dependencies
        working-directory: ui
        run: pnpm install --frozen-lockfile

      - name: Install Playwright Chromium
        working-directory: ui
        run: pnpm exec playwright install --with-deps chromium

      - name: Run browser E2E tests
        env:
          CI: "true"
          DAGU_LICENSE_PRIVKEY_B64: ${{ secrets.DAGU_LICENSE_PRIVKEY_B64 }}
          PLAYWRIGHT_FULLY_PARALLEL: "true"
          PLAYWRIGHT_TEST_ARGS: --shard=${{ matrix.shard }}/${{ matrix.shard_total }}
        run: |
          make test-e2e-build
          make test-e2e-run

  test-windows:
    name: Test on windows-latest (${{ matrix.suite_index }}/${{ matrix.suite_total }})
    needs: changes
    if: needs.changes.outputs.go == 'true'
    runs-on: windows-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        include:
          - suite: unit
            suite_index: 1
            suite_total: 2
          - suite: intg
            suite_index: 2
            suite_total: 2
    steps:
      - name: Check out code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up Go
        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
        with:
          go-version: ${{ env.GO_VERSION }}

      - name: Resolve test packages
        id: test-packages
        shell: bash
        run: |
          suite="${{ matrix.suite }}"
          module_path="$(go list -m)"
          case "${suite}" in
            unit)
              packages="$(
                go list ./... | awk -v prefix="${module_path}/internal/intg" '$0 != prefix && index($0, prefix "/") != 1' | xargs
              )"
              ;;
            intg)
              packages="$(go list ./internal/intg/... | xargs)"
              ;;
            *)
              echo "unknown test suite: ${suite}" >&2
              exit 1
              ;;
          esac
          if [[ -z "${packages}" ]]; then
            echo "failed to resolve packages for ${suite}" >&2
            exit 1
          fi
          echo "packages=${packages}" >> "${GITHUB_OUTPUT}"

      - name: Build (Windows)
        shell: bash
        run: |
          mkdir -p ./.local/bin
          go build -o ./.local/bin/dagu.exe ./cmd

      - name: Test (Windows)
        shell: bash
        run: |
          go test \
            -timeout=20m \
            -ldflags=-extldflags=-Wl \
            ${{ steps.test-packages.outputs.packages }}