ci.yaml

name: test
on:
  push:
    branches:
      - "main"
      - "release-*"
  pull_request:
    branches: [main]
jobs:
  ui:
    name: UI
    runs-on: ubuntu-24.04
    timeout-minutes: 6
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: "20"
      - name: Setup Node-Cache
        uses: actions/cache@v4
        with:
          path: ui/node_modules
          key: ${{ runner.os }}-node-dep-v1-${{ hashFiles('**/yarn.lock') }}
      - name: Yarn Build/Test
        run: make ui-test
      - name: Ensure nothing changed
        run: git diff --exit-code
  codegen:
    name: Codegen
    runs-on: ubuntu-24.04
    timeout-minutes: 5
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Restore go build cache
        uses: actions/cache@v4
        with:
          path: ~/.cache/go-build
          key: ${{ runner.os }}-go-build-v1-${{ hashFiles('**/go.mod') }}
      - name: Setup Golang
        uses: actions/setup-go@v5
        with:
          go-version: "1.24"
      - name: Add bins to PATH
        run: |
          echo /home/runner/go/bin >> $GITHUB_PATH
          echo /usr/local/bin >> $GITHUB_PATH
      - name: Get dependencies
        run: go mod download
      - name: Make codegen
        run: |
          echo 'GOPATH=/home/runner/go' >> $GITHUB_ENV
          make -B codegen
      - name: Ensure nothing changed
        run: git diff --exit-code

  unit-tests:
    name: Unit Tests
    runs-on: ubuntu-24.04
    timeout-minutes: 15
    steps:
      - name: Start JetStream
        run: docker run -d -p 4222:4222 nats:latest -js

      - name: Start Pulsar standalone container
        run: docker run -d -p 6650:6650 -p 8080:8080 apachepulsar/pulsar:4.0.0 bin/pulsar standalone

      - name: Start Kafka container
        run: docker run -d -p 9092:9092 --name kafka-broker apache/kafka:4.0.0

      - name: Start Redis standalone container
        run: docker run -d -p 6379:6379 --name redis-server redis:8-alpine

      - name: Set up Go 1.x
        uses: actions/setup-go@v5
        with:
          go-version: "1.24"
        id: go

      - name: Check out code
        uses: actions/checkout@v4

      - name: Restore Go build cache
        uses: actions/cache@v4
        with:
          path: ~/.cache/go-build
          key: ${{ runner.os }}-go-build-v1-${{ github.run_id }}

      - name: Get dependencies
        run: go mod download -x

      - name: Test Go
        run: make test-coverage

      - name: Install Rust
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          cache-workspaces: rust -> target

      - name: Install llvm-tools-preview
        working-directory: ./rust
        run: rustup component add llvm-tools-preview

      - name: Install grcov
        uses: taiki-e/install-action@v2
        with:
          tool: grcov

      - name: Install Protoc
        uses: arduino/setup-protoc@v3

      - name: Test Rust
        working-directory: ./rust
        run: |
          CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='./target/debug/coverage/cargo-test-%p-%m.profraw' make all-tests
          grcov . -s ./target/debug/coverage/ --binary-path ./target/debug/ -t lcov --branch --ignore-not-existing --ignore "**/build.rs" -o ./target/debug/coverage/lcov.info

      - name: Check Rust formatting
        working-directory: ./rust
        run: |
          cargo fmt -- --check

      - name: Check Clippy
        working-directory: ./rust
        run: |
          cargo clippy-ci

      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@v4
        with:
          files: ./test/profile.cov,./rust/target/debug/coverage/lcov.info
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  lint:
    name: Lint
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    env:
      GOPATH: /home/runner/go
    steps:
      - uses: actions/checkout@v4
      - name: Setup Golang
        uses: actions/setup-go@v5
        with:
          go-version: "1.24"
      - name: Restore Go build cache
        uses: actions/cache@v4
        with:
          path: ~/.cache/go-build
          key: ${{ runner.os }}-go-build-v1-${{ github.run_id }}
      - run: make lint
      - run: git diff --exit-code

  build-rust-amd64:
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v4
      # TODO: Use actions/cache@v4 to cache target/ directory
      - name: Build binary
        run: make build-rust-docker-ghactions
      - name: Rename binaries
        run: |
          cp -pv rust/target/x86_64-unknown-linux-gnu/release/numaflow ./numaflow-rs-linux-amd64
          cp -pv rust/target/x86_64-unknown-linux-gnu/release/entrypoint ./entrypoint-linux-amd64
      - name: Verify numaflow binary is Statically Linked
        run: |
          file ./numaflow-rs-linux-amd64
          file ./numaflow-rs-linux-amd64 | grep -q 'static-pie linked'
      - name: Verify entrypoint binary is Statically Linked
        run: |
          file ./entrypoint-linux-amd64
          file ./entrypoint-linux-amd64 | grep -q 'static-pie linked'
      - name: Upload numaflow binary
        uses: actions/upload-artifact@v4
        with:
          name: numaflow-rs-linux-amd64
          path: numaflow-rs-linux-amd64
          if-no-files-found: error
      - name: Upload entrypoint binary
        uses: actions/upload-artifact@v4
        with:
          name: entrypoint-linux-amd64
          path: entrypoint-linux-amd64
          if-no-files-found: error

  e2e-tests:
    name: E2E Tests
    runs-on: ubuntu-24.04
    needs: [build-rust-amd64]
    timeout-minutes: 20
    strategy:
      fail-fast: false
      max-parallel: 13
      matrix:
        driver: [jetstream]
        case:
          [
            e2e,
            diamond-e2e,
            transformer-e2e,
            kafka-e2e,
            map-e2e,
            reduce-one-e2e,
            reduce-two-e2e,
            udsource-e2e,
            api-e2e,
            sideinputs-e2e,
            idle-source-e2e,
            monovertex-e2e,
            builtin-source-e2e,
            serving-e2e,
          ]

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Restore go build cache
        uses: actions/cache@v4
        with:
          path: ~/.cache/go-build
          key: ${{ runner.os }}-go-build-v2-${{ hashFiles('**/go.mod') }}
      - name: Setup Golang
        uses: actions/setup-go@v5
        with:
          go-version: "1.24"
      - name: Add bins to PATH
        run: |
          echo /home/runner/go/bin >> $GITHUB_PATH
          echo /usr/local/bin >> $GITHUB_PATH
      - name: Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: "20"
      - name: Setup Node-Cache
        uses: actions/cache@v4
        with:
          path: ui/node_modules
          key: ${{ runner.os }}-node-dep-v1-${{ hashFiles('**/yarn.lock') }}
      - name: Download Rust amd64 binaries
        uses: actions/download-artifact@v4
        with:
          name: numaflow-rs-linux-amd64
      - name: Download Entrypoint amd64 binaries
        uses: actions/download-artifact@v4
        with:
          name: entrypoint-linux-amd64
      - name: Install k3d
        run: curl -sfL https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash &
      - name: Create a cluster
        run: |
          k3d cluster create e2e
          k3d kubeconfig get e2e > ~/.kube/numaflow-e2e-config
      - name: Install Numaflow
        env:
          GOPATH: /home/runner/go
        run: |
          KUBECONFIG=~/.kube/numaflow-e2e-config VERSION=${{ github.sha }} make start
      - name: Run tests
        env:
          GOPATH: /home/runner/go
        run: KUBECONFIG=~/.kube/numaflow-e2e-config VERSION=${{ github.sha }} ISBSVC=${{matrix.driver}} SKIP_IMAGE_BUILD=true make test-${{matrix.case}}