ci.yml

name: CI

on: [push, pull_request]

jobs:
  lint-console:
    runs-on: ubuntu-latest
    steps:
      - name: Check out
        uses: actions/checkout@v3
      - name: Setup pnpm
        uses: pnpm/action-setup@v4
        with:
          version: 9.0.6
          run_install: |
            - recursive: true
              args: [--frozen-lockfile, --strict-peer-dependencies]
      - name: Run prettier
        run: |
          cd webui
          pnpm check

  build-rust:
    runs-on: ubuntu-22.04-32
    env:
      DATABASE_URL: "postgres://arroyo:arroyo@localhost:5432/arroyo"
    steps:
      - name: Check out
        uses: actions/checkout@v3
      - uses: actions/setup-java@v3
        with:
          distribution: "temurin"
          java-version: "17"
      - name: Install Rust
        uses: dtolnay/rust-toolchain@master
        with:
          components: clippy, rustfmt
          toolchain: 1.95
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
      - name: Check Formatting
        run: cargo fmt -- --check
      - uses: actions/setup-python@v5
        name: Setup Python
        with:
          python-version: "3.12"
      - name: Setup pnpm
        uses: pnpm/action-setup@v4
        with:
          version: 9.0.6
          run_install: |
            - recursive: true
              args: [--frozen-lockfile, --strict-peer-dependencies]
      - name: Install OpenAPI Generator
        run: |
          pnpm install @openapitools/openapi-generator-cli -g
          cd crates/arroyo-openapi && openapi-generator-cli version
      - name: Setup Postgres
        run: |
          sudo apt-get update
          sudo apt-get install postgresql
          sudo systemctl start postgresql
          sudo -u postgres psql -c "CREATE USER arroyo WITH PASSWORD 'arroyo' SUPERUSER;"
          sudo -u postgres createdb arroyo
          pushd /tmp
            wget https://github.com/rust-db/refinery/releases/download/0.8.7/refinery-0.8.7-x86_64-unknown-linux-musl.tar.gz
            tar xvfz refinery*.tar.gz
            mv /tmp/refinery*-musl/refinery /tmp
          popd
          /tmp/refinery migrate -e DATABASE_URL -p crates/arroyo-api/migrations
      - name: Install dependencies
        run: |
          curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin
          sudo apt-get install -y cmake clang ruby unzip postgresql libsasl2-dev libcurl4-openssl-dev netcat
          wget https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protoc-21.8-linux-x86_64.zip
          unzip protoc*.zip
          sudo mv bin/protoc /usr/local/bin
          curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
      - name: Install Kafka
        run: |
          wget --progress=dot --show-progress -O kafka.tgz 'https://www.apache.org/dyn/closer.lua/kafka/4.3.1/kafka_2.13-4.3.1.tgz?action=download'
          tar xvfz kafka.tgz
          mkdir /tmp/kraft-combined-logs
          kafka_*/bin/kafka-storage.sh format --standalone -t 9v5PspiySuWU2l5NjTgRuA -c kafka_*/config/server.properties
          kafka_*/bin/kafka-server-start.sh -daemon kafka_*/config/server.properties
      - name: Install mosquitto
        run: |
          sudo apt-get install -y mosquitto
          sudo service mosquitto start
      - name: Check Formatting
        run: cargo fmt -- --check
      - name: Build console
        run: |
          cd webui
          pnpm build
      - name: Build
        run: cargo build --all-features
      - name: Run Clippy
        run: cargo clippy --no-deps --all-features --all-targets --workspace -- -D warnings
      - name: Test
        run: cargo nextest run -E 'kind(lib)' --all-features
      - name: Integ
        run: |
          set -euo pipefail

          cluster_pid=""

          cleanup_cluster() {
            if [ -n "${cluster_pid}" ]; then
              kill "${cluster_pid}" 2>/dev/null || true
              wait "${cluster_pid}" 2>/dev/null || true
              cluster_pid=""
            fi
            killall arroyo 2>/dev/null || true
          }

          trap cleanup_cluster EXIT

          run_integ() {
            name="$1"
            shift

            echo "Starting integration scenario: ${name}"
            cleanup_cluster
            rm -rf "/tmp/arroyo-integ-${name}"
            mkdir -p "/tmp/arroyo-integ-${name}" /tmp/artifacts

            env \
              ARROYO__DISABLE_TELEMETRY=true \
              ARROYO__CHECKPOINT_URL="file:///tmp/arroyo-integ-${name}" \
              ARROYO__COMPILER__ARTIFACT_URL=file:///tmp/artifacts \
              "$@" \
              target/debug/arroyo cluster &
            cluster_pid=$!

            timeout=20
            while ! nc -z localhost 5115 && [ "${timeout}" -gt 0 ]; do
              sleep 1
              timeout=$((timeout - 1))
            done

            if [ "${timeout}" -le 0 ]; then
              echo "Timed out waiting for integration scenario ${name} to start"
              cleanup_cluster
              return 1
            fi

            status=0
            cargo nextest run --package integ -E 'kind(test)' || status=$?
            cleanup_cluster
            return "${status}"
          }

          run_integ controller-postgres ARROYO__DATABASE__TYPE=postgres
          run_integ controller-sqlite ARROYO__DATABASE__TYPE=sqlite
          run_integ leader-postgres ARROYO__DATABASE__TYPE=postgres ARROYO__JOB_CONTROLLER=worker
          run_integ leader-sqlite ARROYO__DATABASE__TYPE=sqlite ARROYO__JOB_CONTROLLER=worker

  build-console:
    runs-on: ubuntu-latest
    steps:
      - name: Check out
        uses: actions/checkout@v3
      - name: Setup pnpm
        uses: pnpm/action-setup@v4
        with:
          version: 9.0.6
          run_install: |
            - recursive: true
              args: [--frozen-lockfile, --strict-peer-dependencies]
      - name: Build console
        run: |
          cd webui
          pnpm build