pull-request.yml

auto-retry: &auto-retry
  automatic:
    # Agent terminated because the AWS EC2 spot instance killed by AWS.
    - signal_reason: agent_stop
      limit: 3
  manual:
    permit_on_passed: true

cargo-cache: &cargo-cache
  id: cargo
  key: "v1-cache-{{ id }}-{{ runner.os }}-{{ checksum 'Cargo.lock' }}"
  restore-keys:
    - "v1-cache-{{ id }}-{{ runner.os }}-"
    - "v1-cache-{{ id }}-"
  backend: s3
  s3:
    bucket: rw-ci-cache-bucket
    args: "--no-progress"
  paths:
    - ".cargo/registry/index"
    - ".cargo/registry/cache"
    - ".cargo/git"

other-sql-backend: &other-sql-backend
  matrix:
    setup:
      label: [""]
      endpoint: [""]
    adjustments:
      - with:
          label: ""
          endpoint: ""
        skip: true # hack
      - with:
          label: "postgres"
          # PGPASSWORD=postgres psql -h db -p 5432 -U postgres -d rwmeta
          endpoint: "postgres://postgres:postgres@db:5432/rwmeta"
      # Temporarily disable tests for mysql backend as there are unresolved issues.
      # - with:
      #     label: "mysql"
      #     # mysql -h mysql-meta -P 3306 -u root -p123456 -D rwmeta
      #     endpoint: "mysql://root:123456@mysql-meta:3306/rwmeta"
  env:
    RISEDEV_SQL_ENDPOINT: "{{matrix.endpoint}}"

docker-compose-common: &docker-compose-common
    config: ci/docker-compose.yml
    mount-buildkite-agent: true
    propagate-environment: true
    run: ci-standard-env

steps:
  - label: "check ci image rebuild"
    plugins:
      - monorepo-diff#v1.2.0:
          diff: "git diff --name-only origin/main"
          watch:
            - path: "ci/build-ci-image.sh"
              config:
                command: "ci/build-ci-image.sh"
                label: "ci build images"
  - wait

  - label: "build"
    command: "ci/scripts/build.sh -p ci-dev"
    key: "build"
    plugins:
      - nienbo/cache#v2.4.20: *cargo-cache
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
    timeout_in_minutes: 15
    retry: *auto-retry

  - label: "build other components"
    command: "ci/scripts/build-other.sh"
    key: "build-other"
    plugins:
      - nienbo/cache#v2.4.20: *cargo-cache
      - seek-oss/aws-sm#v2.3.2:
          env:
            GITHUB_TOKEN: github-token
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
          environment:
            - GITHUB_TOKEN
    timeout_in_minutes: 14
    retry: *auto-retry

  - label: "build (deterministic simulation)"
    command: "ci/scripts/build-simulation.sh"
    key: "build-simulation"
    plugins:
      - nienbo/cache#v2.4.20: *cargo-cache
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
    retry: *auto-retry

  - label: "docslt"
    command: "ci/scripts/docslt.sh"
    key: "docslt"
    plugins:
      - nienbo/cache#v2.4.20: *cargo-cache
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "end-to-end test"
    command: "ci/scripts/e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe"
    if: |
      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
      || build.pull_request.labels includes "ci/run-e2e-test"
      || build.env("CI_STEPS") =~ /(^|,)e2e-tests?(,|$$)/
    depends_on:
      - "build"
      - "build-other"
      - "docslt"
    plugins:
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 30
    retry: *auto-retry

  - label: "slow end-to-end test"
    key: "slow-e2e-test"
    command: "ci/scripts/slow-e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe"
    if: |
      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
      || build.pull_request.labels includes "ci/run-slow-e2e-tests"
      || build.env("CI_STEPS") =~ /(^|,)slow-e2e-tests?(,|$$)/
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 8
    retry: *auto-retry

  - label: "meta backup test"
    key: "e2e-meta-backup-test"
    command: "ci/scripts/run-meta-backup-test.sh -p ci-dev -m ci-3streaming-2serving-3fe"
    if: |
      build.pull_request.labels includes "ci/run-e2e-meta-backup-test"
    depends_on:
      - "build"
      - "build-other"
      - "docslt"
    plugins:
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 45
    retry: *auto-retry

  - label: "end-to-end test (parallel)"
    command: "ci/scripts/e2e-test-parallel.sh -p ci-dev"
    if: |
      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
      || build.pull_request.labels includes "ci/run-e2e-parallel-tests"
      || build.env("CI_STEPS") =~ /(^|,)e2e-parallel-tests?(,|$$)/
    depends_on:
      - "build"
      - "docslt"
    plugins:
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 25
    retry: *auto-retry

  - label: "end-to-end test for opendal (parallel)"
    if: build.pull_request.labels includes "ci/run-opendal-tests" || build.env("CI_STEPS") =~ /(^|,)opendal-tests?(,|$$)/
    command: "ci/scripts/e2e-test-parallel-for-opendal.sh -p ci-dev"
    depends_on:
      - "build"
      - "docslt"
    plugins:
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 14
    retry: *auto-retry

  - label: "end-to-end test (parallel, in-memory)"
    if: build.pull_request.labels includes "ci/run-e2e-parallel-in-memory-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-parallel-in-memory-tests?(,|$$)/
    command: "ci/scripts/e2e-test-parallel-in-memory.sh -p ci-dev"
    depends_on: "build"
    plugins:
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 12
    retry: *auto-retry

  - label: "end-to-end source test"
    command: "ci/scripts/e2e-source-test.sh -p ci-dev"
    if: |
      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
      || build.pull_request.labels includes "ci/run-e2e-source-tests"
      || build.env("CI_STEPS") =~ /(^|,)e2e-source-tests?(,|$$)/
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: source-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
          upload-container-logs: always
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 20
    retry: *auto-retry

  - label: "end-to-end sink test"
    command: "ci/scripts/e2e-sink-test.sh -p ci-dev"
    if: |
      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
      || build.pull_request.labels includes "ci/run-e2e-sink-tests"
      || build.env("CI_STEPS") =~ /(^|,)e2e-sink-tests?(,|$$)/
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: sink-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 20
    cancel_on_build_failing: true
    retry: *auto-retry

  - label: "connector node integration test Java {{matrix.java_version}}"
    if: build.pull_request.labels includes "ci/run-connector-node-integration-tests" || build.env("CI_STEPS") =~ /(^|,)java-connector-node-integration-tests?(,|$$)/
    command: "ci/scripts/connector-node-integration-test.sh -p ci-dev -v {{matrix.java_version}}"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    matrix:
      setup:
        java_version:
          - "11"
          - "17"
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "end-to-end iceberg sink test"
    if: build.pull_request.labels includes "ci/run-e2e-iceberg-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-iceberg-sink-tests?(,|$$)/
    command: "ci/scripts/e2e-iceberg-sink-test.sh -p ci-dev"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 15
    retry: *auto-retry

  - label: "end-to-end iceberg sink v2 test"
    if: build.pull_request.labels includes "ci/run-e2e-iceberg-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-iceberg-sink-tests?(,|$$)/
    command: "ci/scripts/e2e-iceberg-sink-v2-test.sh -p ci-dev"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: sink-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 17
    retry: *auto-retry

  - label: "end-to-end iceberg cdc test"
    if: build.pull_request.labels includes "ci/run-e2e-iceberg-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-iceberg-sink-tests?(,|$$)/
    command: "ci/scripts/e2e-iceberg-cdc.sh -p ci-dev"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: sink-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 15
    retry: *auto-retry

  - label: "end-to-end iceberg engine test"
    if: build.pull_request.labels includes "ci/run-e2e-iceberg-engine-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-iceberg-engine-tests?(,|$$)/
    command: "ci/scripts/e2e-iceberg-engine-test.sh -p ci-dev"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: iceberg-engine-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 15
    retry: *auto-retry

  - label: "end-to-end pulsar sink test"
    if: build.pull_request.labels includes "ci/run-e2e-pulsar-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-pulsar-sink-tests?(,|$$)/
    command: "ci/scripts/e2e-pulsar-sink-test.sh -p ci-dev"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: sink-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "end-to-end mqtt sink test"
    if: build.pull_request.labels includes "ci/run-e2e-mqtt-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-mqtt-sink-tests?(,|$$)/
    command: "ci/scripts/e2e-mqtt-sink-test.sh -p ci-dev"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: sink-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "end-to-end clickhouse sink test"
    if: build.pull_request.labels includes "ci/run-e2e-clickhouse-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-clickhouse-sink-tests?(,|$$)/
    command: "ci/scripts/e2e-clickhouse-sink-test.sh -p ci-dev"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: sink-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "end-to-end time travel test"
    key: "e2e-time-travel-tests"
    command: "ci/scripts/e2e-time-travel-test.sh -p ci-dev"
    if: build.pull_request.labels includes "ci/run-e2e-time-travel-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-time-travel-tests?(,|$$)/
    depends_on:
      - "build"
      - "build-other"
      - "docslt"
    plugins:
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 15
    retry: *auto-retry

  - label: "end-to-end sqlserver sink test"
    if: build.pull_request.labels includes "ci/run-e2e-sqlserver-sink-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-sqlserver-sink-tests?(,|$$)/
    command: "ci/scripts/e2e-sqlserver-sink-test.sh -p ci-dev"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: sink-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "end-to-end deltalake sink test"
    if: build.pull_request.labels includes "ci/run- e2e-deltalake-sink-rust-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-deltalake-sink-rust-tests?(,|$$)/
    command: "ci/scripts/e2e-deltalake-sink-rust-test.sh -p ci-dev"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: sink-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "end-to-end redis sink test"
    if: build.pull_request.labels includes "ci/run-e2e-redis-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-redis-sink-tests?(,|$$)/
    command: "ci/scripts/e2e-redis-sink-test.sh -p ci-dev"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: sink-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "end-to-end doris sink test"
    if: build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-doris-sink-tests?(,|$$)/
    command: "ci/scripts/e2e-doris-sink-test.sh -p ci-dev"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: sink-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "end-to-end starrocks sink test"
    if: build.pull_request.labels includes "ci/run-e2e-starrocks-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-starrocks-sink-tests?(,|$$)/
    command: "ci/scripts/e2e-starrocks-sink-test.sh -p ci-dev"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: sink-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "end-to-end cassandra sink test"
    if: build.pull_request.labels includes "ci/run-e2e-cassandra-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-cassandra-sink-tests?(,|$$)/
    command: "ci/scripts/e2e-cassandra-sink-test.sh -p ci-dev"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: sink-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "end-to-end mongodb sink test"
    if: build.pull_request.labels includes "ci/run-e2e-mongodb-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-mongodb-sink-tests?(,|$$)/
    command: "ci/scripts/e2e-mongodb-sink-test.sh -p ci-dev"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: sink-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "e2e java-binding test"
    if: build.pull_request.labels includes "ci/run-java-binding-tests" || build.env("CI_STEPS") =~ /(^|,)java-binding-tests?(,|$$)/
    command: "ci/scripts/java-binding-test.sh -p ci-dev"
    depends_on:
      - "build"
      - "build-other"
    plugins:
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 10
    retry: *auto-retry

  - label: "regress test"
    command: "ci/scripts/regress-test.sh -p ci-dev"
    if: |
      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
      || build.pull_request.labels includes "ci/run-regress-test"
      || build.env("CI_STEPS") =~ /(^|,)regress-tests?(,|$$)/
    depends_on: "build"
    plugins:
      - docker-compose#v5.5.0:
          run: regress-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 5
    retry: *auto-retry

  # The timeout should be strictly less than timeout in `main-cron.yml`.
  # It should be as conservative as possible.
  # This ensures our `main-cron` workflow will be stable.
  - label: "unit test"
    command: "ci/scripts/pr-unit-test.sh"
    if: |
      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
      || build.pull_request.labels includes "ci/run-unit-test"
      || build.env("CI_STEPS") =~ /(^|,)unit-tests?(,|$$)/
    plugins:
      - nienbo/cache#v2.4.20: *cargo-cache
      - ./ci/plugins/swapfile
      - seek-oss/aws-sm#v2.3.2:
          env:
            CODECOV_TOKEN: my-codecov-token
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          environment:
            - CODECOV_TOKEN
    timeout_in_minutes: 22
    retry: *auto-retry

  - label: "check"
    command: "ci/scripts/check.sh"
    if: |
      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
      || build.pull_request.labels includes "ci/run-check"
      || build.env("CI_STEPS") =~ /(^|,)check(,|$$)/
    plugins:
      - nienbo/cache#v2.4.20: *cargo-cache
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
    timeout_in_minutes: 25
    retry: *auto-retry

  - label: "check dylint"
    command: "ci/scripts/check-dylint.sh"
    if: |
      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
      || build.pull_request.labels includes "ci/run-check"
      || build.env("CI_STEPS") =~ /(^|,)check(,|$$)/
    plugins:
      - nienbo/cache#v2.4.20: *cargo-cache
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
    timeout_in_minutes: 25
    retry: *auto-retry

  - label: "unit test (deterministic simulation)"
    command: "ci/scripts/deterministic-unit-test.sh"
    if: |
      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
      || build.pull_request.labels includes "ci/run-unit-test-deterministic-simulation"
      || build.env("CI_STEPS") =~ /(^|,)unit-tests?-deterministic-simulation(,|$$)/
    plugins:
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
    timeout_in_minutes: 12
    cancel_on_build_failing: true
    retry: *auto-retry

  - label: "integration test (deterministic simulation)"
    command: "TEST_NUM=5 ci/scripts/deterministic-it-test.sh"
    if: |
      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
      || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation"
      || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/
    depends_on: "build-simulation"
    plugins:
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 20
    retry: *auto-retry

  - label: "end-to-end test (deterministic simulation)"
    command: "TEST_NUM=4 ci/scripts/deterministic-e2e-test.sh"
    if: |
      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
      || build.pull_request.labels includes "ci/run-e2e-test-deterministic-simulation"
      || build.env("CI_STEPS") =~ /(^|,)e2e-tests?-deterministic-simulation(,|$$)/
    depends_on: "build-simulation"
    plugins:
      - seek-oss/aws-sm#v2.3.2:
          env:
            GITHUB_TOKEN: github-token
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
          environment:
            - GITHUB_TOKEN
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 30
    cancel_on_build_failing: true
    retry: *auto-retry

  - label: "recovery test (deterministic simulation)"
    command: "TEST_NUM=4 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.0 ci/scripts/deterministic-recovery-test.sh"
    if: |
      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
      || build.pull_request.labels includes "ci/run-recovery-test-deterministic-simulation"
      || build.env("CI_STEPS") =~ /(^|,)recovery-tests?-deterministic-simulation(,|$$)/
    depends_on: "build-simulation"
    plugins:
      # - seek-oss/aws-sm#v2.3.2:
      #     env:
      #       BUILDKITE_ANALYTICS_TOKEN: buildkite-build-analytics-deterministic-token
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      # Only upload zipped files, otherwise the logs is too much.
      - ./ci/plugins/upload-failure-logs-zipped
      # - test-collector#v1.0.0:
      #     files: "*-junit.xml"
      #     format: "junit"
    timeout_in_minutes: 35
    cancel_on_build_failing: true
    retry: *auto-retry

  - label: "misc check"
    command: "ci/scripts/misc-check.sh"
    if: |
      !(build.pull_request.labels includes "ci/pr/run-selected") && build.env("CI_STEPS") == null
      || build.pull_request.labels includes "ci/run-misc-check"
      || build.env("CI_STEPS") =~ /(^|,)misc-check(,|$$)/
    plugins:
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
      - shellcheck#v1.2.0:
          files: ./**/*.sh
    timeout_in_minutes: 5
    retry: *auto-retry

  # The following jobs are triggered only when PR has corresponding labels.

  # Generates cpu flamegraph env
  - label: "flamegraph-env-build"
    key: "flamegraph-env-build"
    command: "ci/scripts/flamegraph-env-build.sh"
    if: |
      build.pull_request.labels includes "cpu_flamegraph"
      || build.pull_request.labels includes "ci/run-cpu-flamegraph"
      || build.pull_request.labels includes "heap_flamegraph"
      || build.pull_request.labels includes "ci/run-heap-flamegraph"
      || build.env("CI_STEPS") =~ /(^|,)(cpu-flamegraph|heap-flamegraph)(,|$$)/
    plugins:
      - seek-oss/aws-sm#v2.3.2:
          env:
            GITHUB_TOKEN: github-token
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
          environment:
            - GITHUB_TOKEN
    timeout_in_minutes: 20

  # Generates cpu flamegraph if label `cpu_flamegraph` is added to PR.
  - label: "Generate CPU flamegraph"
    command: "PULL_REQUEST=$BUILDKITE_PULL_REQUEST ci/scripts/gen-flamegraph.sh cpu"
    depends_on: "flamegraph-env-build"
    if: build.pull_request.labels includes "cpu_flamegraph" || build.pull_request.labels includes "ci/run-cpu-flamegraph" || build.env("CI_STEPS") =~ /(^|,)cpu-flamegraph(,|$$)/
    plugins:
      - seek-oss/aws-sm#v2.3.2:
          env:
            GITHUB_TOKEN: github-token
      - docker-compose#v5.5.0:
          run: ci-flamegraph-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
          environment:
            - GITHUB_TOKEN
    # TODO(kwannoel): Here are the areas that can be further optimized:
    # - Nexmark event generation: ~3min for 100mil records.
    # - Generate Flamegraph: ~15min (see https://github.com/koute/not-perf/issues/30 on optimizing)
    # - Building RW artifacts: ~8min
    timeout_in_minutes: 540

  # Generates heap flamegraph if label `heap_flamegraph` is added to PR.
  - label: "Generate Heap flamegraph"
    command: "PULL_REQUEST=$BUILDKITE_PULL_REQUEST ci/scripts/gen-flamegraph.sh heap"
    depends_on: "flamegraph-env-build"

    if: build.pull_request.labels includes "heap_flamegraph" || build.pull_request.labels includes "ci/run-heap-flamegraph" || build.env("CI_STEPS") =~ /(^|,)heap-flamegraph(,|$$)/

    plugins:
      - seek-oss/aws-sm#v2.3.2:
          env:
            GITHUB_TOKEN: github-token
      - docker-compose#v5.5.0:
          run: ci-flamegraph-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
          environment:
            - GITHUB_TOKEN
    # TODO(kwannoel): Here are the areas that can be further optimized:
    # - Nexmark event generation: ~3min for 100mil records.
    # - Generate Flamegraph: ~15min (see https://github.com/koute/not-perf/issues/30 on optimizing)
    # - Building RW artifacts: ~8min
    timeout_in_minutes: 60 # ~3-4 queries can run

  # Backwards compatibility tests
  - label: "Backwards compatibility tests"
    command: "VERSION_OFFSET={{matrix.version_offset}} RW_COMMIT=$BUILDKITE_COMMIT ci/scripts/backwards-compat-test.sh -p ci-dev"
    if: |
      build.pull_request.labels includes "breaking-change" ||
        build.pull_request.labels includes "ci/run-backwards-compat-tests" ||
        build.env("CI_STEPS") =~ /(^|,)backwards?-compat-tests?(,|$$)/
    depends_on:
      - "build"
    plugins:
      - docker-compose#v5.5.0:
          run: source-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
          environment:
            - BUILDKITE_BRANCH
      - ./ci/plugins/upload-failure-logs
    matrix:
      setup:
        # Test the 4 latest versions against the latest main.
        # e.g.
        # 1: 2.0.0
        # 2: 1.1.1
        # 3: 1.0.1
        # 4: 1.0.0
        # It is ordered by the full version number, rather than minor / major version.
        # We can change to just be on major version in the future.
        version_offset:
          - "1"
          - "2"
          - "3"
          - "4"
    timeout_in_minutes: 25

  # Sqlsmith differential testing
  - label: "Sqlsmith Differential Testing"
    command: "ci/scripts/sqlsmith-differential-test.sh -p ci-dev"
    if: build.pull_request.labels includes "ci/run-sqlsmith-differential-tests" || build.env("CI_STEPS") =~ /(^|,)sqlsmith-differential-tests?(,|$$)/
    depends_on:
      - "build"
    plugins:
      - docker-compose#v5.5.0:
          run: ci-flamegraph-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
    timeout_in_minutes: 40

  - label: "Backfill tests"
    command: "BUILDKITE=${BUILDKITE:-} ci/scripts/backfill-test.sh -p ci-dev"
    if: build.pull_request.labels includes "ci/run-backfill-tests" || build.env("CI_STEPS") =~ /(^|,)backfill-tests?(,|$$)/
    depends_on:
      - "build"
    plugins:
      - docker-compose#v5.5.0:
          run: source-test-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 30

  - label: "e2e standalone binary test"
    command: "ci/scripts/e2e-test.sh -p ci-dev -m standalone"
    if: build.pull_request.labels includes "ci/run-e2e-standalone-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-standalone-tests?(,|$$)/
    depends_on:
      - "build"
      - "build-other"
      - "docslt"
    plugins:
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 40
    retry: *auto-retry

  - label: "e2e single-node binary test"
    command: "ci/scripts/e2e-test.sh -p ci-dev -m single-node"
    if: build.pull_request.labels includes "ci/run-e2e-single-node-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-single-node-tests?(,|$$)/
    depends_on:
      - "build"
      - "build-other"
      - "docslt"
    plugins:
      - docker-compose#v4.9.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 30
    retry: *auto-retry

  - label: "end-to-end test ({{matrix.label}} backend)"
    <<: *other-sql-backend
    command: "ci/scripts/e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe"
    if: build.pull_request.labels includes "ci/run-e2e-test-other-backends" || build.env("CI_STEPS") =~ /(^|,)e2e-test-other-backends?(,|$$)/
    depends_on:
      - "build"
      - "build-other"
      - "docslt"
    plugins:
      - docker-compose#v5.5.0: *docker-compose-common
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 32
    retry: *auto-retry

  # FIXME(kwannoel): Let the github PR labeller label it, if sqlsmith source files has changes.
  - label: "fuzz test"
    command: "ci/scripts/pr-fuzz-test.sh -p ci-dev"
    if: build.pull_request.labels includes "ci/run-sqlsmith-fuzzing-tests" || build.env("CI_STEPS") =~ /(^|,)sqlsmith-fuzzing-tests?(,|$$)/
    depends_on:
      - "build"
      - "build-simulation"
    plugins:
      - ./ci/plugins/swapfile
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 15
    retry: *auto-retry

  - label: "deterministic fuzz test"
    command: "ci/scripts/run-deterministic-fuzz-test.sh -p ci-dev"
    if: build.pull_request.labels includes "ci/run-deterministic-sqlsmith-fuzzing-tests" || build.env("CI_STEPS") =~ /(^|,)deterministic-sqlsmith-fuzzing-tests?(,|$$)/
    depends_on:
      - "build-simulation"
    plugins:
      - ./ci/plugins/swapfile
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
      - ./ci/plugins/upload-failure-logs
    timeout_in_minutes: 15
    retry: *auto-retry

  - label: "enable ci/pr/run-selected only in draft PRs"
    if: build.pull_request.labels includes "ci/pr/run-selected" && !build.pull_request.draft
    commands:
      - echo "ci/pr/run-selected is only usable for draft Pull Requests"
      - exit 1

  - label: "micro benchmark"
    command: "ci/scripts/run-micro-benchmarks.sh"
    key: "run-micro-benchmarks"
    if: build.pull_request.labels includes "ci/run-micro-benchmarks" || build.env("CI_STEPS") =~ /(^|,)micro-benchmarks?(,|$$)/
    plugins:
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
    timeout_in_minutes: 60
    retry: *auto-retry

  - label: "upload micro-benchmark"
    if: build.pull_request.labels includes "ci/run-upload-micro-benchmark" || build.env("CI_STEPS") =~ /(^|,)upload-micro-benchmarks?(,|$$)/
    command:
      - "BUILDKITE_BUILD_NUMBER=$BUILDKITE_BUILD_NUMBER ci/scripts/upload-micro-bench-results.sh"
    depends_on: "run-micro-benchmarks"
    plugins:
      - seek-oss/aws-sm#v2.3.2:
          env:
            BUILDKITE_TOKEN: buildkite_token
            GITHUB_TOKEN: github-token
      - docker-compose#v5.5.0:
          run: rw-build-env
          config: ci/docker-compose.yml
          mount-buildkite-agent: true
          environment:
            - BUILDKITE_TOKEN
            - GITHUB_TOKEN
    timeout_in_minutes: 5