rbln_optimum_ci.yaml

name: CI / PR / vllm-rbln with optimum

on:
  workflow_dispatch:
    inputs:
      ref:
        description: "ref to checkout"
        required: false
        type: string
      pr_number:
        description: "PR number to checkout (optional)"
        required: false
        type: string
      rebel_compiler_version:
        description: "Version of rebel-compiler to use (optional, will fetch latest if not provided)"
        required: false
        type: string
      optimum_rbln_version:
        description: "Version of optimum-rbln to use (optional)"
        required: false
        type: string
  workflow_call:
    inputs:
      ref:
        description: "ref to checkout"
        required: false
        type: string
      pr_number:
        description: "PR number to checkout (optional)"
        required: false
        type: string
      rebel_compiler_version:
        description: "Version of rebel-compiler to use"
        required: false
        type: string
      optimum_rbln_version:
        description: "Version of optimum-rbln to use (optional)"
        required: false
        type: string

env:
  REBEL_VLLM_PRE_COMPILED_DIR: ${{ secrets.VLLM_PRE_COMPILED_DIR }}

jobs:
  get-compiler-version:
    if: ${{ !inputs.rebel_compiler_version }}
    uses: ./.github/workflows/rbln_utils_get_compiler_version_ci.yaml
    secrets: inherit

  test_vllm_rbln_rsd1:
    needs: [get-compiler-version]
    if: ${{ always() && !cancelled() }}
    name: Optimum Test CI RSD 1
    runs-on: rbln-sw-k8s-ca22-1
    container:
      image: mcr.microsoft.com/devcontainers/python:3.12
    defaults:
      run:
        shell: bash
    env:
      NUM_INPUT_PROMPT: 1
      RBLN_VERBOSE: 6
      VLLM_LOGGING_LEVEL: DEBUG
    steps:
      - name: Install uv CLI
        run: |
          pip install uv

      - name: Install rebel-compiler
        run: |
          for i in {1..3}; do
            apt-get update && break || sleep 10
          done
          apt-get install -y build-essential
          export CXX=$(which g++)
          uv pip uninstall --system rebel-compiler
          VERSION=${{ inputs.rebel_compiler_version || needs.get-compiler-version.outputs.compiler_version }}
          uv pip install --system --extra-index-url ${{ secrets.DEV_PKG_URL }} rebel-compiler==${VERSION}

      - name: Checkout current repository
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}
          submodules: recursive
          fetch-depth: 0

      - name: Fix git ownership
        run: |
          git config --global --add safe.directory "$GITHUB_WORKSPACE"

      - name: Uninstall existing vllm-rbln
        run: |
          uv pip uninstall --system vllm-rbln

      - name: Build vllm-rbln wheel
        run: |
          uv pip install --system build
          python -m build --wheel

      - name: Install local vllm-rbln package and dependencies
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 15
          max_attempts: 3
          retry_wait_seconds: 30
          retry_on: any
          shell: bash
          command: |
            uv pip install --system packaging setuptools wheel simphile pynvml huggingface_hub setuptools_scm fire pytest pytest-asyncio
            VERSION=${{ inputs.optimum_rbln_version }}
            if [ -n "${{ inputs.optimum_rbln_version }}" ]; then
              uv pip install --system --force-reinstall --no-cache-dir dist/vllm_rbln*.whl --constraint <(echo "optimum-rbln==${{ inputs.optimum_rbln_version }}") --index-url http://pypi-cache.devpi.svc.cluster.local/root/pypi/+simple/ --extra-index-url https://pypi.org/simple
              echo "optimum-rbln ${{ inputs.optimum_rbln_version }}"
            else
              uv pip install --system --force-reinstall --no-cache-dir dist/vllm_rbln*.whl --index-url http://pypi-cache.devpi.svc.cluster.local/root/pypi/+simple/ --extra-index-url https://pypi.org/simple
            fi

      - name: Run gemma2 model (hybrid attention) (V1)
        run: >
          python3 examples/optimum/run_basic.py
          --model_id ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/gemma-2-2b-8k_batch4

      - name : Run encoder-decoder (V1)
        run: >
          python3 examples/optimum/run_encoder_decoder.py
          --num_input_prompt ${{ env.NUM_INPUT_PROMPT }}
          --model_id ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/rbln_bart-small_batch2

      - name : Run text embedding model (V1)
        run: >
          python3 examples/optimum/run_encoder_only.py
          --num_input_prompt ${{ env.NUM_INPUT_PROMPT }}
          --model_id ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/bge-m3-1k-batch4
          --q_prompt_txt ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/prompts/q_prompts.txt
          --p_prompt_txt ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/prompts/p_prompts.txt
          --golden_json ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/golden/golden_bge_m3_result_qp_prompts.json

  test_vllm_rbln_rsd4:
    needs: [get-compiler-version]
    if: ${{ always() && !cancelled() }}
    name: Optimum Test CI RSD 4
    runs-on: rbln-sw-k8s-ca22-4
    container:
      image: mcr.microsoft.com/devcontainers/python:3.12
    defaults:
      run:
        shell: bash
    env:
      NUM_INPUT_PROMPT: 1
      RBLN_VERBOSE: 6
      VLLM_LOGGING_LEVEL: DEBUG
    steps:
      - name: Install uv CLI
        run: |
          pip install uv

      - name: Install rebel-compiler
        run: |
          for i in {1..3}; do
            apt-get update && break || sleep 10
          done
          apt-get install -y build-essential
          export CXX=$(which g++)
          uv pip uninstall --system rebel-compiler
          VERSION=${{ inputs.rebel_compiler_version || needs.get-compiler-version.outputs.compiler_version }}
          uv pip install --system --extra-index-url ${{ secrets.DEV_PKG_URL }} rebel-compiler==${VERSION}

      - name: Checkout current repository
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}
          submodules: recursive
          fetch-depth: 0

      - name: Fix git ownership
        run: |
          git config --global --add safe.directory "$GITHUB_WORKSPACE"

      - name: Uninstall existing vllm-rbln
        run: |
          uv pip uninstall --system vllm-rbln

      - name: Build vllm-rbln wheel
        run: |
          uv pip install --system build
          python -m build --wheel

      - name: Install local vllm-rbln package and dependencies
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 15
          max_attempts: 3
          retry_wait_seconds: 30
          retry_on: any
          shell: bash
          command: |
            uv pip install --system packaging setuptools wheel simphile pynvml huggingface_hub setuptools_scm fire pytest pytest-asyncio
            VERSION=${{ inputs.optimum_rbln_version }}
            if [ -n "${{ inputs.optimum_rbln_version }}" ]; then
              uv pip install --system --force-reinstall --no-cache-dir dist/vllm_rbln*.whl --constraint <(echo "optimum-rbln==${{ inputs.optimum_rbln_version }}") --index-url http://pypi-cache.devpi.svc.cluster.local/root/pypi/+simple/ --extra-index-url https://pypi.org/simple
              echo "optimum-rbln ${{ inputs.optimum_rbln_version }}"
            else
              uv pip install --system --force-reinstall --no-cache-dir dist/vllm_rbln*.whl --index-url http://pypi-cache.devpi.svc.cluster.local/root/pypi/+simple/ --extra-index-url https://pypi.org/simple
            fi

      - name: Run decoder-only test (eager attn) (V1)
        run: >
          python3 examples/optimum/run_decoder_only.py
          --num_input_prompt ${{ env.NUM_INPUT_PROMPT }}
          --model_id ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/llama2-7b_batch2
          --prompt_txt ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/prompts/copy_prompts.txt
          --golden_json ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/golden/golden_llama7b_result_copy_prompts.json

      - name: Run decoder-only test (Flash-attention mode) (V1)
        run: >
          python3 examples/optimum/run_decoder_only.py
          --num_input_prompt ${{ env.NUM_INPUT_PROMPT }}
          --model_id ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/llama3_2-3b-128k_kv16k_batch4
          --prompt_txt ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/prompts/copy_prompts.txt
          --golden_json ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/golden/golden_llama3_2_3b_instruct_128k_copy_prompts.json

      - name : Run Llava-next (Eager mode) (V1)
        run: >
           python3 examples/optimum/run_llava.py
           --num_input_prompt ${{ env.NUM_INPUT_PROMPT }}
           --model_id ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/llava-v1.6-mistral-7b-hf-32k-b4/
      
      - name : Run Llava-next (Eager mode) (V0)
        run: >
           VLLM_USE_V1=0 python3 examples/optimum/run_llava.py
           --num_input_prompt ${{ env.NUM_INPUT_PROMPT }}
           --model_id ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/llava-v1.6-mistral-7b-hf-32k-b4/

      - name : Run Llava-next (Flash-attention mode) (V1)
        run: >
          python3 examples/optimum/run_llava.py
          --num_input_prompt ${{ env.NUM_INPUT_PROMPT }}
          --model_id ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/llava-v1.6-mistral-7b-hf-32k-b4-kv16k
      
      - name : Run Llava-next (Flash-attention mode) (V0)
        run: >
          VLLM_USE_V1=0 python3 examples/optimum/run_llava.py
          --num_input_prompt ${{ env.NUM_INPUT_PROMPT }}
          --model_id ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/llava-v1.6-mistral-7b-hf-32k-b4-kv16k

      - name : Run Idefics3 (Eager mode) (V1)
        run: >
          python3 examples/optimum/run_idefics3.py
          --num_input_prompt ${{ env.NUM_INPUT_PROMPT }}
          --model_id ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/idefics3-8b-llama3-32k-b4

      - name : Run Idefics3 (Flash-attention mode) (V1)
        run: >
          python3 examples/optimum/run_idefics3.py
          --num_input_prompt ${{ env.NUM_INPUT_PROMPT }}
          --model_id ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/idefics3-8b-llama3-32k-b4-kv16k

      - name : Run Blip2 (V1)
        run: >
          python3 examples/optimum/run_blip2.py
          --num_input_prompt ${{ env.NUM_INPUT_PROMPT }}
          --model_id ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/blip2-opt-2.7b-2k-b4

      - name : Run PaliGemma (Eager mode) (V1)
        run: >
          python3 examples/optimum/run_paligemma.py
          --num_input_prompt ${{ env.NUM_INPUT_PROMPT }}
          --model_id ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/paligemma-3b-8k-b4

  test_vllm_rbln_rsd8:
    needs: [get-compiler-version]
    if: ${{ always() && !cancelled() }}
    name: Optimum Test CI RSD 8
    runs-on: rbln-sw-k8s-ca22-8
    container:
      image: mcr.microsoft.com/devcontainers/python:3.12
    defaults:
      run:
        shell: bash
    env:
      NUM_INPUT_PROMPT: 1
      RBLN_VERBOSE: 6
      VLLM_LOGGING_LEVEL: DEBUG
    steps:
      - name: Install uv CLI
        run: |
          pip install uv

      - name: Install rebel-compiler
        run: |
          for i in {1..3}; do
            apt-get update && break || sleep 10
          done
          apt-get install -y build-essential
          export CXX=$(which g++)
          uv pip uninstall --system rebel-compiler
          VERSION=${{ inputs.rebel_compiler_version || needs.get-compiler-version.outputs.compiler_version }}
          uv pip install --system --extra-index-url ${{ secrets.DEV_PKG_URL }} rebel-compiler==${VERSION}

      - name: Checkout current repository
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.sha }}
          submodules: recursive
          fetch-depth: 0

      - name: Fix git ownership
        run: |
          git config --global --add safe.directory "$GITHUB_WORKSPACE"

      - name: Uninstall existing vllm-rbln
        run: |
          uv pip uninstall --system vllm-rbln

      - name: Build vllm-rbln wheel
        run: |
          uv pip install --system build
          python -m build --wheel

      - name: Install local vllm-rbln package and dependencies
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 15
          max_attempts: 3
          retry_wait_seconds: 30
          retry_on: any
          shell: bash
          command: |
            uv pip install --system packaging setuptools wheel simphile pynvml huggingface_hub setuptools_scm fire pytest pytest-asyncio
            VERSION=${{ inputs.optimum_rbln_version }}
            if [ -n "${{ inputs.optimum_rbln_version }}" ]; then
              uv pip install --system --force-reinstall --no-cache-dir dist/vllm_rbln*.whl --constraint <(echo "optimum-rbln==${{ inputs.optimum_rbln_version }}") --index-url http://pypi-cache.devpi.svc.cluster.local/root/pypi/+simple/ --extra-index-url https://pypi.org/simple
              echo "optimum-rbln ${{ inputs.optimum_rbln_version }}"
            else
              uv pip install --system --force-reinstall --no-cache-dir dist/vllm_rbln*.whl --index-url http://pypi-cache.devpi.svc.cluster.local/root/pypi/+simple/ --extra-index-url https://pypi.org/simple
            fi

      - name : Run Qwen2.5_VL (V1)
        run: >
          python3 examples/optimum/run_qwen_vl.py
          --num_input_prompt ${{ env.NUM_INPUT_PROMPT }}
          --model_id ${{ env.REBEL_VLLM_PRE_COMPILED_DIR }}/qwen2_5-vl-7b-32k-b4-kv16k