test_lemonade_eval.yml

# This workflow tests lemonade-eval CLI tools with lemonade-server
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python

name: Test Lemonade Eval

on:
  push:
    branches: ["main"]
    paths:
      - '.github/workflows/test_lemonade_eval.yml'
      - 'setup.py'
      - 'src/lemonade/**'
  pull_request:
    paths:
      - '.github/workflows/test_lemonade_eval.yml'
      - 'setup.py'
      - 'src/lemonade/**'
  merge_group:

permissions:
  contents: read

jobs:
  test-lemonade-eval:
    env:
      LEMONADE_CI_MODE: "True"
      LEMONADE_SERVER_URL: "http://localhost:8000"
    runs-on: windows-latest
    concurrency:
      group: ${{ github.workflow }}-${{ github.ref }}
      cancel-in-progress: true
    steps:
      - uses: actions/checkout@v3

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Create virtual environment and install dependencies
        shell: bash
        run: |
          python -m venv .venv
          venvPython=".venv/Scripts/python"
          venvPip=".venv/Scripts/pip"
          $venvPython -m pip install --upgrade pip
          $venvPip install pylint
          $venvPython -m pip check
          $venvPip install -e .[oga-cpu]

      - name: Lint with Black
        uses: psf/black@stable
        with:
          options: "--check --verbose"
          src: "./src/lemonade"

      - name: Lint with PyLint
        shell: bash
        run: |
          venvPylint=".venv/Scripts/pylint"
          $venvPylint src/lemonade --rcfile .pylintrc --disable E0401

      - name: Download latest Lemonade Server MSI
        shell: PowerShell
        run: |
          Write-Host "Downloading latest lemonade-server-minimal.msi from releases..." -ForegroundColor Cyan
          
          # Use GitHub CLI to download the latest release asset
          gh release download --repo lemonade-sdk/lemonade --pattern "lemonade-server-minimal.msi" --output lemonade-server-minimal.msi
          
          if (-not (Test-Path "lemonade-server-minimal.msi")) {
            Write-Host "ERROR: Failed to download lemonade-server-minimal.msi" -ForegroundColor Red
            exit 1
          }
          
          Write-Host "Downloaded lemonade-server-minimal.msi successfully" -ForegroundColor Green
          Get-ChildItem lemonade-server-minimal.msi
        env:
          GH_TOKEN: ${{ github.token }}

      - name: Install Lemonade Server
        shell: PowerShell
        run: |
          $installPath = "${{ github.workspace }}\lemonade-server"
          $logPath = "${{ github.workspace }}\lemonade-msi-install.log"
          Write-Host "Installing Lemonade Server to: $installPath" -ForegroundColor Cyan
          
          # Run installer silently with verbose logging
          $args = "/i lemonade-server-minimal.msi /qn INSTALLDIR=`"$installPath`" /L*V `"$logPath`""
          $p = Start-Process -FilePath "msiexec.exe" -ArgumentList $args -Wait -PassThru
          
          if ($p.ExitCode -ne 0) {
            Write-Host "ERROR: Installation failed with exit code $($p.ExitCode)" -ForegroundColor Red
            if (Test-Path $logPath) {
              Get-Content $logPath | Select-Object -Last 100
            }
            exit $p.ExitCode
          }
          
          # Add to PATH for subsequent steps
          echo "$installPath\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
          
          Write-Host "Installation complete. Verifying..." -ForegroundColor Green
          & "$installPath\bin\lemonade-server.exe" --version

      - name: Test OGA-Load (CPU ONNX model)
        shell: bash
        run: |
          venvPython=".venv/Scripts/python"
          venvLemonade=".venv/Scripts/lemonade-eval"
          
          echo "Testing OGA-Load with CPU ONNX model..."
          $venvLemonade -i amd/Qwen2.5-0.5B-Instruct-quantized_int4-float16-cpu-onnx oga-load --device cpu --dtype int4 llm-prompt -p "tell me a story" --max-new-tokens 5
          
          echo "Running OGA CPU API tests..."
          $venvPython test/oga_cpu_api.py

      - name: Test Server Integration (GGUF model)
        shell: PowerShell
        run: |
          $installPath = "${{ github.workspace }}\lemonade-server"
          $serverExe = "$installPath\bin\lemonade-server.exe"
          $logFile = "${{ github.workspace }}\lemonade-server.log"
          $venvPython = ".venv\Scripts\python.exe"
          $venvLemonade = ".venv\Scripts\lemonade-eval"
          
          Write-Host "Starting Lemonade Server..." -ForegroundColor Cyan
          
          # Start server in background
          $serverProcess = Start-Process -FilePath $serverExe -ArgumentList "serve" -RedirectStandardOutput $logFile -RedirectStandardError "$logFile.err" -PassThru
          
          # Wait for server to be ready
          $maxAttempts = 30
          $attempt = 0
          $serverReady = $false
          
          while ($attempt -lt $maxAttempts -and -not $serverReady) {
            Start-Sleep -Seconds 2
            $attempt++
            Write-Host "Checking server health (attempt $attempt/$maxAttempts)..."
            
            try {
              $response = Invoke-WebRequest -Uri "http://localhost:8000/api/v1/health" -UseBasicParsing -TimeoutSec 5
              if ($response.StatusCode -eq 200) {
                $serverReady = $true
                Write-Host "Server is ready!" -ForegroundColor Green
              }
            } catch {
              Write-Host "Server not ready yet..."
            }
          }
          
          if (-not $serverReady) {
            Write-Host "ERROR: Server failed to start within timeout" -ForegroundColor Red
            if (Test-Path $logFile) {
              Write-Host "=== Server Log ===" -ForegroundColor Yellow
              Get-Content $logFile
            }
            if (Test-Path "$logFile.err") {
              Write-Host "=== Server Error Log ===" -ForegroundColor Yellow
              Get-Content "$logFile.err"
            }
            exit 1
          }
          
          # Test CLI
          Write-Host "Testing lemonade-eval CLI..."
          & $venvLemonade -m -i Llama-3.2-1B-Instruct-GGUF load bench -w 0 -i 5
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          
          try {
            Write-Host "Running server integration tests with Qwen3-4B-Instruct-2507-GGUF..." -ForegroundColor Cyan
            & $venvPython test/llm_api.py
            if ($LASTEXITCODE -ne 0) {
              throw "Tests failed with exit code $LASTEXITCODE"
            }
          } finally {
            Write-Host "Stopping server..." -ForegroundColor Cyan
            Stop-Process -Id $serverProcess.Id -Force -ErrorAction SilentlyContinue
            
            # Show logs if tests failed
            if ($LASTEXITCODE -ne 0) {
              if (Test-Path $logFile) {
                Write-Host "=== Server Log ===" -ForegroundColor Yellow
                Get-Content $logFile | Select-Object -Last 50
              }
              if (Test-Path "$logFile.err") {
                Write-Host "=== Server Error Log ===" -ForegroundColor Yellow
                Get-Content "$logFile.err" | Select-Object -Last 50
              }
            }
          }

      - name: Stop Lemonade Server
        if: always()
        shell: PowerShell
        run: |
          Write-Host "Stopping Lemonade Server..." -ForegroundColor Cyan
          Get-Process -Name "lemonade-server", "lemonade-router", "llama-server" -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue
          Write-Host "Server stopped." -ForegroundColor Green

# This file was originally licensed under Apache 2.0. It has been modified.
# Modifications Copyright (c) 2025 AMD