Make timestamps consistent (#10) #47
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow tests lemonade-eval CLI tools with lemonade-server | |
| # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python | |
| name: Test Lemonade Eval | |
| on: | |
| push: | |
| branches: ["main"] | |
| paths: | |
| - '.github/workflows/test_lemonade_eval.yml' | |
| - 'setup.py' | |
| - 'src/lemonade/**' | |
| pull_request: | |
| paths: | |
| - '.github/workflows/test_lemonade_eval.yml' | |
| - 'setup.py' | |
| - 'src/lemonade/**' | |
| merge_group: | |
| permissions: | |
| contents: read | |
| jobs: | |
| test-lemonade-eval: | |
| env: | |
| LEMONADE_CI_MODE: "True" | |
| LEMONADE_SERVER_URL: "http://localhost:8000" | |
| runs-on: windows-latest | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Create virtual environment and install dependencies | |
| shell: bash | |
| run: | | |
| python -m venv .venv | |
| venvPython=".venv/Scripts/python" | |
| venvPip=".venv/Scripts/pip" | |
| $venvPython -m pip install --upgrade pip | |
| $venvPip install pylint | |
| $venvPython -m pip check | |
| $venvPip install -e .[oga-cpu] | |
| - name: Lint with Black | |
| uses: psf/black@stable | |
| with: | |
| options: "--check --verbose" | |
| src: "./src/lemonade" | |
| - name: Lint with PyLint | |
| shell: bash | |
| run: | | |
| venvPylint=".venv/Scripts/pylint" | |
| $venvPylint src/lemonade --rcfile .pylintrc --disable E0401 | |
| - name: Download latest Lemonade Server MSI | |
| shell: PowerShell | |
| run: | | |
| Write-Host "Downloading latest lemonade-server-minimal.msi from releases..." -ForegroundColor Cyan | |
| # Use GitHub CLI to download the latest release asset | |
| gh release download --repo lemonade-sdk/lemonade --pattern "lemonade-server-minimal.msi" --output lemonade-server-minimal.msi | |
| if (-not (Test-Path "lemonade-server-minimal.msi")) { | |
| Write-Host "ERROR: Failed to download lemonade-server-minimal.msi" -ForegroundColor Red | |
| exit 1 | |
| } | |
| Write-Host "Downloaded lemonade-server-minimal.msi successfully" -ForegroundColor Green | |
| Get-ChildItem lemonade-server-minimal.msi | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| - name: Install Lemonade Server | |
| shell: PowerShell | |
| run: | | |
| $installPath = "${{ github.workspace }}\lemonade-server" | |
| $logPath = "${{ github.workspace }}\lemonade-msi-install.log" | |
| Write-Host "Installing Lemonade Server to: $installPath" -ForegroundColor Cyan | |
| # Run installer silently with verbose logging | |
| $args = "/i lemonade-server-minimal.msi /qn INSTALLDIR=`"$installPath`" /L*V `"$logPath`"" | |
| $p = Start-Process -FilePath "msiexec.exe" -ArgumentList $args -Wait -PassThru | |
| if ($p.ExitCode -ne 0) { | |
| Write-Host "ERROR: Installation failed with exit code $($p.ExitCode)" -ForegroundColor Red | |
| if (Test-Path $logPath) { | |
| Get-Content $logPath | Select-Object -Last 100 | |
| } | |
| exit $p.ExitCode | |
| } | |
| # Add to PATH for subsequent steps | |
| echo "$installPath\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append | |
| Write-Host "Installation complete. Verifying..." -ForegroundColor Green | |
| & "$installPath\bin\lemonade-server.exe" --version | |
| - name: Test OGA-Load (CPU ONNX model) | |
| shell: bash | |
| run: | | |
| venvPython=".venv/Scripts/python" | |
| venvLemonade=".venv/Scripts/lemonade-eval" | |
| echo "Testing OGA-Load with CPU ONNX model..." | |
| $venvLemonade -i amd/Qwen2.5-0.5B-Instruct-quantized_int4-float16-cpu-onnx oga-load --device cpu --dtype int4 llm-prompt -p "tell me a story" --max-new-tokens 5 | |
| echo "Running OGA CPU API tests..." | |
| $venvPython test/oga_cpu_api.py | |
| - name: Test Server Integration (GGUF model) | |
| shell: PowerShell | |
| run: | | |
| $installPath = "${{ github.workspace }}\lemonade-server" | |
| $serverExe = "$installPath\bin\lemonade-server.exe" | |
| $logFile = "${{ github.workspace }}\lemonade-server.log" | |
| $venvPython = ".venv\Scripts\python.exe" | |
| $venvLemonade = ".venv\Scripts\lemonade-eval" | |
| Write-Host "Starting Lemonade Server..." -ForegroundColor Cyan | |
| # Start server in background | |
| $serverProcess = Start-Process -FilePath $serverExe -ArgumentList "serve" -RedirectStandardOutput $logFile -RedirectStandardError "$logFile.err" -PassThru | |
| # Wait for server to be ready | |
| $maxAttempts = 30 | |
| $attempt = 0 | |
| $serverReady = $false | |
| while ($attempt -lt $maxAttempts -and -not $serverReady) { | |
| Start-Sleep -Seconds 2 | |
| $attempt++ | |
| Write-Host "Checking server health (attempt $attempt/$maxAttempts)..." | |
| try { | |
| $response = Invoke-WebRequest -Uri "http://localhost:8000/api/v1/health" -UseBasicParsing -TimeoutSec 5 | |
| if ($response.StatusCode -eq 200) { | |
| $serverReady = $true | |
| Write-Host "Server is ready!" -ForegroundColor Green | |
| } | |
| } catch { | |
| Write-Host "Server not ready yet..." | |
| } | |
| } | |
| if (-not $serverReady) { | |
| Write-Host "ERROR: Server failed to start within timeout" -ForegroundColor Red | |
| if (Test-Path $logFile) { | |
| Write-Host "=== Server Log ===" -ForegroundColor Yellow | |
| Get-Content $logFile | |
| } | |
| if (Test-Path "$logFile.err") { | |
| Write-Host "=== Server Error Log ===" -ForegroundColor Yellow | |
| Get-Content "$logFile.err" | |
| } | |
| exit 1 | |
| } | |
| # Test CLI | |
| Write-Host "Testing lemonade-eval CLI..." | |
| & $venvLemonade -m -i Llama-3.2-1B-Instruct-GGUF load bench -w 0 -i 5 | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| try { | |
| Write-Host "Running server integration tests with Qwen3-4B-Instruct-2507-GGUF..." -ForegroundColor Cyan | |
| & $venvPython test/llm_api.py | |
| if ($LASTEXITCODE -ne 0) { | |
| throw "Tests failed with exit code $LASTEXITCODE" | |
| } | |
| } finally { | |
| Write-Host "Stopping server..." -ForegroundColor Cyan | |
| Stop-Process -Id $serverProcess.Id -Force -ErrorAction SilentlyContinue | |
| # Show logs if tests failed | |
| if ($LASTEXITCODE -ne 0) { | |
| if (Test-Path $logFile) { | |
| Write-Host "=== Server Log ===" -ForegroundColor Yellow | |
| Get-Content $logFile | Select-Object -Last 50 | |
| } | |
| if (Test-Path "$logFile.err") { | |
| Write-Host "=== Server Error Log ===" -ForegroundColor Yellow | |
| Get-Content "$logFile.err" | Select-Object -Last 50 | |
| } | |
| } | |
| } | |
| - name: Stop Lemonade Server | |
| if: always() | |
| shell: PowerShell | |
| run: | | |
| Write-Host "Stopping Lemonade Server..." -ForegroundColor Cyan | |
| Get-Process -Name "lemonade-server", "lemonade-router", "llama-server" -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue | |
| Write-Host "Server stopped." -ForegroundColor Green | |
| # This file was originally licensed under Apache 2.0. It has been modified. | |
| # Modifications Copyright (c) 2025 AMD |