[Feat] implement ModelEvaluator for automated onboarding flow (#1153) #2337
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Test [Kubernetes] | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| branches: | |
| - main | |
| paths-ignore: | |
| - 'website/**' | |
| - '**/*.md' | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - 'website/**' | |
| - '**/*.md' | |
| workflow_dispatch: # Allow manual triggering | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| # Detect which files changed to determine which profiles to test | |
| changes: | |
| uses: ./.github/workflows/ci-changes.yml | |
| # Determine which profiles need to be tested based on file changes | |
| determine-profiles: | |
| needs: changes | |
| runs-on: ubuntu-latest | |
| outputs: | |
| profiles: ${{ steps.set-matrix.outputs.profiles }} | |
| should_run: ${{ steps.set-matrix.outputs.should_run }} | |
| steps: | |
| - id: set-matrix | |
| run: | | |
| # Run all profiles if common e2e code, core code changes, or manual/scheduled trigger | |
| if [[ "${{ needs.changes.outputs.e2e_common }}" == "true" ]] || \ | |
| [[ "${{ needs.changes.outputs.core }}" == "true" ]] || \ | |
| [[ "${{ github.event_name }}" == "schedule" ]] || \ | |
| [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| echo 'profiles=["ai-gateway", "aibrix", "routing-strategies", "dynamic-config", "llm-d", "istio", "production-stack", "response-api", "response-api-redis", "response-api-redis-cluster", "ml-model-selection"]' >> $GITHUB_OUTPUT | |
| echo 'should_run=true' >> $GITHUB_OUTPUT | |
| echo "Running all profiles due to common/core changes or push/schedule/manual trigger" | |
| exit 0 | |
| fi | |
| # Only run affected profiles for PRs | |
| profiles=() | |
| [[ "${{ needs.changes.outputs.e2e_istio }}" == "true" ]] && profiles+=("istio") | |
| [[ "${{ needs.changes.outputs.e2e_ai_gateway }}" == "true" ]] && profiles+=("ai-gateway") | |
| [[ "${{ needs.changes.outputs.e2e_aibrix }}" == "true" ]] && profiles+=("aibrix") | |
| [[ "${{ needs.changes.outputs.e2e_llm_d }}" == "true" ]] && profiles+=("llm-d") | |
| [[ "${{ needs.changes.outputs.e2e_routing_strategies }}" == "true" ]] && profiles+=("routing-strategies") | |
| [[ "${{ needs.changes.outputs.e2e_production_stack }}" == "true" ]] && profiles+=("production-stack") | |
| [[ "${{ needs.changes.outputs.e2e_dynamic_config }}" == "true" ]] && profiles+=("dynamic-config") | |
| [[ "${{ needs.changes.outputs.e2e_response_api }}" == "true" ]] && profiles+=("response-api") | |
| [[ "${{ needs.changes.outputs.e2e_response_api_redis }}" == "true" ]] && profiles+=("response-api-redis") | |
| [[ "${{ needs.changes.outputs.e2e_response_api_redis_cluster }}" == "true" ]] && profiles+=("response-api-redis-cluster") | |
| [[ "${{ needs.changes.outputs.e2e_ml_model_selection }}" == "true" ]] && profiles+=("ml-model-selection") | |
| # Convert to JSON array | |
| if [ ${#profiles[@]} -eq 0 ]; then | |
| echo 'profiles=[]' >> $GITHUB_OUTPUT | |
| echo 'should_run=false' >> $GITHUB_OUTPUT | |
| echo "No profile changes detected, skipping all e2e tests" | |
| else | |
| printf -v json '"%s",' "${profiles[@]}" | |
| echo "profiles=[${json%,}]" >> $GITHUB_OUTPUT | |
| echo 'should_run=true' >> $GITHUB_OUTPUT | |
| echo "Running profiles: ${profiles[*]}" | |
| fi | |
| integration-test: | |
| needs: [changes, determine-profiles] | |
| if: ${{ needs.determine-profiles.outputs.should_run == 'true' && !github.event.pull_request.draft }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 75 | |
| strategy: | |
| fail-fast: false # Continue testing other profiles even if one fails | |
| matrix: | |
| # Dynamic profile matrix based on detected changes | |
| profile: ${{ fromJson(needs.determine-profiles.outputs.profiles) }} | |
| steps: | |
| - name: Check out the repo | |
| uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.24' | |
| - name: Set up Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: "1.90" | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| make \ | |
| build-essential \ | |
| pkg-config | |
| - name: Install Kind | |
| run: | | |
| ARCH="$(uname -m)" | |
| case "$ARCH" in | |
| x86_64) KIND_ARCH="amd64" ;; | |
| aarch64) KIND_ARCH="arm64" ;; | |
| *) echo "unsupported arch: $ARCH" && exit 1 ;; | |
| esac | |
| curl --retry 5 --retry-delay 5 --retry-all-errors -Lo ./kind "https://kind.sigs.k8s.io/dl/v0.22.0/kind-linux-${KIND_ARCH}" | |
| chmod +x ./kind | |
| sudo mv ./kind /usr/local/bin/kind | |
| - name: Download E2E test dependencies | |
| run: | | |
| cd e2e && go mod download | |
| - name: Build E2E test binary | |
| run: | | |
| make build-e2e | |
| - name: Run Integration E2E tests (${{ matrix.profile }}) | |
| id: e2e-test | |
| env: | |
| # HF_TOKEN is required for downloading gated models (e.g., embeddinggemma-300m) | |
| # For PRs from forks, this will be empty and the E2E framework will gracefully skip gated model downloads | |
| # The hf CLI uses HUGGINGFACE_HUB_TOKEN, so we set both for compatibility | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| HUGGINGFACE_HUB_TOKEN: ${{ secrets.HF_TOKEN }} | |
| run: | | |
| set +e # Don't exit on error, we want to capture the result | |
| make e2e-test E2E_PROFILE=${{ matrix.profile }} E2E_VERBOSE=true E2E_KEEP_CLUSTER=false | |
| TEST_EXIT_CODE=$? | |
| echo "test_exit_code=${TEST_EXIT_CODE}" >> $GITHUB_OUTPUT | |
| exit ${TEST_EXIT_CODE} | |
| - name: Upload test reports | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-reports-${{ matrix.profile }} | |
| path: | | |
| test-report.json | |
| test-report.md | |
| semantic-router-logs.txt | |
| retention-days: 30 | |
| - name: Create test summary from report | |
| if: always() | |
| run: | | |
| if [ -f "test-report.md" ]; then | |
| echo "=== Reading test report from test-report.md ===" | |
| cat test-report.md >> $GITHUB_STEP_SUMMARY | |
| # Add semantic-router logs section if available | |
| if [ -f "semantic-router-logs.txt" ]; then | |
| cat >> $GITHUB_STEP_SUMMARY << 'EOF' | |
| --- | |
| ### 📝 Semantic Router Logs | |
| <details> | |
| <summary>Click to view semantic-router logs</summary> | |
| ``` | |
| EOF | |
| # Add first 500 lines of logs to summary (to avoid exceeding GitHub limits) | |
| head -n 500 semantic-router-logs.txt >> $GITHUB_STEP_SUMMARY | |
| # Check if there are more lines | |
| TOTAL_LINES=$(wc -l < semantic-router-logs.txt) | |
| if [ "$TOTAL_LINES" -gt 500 ]; then | |
| cat >> $GITHUB_STEP_SUMMARY << EOF | |
| ... (showing first 500 lines of $TOTAL_LINES total lines) | |
| 📦 Full logs are available in the workflow artifacts: semantic-router-logs.txt | |
| EOF | |
| fi | |
| cat >> $GITHUB_STEP_SUMMARY << 'EOF' | |
| ``` | |
| </details> | |
| EOF | |
| fi | |
| # Add additional context | |
| cat >> $GITHUB_STEP_SUMMARY << EOF | |
| --- | |
| ### 📚 Additional Resources | |
| - **Profile:** \`${{ matrix.profile }}\` | |
| - **Trigger:** ${{ github.event_name }} | |
| - **Branch:** \`${{ github.ref_name }}\` | |
| - **Commit:** \`${{ github.sha }}\` | |
| - **Workflow Run:** [${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| - [E2E Test Framework Documentation](https://github.com/${{ github.repository }}/tree/main/e2e) | |
| - [${{ matrix.profile }} Profile](https://github.com/${{ github.repository }}/tree/main/e2e/profiles/${{ matrix.profile }}) | |
| ### 📦 Artifacts | |
| - **test-report.json** - Detailed test results in JSON format | |
| - **test-report.md** - Human-readable test report | |
| - **semantic-router-logs.txt** - Complete semantic-router pod logs | |
| - All artifacts are retained for 30 days as \`test-reports-${{ matrix.profile }}\` | |
| EOF | |
| else | |
| echo "⚠️ Test report file not found!" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "The E2E test framework did not generate a report file." >> $GITHUB_STEP_SUMMARY | |
| echo "This might indicate that the test failed before report generation." >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Clean up | |
| if: always() | |
| run: | | |
| make e2e-cleanup || true |