Integration Tests #241
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Tests | |
| on: | |
| # Trigger when a status check is posted (OpenShift CI completion) | |
| status: | |
| # Allow manual trigger with custom image tag | |
| workflow_dispatch: | |
| inputs: | |
| image_tag: | |
| description: 'Image tag to test (e.g., pr-123)' | |
| required: true | |
| type: string | |
| source_ref: | |
| description: 'Git ref to checkout for tests (branch, tag, or commit SHA)' | |
| required: true | |
| type: string | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| statuses: write | |
| jobs: | |
| # Extract image information from the trigger event | |
| setup: | |
| name: Determine Image Tag | |
| runs-on: ubuntu-latest | |
| # Only run for workflow_dispatch or relevant status events | |
| if: | | |
| github.event_name == 'workflow_dispatch' || | |
| (github.event_name == 'status' && | |
| github.event.state == 'success' && | |
| contains(github.event.context, 'ci/prow/pr-image-mirror-stable')) | |
| outputs: | |
| image_tag: ${{ steps.set_image_tag.outputs.image_tag }} | |
| source_ref: ${{ steps.set_source_ref.outputs.source_ref }} | |
| steps: | |
| - name: Extract branch name | |
| id: extract_branch_name | |
| if: github.event_name == 'status' | |
| run: | | |
| # Extract branch name from context: ci/prow/pr-image-mirror-stable -> stable | |
| CONTEXT="${{ github.event.context }}" | |
| BRANCH_NAME="${CONTEXT#ci/prow/pr-image-mirror-}" # Remove ci/prow/pr-image-mirror- prefix | |
| echo "branch_name=${BRANCH_NAME}" >> $GITHUB_OUTPUT | |
| echo "Branch name: ${BRANCH_NAME}" | |
| - name: Get PR information | |
| if: github.event_name == 'status' | |
| id: get_pr_info | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| COMMIT_SHA="${{ github.event.sha }}" | |
| BASE_BRANCH="${{ steps.extract_branch_name.outputs.branch_name }}" | |
| echo "Finding open PRs with commit ${COMMIT_SHA} at HEAD targeting branch ${BASE_BRANCH}" | |
| # Find open PRs targeting the extracted branch where this commit is at HEAD (not just in history) | |
| # This ensures we only test the image if the PR hasn't moved forward with new commits | |
| MATCHING_PRS=$(gh api "repos/${{ github.repository }}/commits/${COMMIT_SHA}/pulls" | \ | |
| jq --arg base "${BASE_BRANCH}" --arg sha "${COMMIT_SHA}" \ | |
| '[.[] | select(.state == "open" and .base.ref == $base and .head.sha == $sha)]') | |
| PR_COUNT=$(echo "${MATCHING_PRS}" | jq 'length') | |
| if [[ "${PR_COUNT}" -eq 0 ]]; then | |
| echo "No open PRs targeting ${BASE_BRANCH} branch found with commit ${COMMIT_SHA} at HEAD" | |
| exit 1 | |
| fi | |
| # Select the PR with the highest number (most recently created) | |
| PR_NUM=$(echo "${MATCHING_PRS}" | jq -r 'max_by(.number).number') | |
| if [[ "${PR_COUNT}" -gt 1 ]]; then | |
| echo "Found ${PR_COUNT} open PRs targeting ${BASE_BRANCH} branch with commit ${COMMIT_SHA} at HEAD" | |
| echo "${MATCHING_PRS}" | jq -r '.[] | " - PR #\(.number): \(.head.ref) -> \(.base.ref)"' | |
| echo "Using latest PR #${PR_NUM} for test image (images are identical since they are built from the same commit)" | |
| else | |
| echo "Found PR #${PR_NUM} targeting ${BASE_BRANCH}" | |
| fi | |
| echo "pr_number=${PR_NUM}" >> $GITHUB_OUTPUT | |
| - name: Set image tag | |
| id: set_image_tag | |
| run: | | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| TAG="${{ github.event.inputs.image_tag }}" | |
| else | |
| TAG="pr-${{ steps.get_pr_info.outputs.pr_number }}" | |
| fi | |
| echo "image_tag=${TAG}" >> $GITHUB_OUTPUT | |
| echo "Image tag: ${TAG}" | |
| - name: Set source ref | |
| id: set_source_ref | |
| run: | | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| REF="${{ github.event.inputs.source_ref }}" | |
| else | |
| REF="${{ github.event.sha }}" | |
| fi | |
| echo "source_ref=${REF}" >> $GITHUB_OUTPUT | |
| echo "Source ref: ${REF}" | |
| # Pull the image built by OpenShift CI | |
| pull-image: | |
| name: Pull Image from Quay.io | |
| runs-on: ubuntu-latest | |
| needs: setup | |
| steps: | |
| - name: Pull image from Quay.io | |
| env: | |
| IMAGE_TAG: ${{ needs.setup.outputs.image_tag }} | |
| run: | | |
| FULL_IMAGE="quay.io/opendatahub/openvino_model_server:${IMAGE_TAG}" | |
| echo "Pulling image: ${FULL_IMAGE}" | |
| docker pull ${FULL_IMAGE} | |
| docker tag ${FULL_IMAGE} openvino/model_server:latest | |
| - name: Save Docker image | |
| run: | | |
| docker save openvino/model_server:latest | gzip > model-server-image.tar.gz | |
| - name: Upload image artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: model-server-image | |
| path: model-server-image.tar.gz | |
| retention-days: 1 | |
| # File integrity test | |
| file-integrity: | |
| name: File Integrity Test | |
| runs-on: ubuntu-latest | |
| needs: [setup, pull-image] | |
| if: always() && needs.setup.result == 'success' | |
| steps: | |
| - name: Post pending status to commit | |
| if: github.event_name == 'status' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/statuses/${{ needs.setup.outputs.source_ref }} \ | |
| -f state=pending \ | |
| -f context="File Integrity Test" \ | |
| -f description="Running file integrity tests..." \ | |
| -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.setup.outputs.source_ref }} | |
| - name: Download Docker image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: model-server-image | |
| - name: Load Docker image | |
| run: | | |
| docker load < model-server-image.tar.gz | |
| - name: Run file integrity test | |
| run: | | |
| make run_lib_files_test | |
| - name: Post success status to commit | |
| if: github.event_name == 'status' && success() | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/statuses/${{ needs.setup.outputs.source_ref }} \ | |
| -f state=success \ | |
| -f context="File Integrity Test" \ | |
| -f description="File integrity tests passed" \ | |
| -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| - name: Post failure status to commit | |
| if: github.event_name == 'status' && failure() | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/statuses/${{ needs.setup.outputs.source_ref }} \ | |
| -f state=failure \ | |
| -f context="File Integrity Test" \ | |
| -f description="File integrity tests failed" \ | |
| -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| # Python clients test | |
| clients-test: | |
| name: Python Clients Test | |
| runs-on: ubuntu-latest | |
| needs: [setup, pull-image] | |
| if: always() && needs.setup.result == 'success' | |
| steps: | |
| - name: Post pending status to commit | |
| if: github.event_name == 'status' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/statuses/${{ needs.setup.outputs.source_ref }} \ | |
| -f state=pending \ | |
| -f context="Python Clients Test" \ | |
| -f description="Running Python client tests..." \ | |
| -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.setup.outputs.source_ref }} | |
| - name: Download Docker image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: model-server-image | |
| - name: Load Docker image | |
| run: | | |
| docker load < model-server-image.tar.gz | |
| - name: Run Python clients test | |
| run: | | |
| make test_python_clients | |
| - name: Post success status to commit | |
| if: github.event_name == 'status' && success() | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/statuses/${{ needs.setup.outputs.source_ref }} \ | |
| -f state=success \ | |
| -f context="Python Clients Test" \ | |
| -f description="Python client tests passed" \ | |
| -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| - name: Post failure status to commit | |
| if: github.event_name == 'status' && failure() | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/statuses/${{ needs.setup.outputs.source_ref }} \ | |
| -f state=failure \ | |
| -f context="Python Clients Test" \ | |
| -f description="Python client tests failed" \ | |
| -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| # Latency performance test | |
| latency-test: | |
| name: Latency Performance Test | |
| runs-on: ubuntu-latest | |
| needs: [setup, pull-image] | |
| if: always() && needs.setup.result == 'success' | |
| steps: | |
| - name: Post pending status to commit | |
| if: github.event_name == 'status' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/statuses/${{ needs.setup.outputs.source_ref }} \ | |
| -f state=pending \ | |
| -f context="Latency Performance Test" \ | |
| -f description="Running latency tests..." \ | |
| -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.setup.outputs.source_ref }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Download Docker image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: model-server-image | |
| - name: Load Docker image | |
| run: | | |
| docker load < model-server-image.tar.gz | |
| - name: Run latency test | |
| run: | | |
| make test_perf | |
| - name: Post success status to commit | |
| if: github.event_name == 'status' && success() | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/statuses/${{ needs.setup.outputs.source_ref }} \ | |
| -f state=success \ | |
| -f context="Latency Performance Test" \ | |
| -f description="Latency tests passed" \ | |
| -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| - name: Post failure status to commit | |
| if: github.event_name == 'status' && failure() | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/statuses/${{ needs.setup.outputs.source_ref }} \ | |
| -f state=failure \ | |
| -f context="Latency Performance Test" \ | |
| -f description="Latency tests failed" \ | |
| -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| # Throughput performance test | |
| throughput-test: | |
| name: Throughput Performance Test | |
| runs-on: ubuntu-latest | |
| needs: [setup, pull-image] | |
| if: always() && needs.setup.result == 'success' | |
| steps: | |
| - name: Post pending status to commit | |
| if: github.event_name == 'status' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/statuses/${{ needs.setup.outputs.source_ref }} \ | |
| -f state=pending \ | |
| -f context="Throughput Performance Test" \ | |
| -f description="Running throughput tests..." \ | |
| -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.setup.outputs.source_ref }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Download Docker image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: model-server-image | |
| - name: Load Docker image | |
| run: | | |
| docker load < model-server-image.tar.gz | |
| - name: Run throughput test | |
| run: | | |
| make test_throughput | |
| - name: Post success status to commit | |
| if: github.event_name == 'status' && success() | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/statuses/${{ needs.setup.outputs.source_ref }} \ | |
| -f state=success \ | |
| -f context="Throughput Performance Test" \ | |
| -f description="Throughput tests passed" \ | |
| -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| - name: Post failure status to commit | |
| if: github.event_name == 'status' && failure() | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/statuses/${{ needs.setup.outputs.source_ref }} \ | |
| -f state=failure \ | |
| -f context="Throughput Performance Test" \ | |
| -f description="Throughput tests failed" \ | |
| -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| # Functional tests | |
| functional-tests: | |
| name: Functional Tests | |
| runs-on: ubuntu-latest | |
| needs: [setup, pull-image] | |
| if: always() && needs.setup.result == 'success' | |
| steps: | |
| - name: Post pending status to commit | |
| if: github.event_name == 'status' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/statuses/${{ needs.setup.outputs.source_ref }} \ | |
| -f state=pending \ | |
| -f context="Functional Tests" \ | |
| -f description="Running functional tests..." \ | |
| -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.setup.outputs.source_ref }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Download Docker image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: model-server-image | |
| - name: Load Docker image | |
| run: | | |
| docker load < model-server-image.tar.gz | |
| - name: Run functional tests | |
| env: | |
| IMAGE: openvino/model_server:latest | |
| run: | | |
| make test_functional | |
| - name: Post success status to commit | |
| if: github.event_name == 'status' && success() | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/statuses/${{ needs.setup.outputs.source_ref }} \ | |
| -f state=success \ | |
| -f context="Functional Tests" \ | |
| -f description="Functional tests passed" \ | |
| -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| - name: Post failure status to commit | |
| if: github.event_name == 'status' && failure() | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/statuses/${{ needs.setup.outputs.source_ref }} \ | |
| -f state=failure \ | |
| -f context="Functional Tests" \ | |
| -f description="Functional tests failed" \ | |
| -f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" |