Build, test, and publish Red Hat Distribution Containers #212
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build, test, and publish Red Hat Distribution Containers | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| - rhoai-v* | |
| - konflux-poc* | |
| types: | |
| - opened | |
| - synchronize | |
| paths: | |
| - '.github/actions/**' | |
| - '.github/workflows/redhat-distro-container.yml' | |
| - 'distribution/**' | |
| - 'tests/**' | |
| push: | |
| branches: | |
| - main | |
| - rhoai-v* | |
| # build a custom image from an arbitrary llama-stack commit | |
| # NOTE: workflow_dispatch intentionally skips all tests (vllm setup, postgres setup, smoke tests, integration tests) | |
| # This allows building images for specific SHAs even when CI is failing on other commits, | |
| # useful for testing specific changes without being blocked by unrelated test failures. | |
| workflow_dispatch: | |
| inputs: | |
| llama_stack_commit_sha: | |
| description: 'Llama Stack commit SHA to build from - accept long and short commit SHAs' | |
| required: true | |
| type: string | |
| # do a nightly test of the `main` branch of llama-stack at 6AM UTC every morning | |
| schedule: | |
| - cron: '0 6 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| REGISTRY: quay.io | |
| IMAGE_NAME: quay.io/opendatahub/llama-stack # tags for the image will be added dynamically | |
| jobs: | |
| build-test: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - arch: amd64 | |
| runner: ubuntu-24.04 | |
| platform: linux/amd64 | |
| - arch: arm64 | |
| runner: ubuntu-24.04-arm | |
| platform: linux/arm64 | |
| runs-on: ${{ matrix.runner }} | |
| outputs: | |
| distribution-changed: ${{ steps.distribution-changed.outputs.changed }} | |
| env: | |
| VERTEX_AI_PROJECT: ${{ secrets.VERTEX_AI_PROJECT }} | |
| GCP_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| # Model names - set defaults, overridden by MaaS configuration step | |
| VLLM_INFERENCE_MODEL: vllm-inference/Qwen/Qwen3-0.6B | |
| VERTEX_AI_INFERENCE_MODEL: vertexai/publishers/google/models/gemini-2.0-flash | |
| OPENAI_INFERENCE_MODEL: openai/gpt-4o-mini | |
| EMBEDDING_MODEL: vllm-embedding/ibm-granite/granite-embedding-125m-english | |
| VLLM_URL: http://localhost:8000/v1 | |
| VLLM_EMBEDDING_URL: http://localhost:8001/v1 | |
| LLAMA_STACK_COMMIT_SHA: ${{ github.event.inputs.llama_stack_commit_sha || 'main' }} | |
| permissions: | |
| id-token: write # for Google Cloud authentication | |
| contents: read | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| fetch-depth: 2 # Need parent commit to detect changes | |
| - name: Check if distribution directory changed | |
| id: distribution-changed | |
| if: github.event_name == 'push' | |
| run: | | |
| # Check if any file in the distribution directory was modified in this push | |
| # Use git diff instead of parsing event payload (which is unreliable for merge commits) | |
| if git diff --name-only HEAD^ HEAD | grep -q '^distribution/'; then | |
| echo "changed=true" >> "$GITHUB_OUTPUT" | |
| echo "distribution/ was modified in this push, will publish" | |
| else | |
| echo "changed=false" >> "$GITHUB_OUTPUT" | |
| echo "distribution/ was not modified in this push, skipping publish" | |
| fi | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 | |
| with: | |
| python-version: 3.12 | |
| enable-cache: false | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 | |
| - name: Generate Containerfile to build an image from an arbitrary llama-stack commit (workflow_dispatch/schedule) | |
| if: contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name) | |
| env: | |
| LLAMA_STACK_VERSION: ${{ env.LLAMA_STACK_COMMIT_SHA }} | |
| run: | | |
| tmp_build_dir=$(mktemp -d) | |
| git clone --filter=blob:none --no-checkout https://github.com/opendatahub-io/llama-stack.git "$tmp_build_dir" | |
| cd "$tmp_build_dir" | |
| git checkout "$LLAMA_STACK_VERSION" | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pip install --no-cache -e . | |
| # now remove the install line from the Containerfile | |
| cd - | |
| python3 distribution/build.py | |
| sed -i '/^RUN pip install --no-cache llama-stack==/d' distribution/Containerfile | |
| - name: Build image for testing (${{ matrix.arch }}) | |
| id: build | |
| uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 | |
| with: | |
| context: . | |
| file: distribution/Containerfile | |
| platforms: ${{ matrix.platform }} | |
| push: false | |
| tags: ${{ env.IMAGE_NAME }}:${{ contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name) && format('source-{0}-{1}', env.LLAMA_STACK_COMMIT_SHA, github.sha) || github.sha }} | |
| load: true | |
| cache-from: type=gha,scope=build-${{ matrix.arch }} | |
| cache-to: type=gha,mode=max,scope=build-${{ matrix.arch }} | |
| - name: Unset cloud provider credentials for fork and Dependabot PRs (secrets not available) | |
| if: github.event_name == 'pull_request' && (github.event.pull_request.head.repo.fork == true || github.secret_source == 'Dependabot') | |
| run: | | |
| echo "Unsetting cloud provider credentials for fork/Dependabot PR (secrets not available)" | |
| { | |
| echo "VERTEX_AI_PROJECT=" | |
| echo "GCP_WORKLOAD_IDENTITY_PROVIDER=" | |
| echo "OPENAI_API_KEY=" | |
| } >> "$GITHUB_ENV" | |
| - name: Configure MaaS vLLM endpoints | |
| env: | |
| _MAAS_VLLM_URL: ${{ secrets.MAAS_VLLM_URL }} | |
| _MAAS_VLLM_API_TOKEN: ${{ secrets.MAAS_VLLM_API_TOKEN }} | |
| _MAAS_EMBEDDING_URL: ${{ secrets.MAAS_EMBEDDING_URL }} | |
| _MAAS_EMBEDDING_API_TOKEN: ${{ secrets.MAAS_EMBEDDING_API_TOKEN }} | |
| run: | | |
| if [ -z "$_MAAS_VLLM_URL" ]; then | |
| echo "MaaS vLLM URL not configured, skipping" | |
| exit 0 | |
| fi | |
| echo "Configuring MaaS vLLM endpoints for testing" | |
| { | |
| echo "VLLM_URL=$_MAAS_VLLM_URL" | |
| echo "VLLM_API_TOKEN=$_MAAS_VLLM_API_TOKEN" | |
| echo "VLLM_INFERENCE_MODEL=vllm-inference/llama-3-2-3b" | |
| echo "VLLM_EMBEDDING_URL=$_MAAS_EMBEDDING_URL" | |
| echo "VLLM_EMBEDDING_API_TOKEN=$_MAAS_EMBEDDING_API_TOKEN" | |
| echo "EMBEDDING_MODEL=vllm-embedding/nomic-embed-text-v1-5" | |
| echo "EMBEDDING_PROVIDER=vllm-embedding" | |
| echo "EMBEDDING_PROVIDER_MODEL_ID=nomic-embed-text-v1-5" | |
| echo "USING_MAAS=true" | |
| } >> "$GITHUB_ENV" | |
| - name: Authenticate to Google Cloud (Vertex) | |
| if: matrix.arch == 'amd64' && github.event_name != 'workflow_dispatch' && (github.event_name != 'pull_request' || (github.event.pull_request.head.repo.fork == false && github.secret_source != 'Dependabot')) | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 | |
| with: | |
| project_id: ${{ env.VERTEX_AI_PROJECT }} | |
| workload_identity_provider: ${{ env.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| - name: Setup vLLM inference container | |
| if: github.event_name != 'workflow_dispatch' && env.USING_MAAS != 'true' | |
| id: vllm-inference | |
| uses: ./.github/actions/setup-vllm | |
| env: | |
| VLLM_IMAGE: quay.io/opendatahub/vllm-cpu:Qwen3-0.6B-granite-embedding-125m-english | |
| VLLM_MODE: inference | |
| - name: Setup vLLM embedding container | |
| if: github.event_name != 'workflow_dispatch' && env.USING_MAAS != 'true' | |
| id: vllm-embedding | |
| uses: ./.github/actions/setup-vllm | |
| env: | |
| VLLM_IMAGE: quay.io/opendatahub/vllm-cpu:Qwen3-0.6B-granite-embedding-125m-english | |
| VLLM_MODE: embedding | |
| - name: Setup PostgreSQL container | |
| if: github.event_name != 'workflow_dispatch' | |
| id: postgres | |
| uses: ./.github/actions/setup-postgres | |
| - name: Start and smoke test LLS distro image | |
| if: github.event_name != 'workflow_dispatch' | |
| id: smoke-test | |
| shell: bash | |
| run: ./tests/smoke.sh | |
| - name: Provider tests | |
| if: github.event_name != 'workflow_dispatch' | |
| id: provider-tests | |
| shell: bash | |
| env: | |
| IMAGE_TAG: ${{ contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name) && format('source-{0}-{1}', env.LLAMA_STACK_COMMIT_SHA, github.sha) || github.sha }} | |
| run: ./tests/test_providers.sh | |
| - name: Integration tests | |
| if: github.event_name != 'workflow_dispatch' && (matrix.arch == 'amd64' || env.USING_MAAS == 'true') | |
| id: integration-tests | |
| shell: bash | |
| run: ./tests/run_integration_tests.sh | |
| - name: Gather logs and debugging information | |
| if: always() | |
| shell: bash | |
| run: | | |
| # Create logs directory | |
| mkdir -p logs | |
| docker logs llama-stack > logs/llama-stack.log 2>&1 || echo "Failed to get llama-stack logs" > logs/llama-stack.log | |
| docker logs vllm-inference > logs/vllm-inference.log 2>&1 || echo "Failed to get vllm-inference logs" > logs/vllm-inference.log | |
| docker logs vllm-embedding > logs/vllm-embedding.log 2>&1 || echo "Failed to get vllm-embedding logs" > logs/vllm-embedding.log | |
| docker logs postgres > logs/postgres.log 2>&1 || echo "Failed to get postgres logs" > logs/postgres.log | |
| # Gather system information | |
| echo "=== System information ===" | |
| { | |
| echo "Disk usage:" | |
| df -h | |
| echo "Memory usage:" | |
| free -h | |
| echo "Docker images:" | |
| docker images | |
| echo "Docker containers:" | |
| docker ps -a | |
| } > logs/system-info.log 2>&1 | |
| # Gather integration test logs if they exist | |
| echo "=== Integration test artifacts ===" | |
| if [ -d "/tmp/llama-stack-integration-tests" ]; then | |
| find /tmp/llama-stack-integration-tests -name "*.log" -o -name "pytest.log" -o -name "*.out" 2>/dev/null | while read -r file; do | |
| cp "$file" "logs/$(basename "$file")" || true | |
| done | |
| fi | |
| - name: Upload logs as artifacts | |
| if: always() | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: ci-logs-${{ matrix.arch }}-${{ github.sha }} | |
| path: logs/ | |
| retention-days: 7 | |
| - name: Cleanup | |
| if: always() | |
| shell: bash | |
| run: | | |
| for c in vllm-inference vllm-embedding llama-stack postgres; do | |
| docker rm -f "$c" 2>/dev/null || true | |
| done | |
| # Set status so the single Notify step knows whether to send success or failure message. | |
| - name: Set Slack notify status | |
| if: failure() | |
| run: echo "NOTIFY_FAILURE=1" >> "$GITHUB_ENV" | |
| - name: Notify Slack | |
| if: always() && github.event_name != 'pull_request' && failure() | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.WH_SLACK_TEAM_LLS_CORE }} | |
| IMAGE_NAME: ${{ env.IMAGE_NAME }} | |
| IMAGE_TAG: ${{ contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name) && format('source-{0}-{1}', env.LLAMA_STACK_COMMIT_SHA, github.sha) || github.sha }} | |
| COMMIT_SHA: ${{ github.sha }} | |
| WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| COMMIT_SHA_SHORT="${COMMIT_SHA:0:7}" | |
| TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ) | |
| if [[ "${NOTIFY_FAILURE:-0}" == "1" ]]; then | |
| TEXT=$(printf '%s\n%s\n%s' \ | |
| ":failed: *Build failed for Llama Stack* - [${TIMESTAMP}]" \ | |
| "Commit: ${COMMIT_SHA_SHORT}" \ | |
| "<${WORKFLOW_URL}|View workflow run>") | |
| bash .github/actions/notify-slack/notify.sh "$TEXT" "#d00000" | |
| else | |
| TEXT=$(printf '%s\n%s\n%s\n%s' \ | |
| ":greenchecked: *New image is available for Llama Stack* - [${TIMESTAMP}]" \ | |
| "Image: ${IMAGE_NAME}:${IMAGE_TAG}" \ | |
| "Commit: ${COMMIT_SHA_SHORT}" \ | |
| "<${WORKFLOW_URL}|View workflow run>") | |
| bash .github/actions/notify-slack/notify.sh "$TEXT" | |
| fi | |
| - name: Output custom build information | |
| if: matrix.arch == 'amd64' && contains(fromJSON('["workflow_dispatch", "schedule"]'), github.event_name) | |
| env: | |
| LLS_COMMIT: ${{ env.LLAMA_STACK_COMMIT_SHA }} | |
| run: | | |
| echo "Custom container image built successfully!" | |
| echo "Image: $IMAGE_NAME:source-$LLS_COMMIT-$GITHUB_SHA" | |
| echo "Llama Stack commit: $LLS_COMMIT" | |
| echo "" | |
| echo "You can pull this image using:" | |
| echo "docker pull $IMAGE_NAME:source-$LLS_COMMIT-$GITHUB_SHA" | |
| publish: | |
| needs: build-test | |
| if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && needs.build-test.outputs.distribution-changed == 'true') | |
| runs-on: ubuntu-latest | |
| env: | |
| LLAMA_STACK_COMMIT_SHA: ${{ github.event.inputs.llama_stack_commit_sha || 'main' }} | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Install uv | |
| if: github.event_name == 'workflow_dispatch' | |
| uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 | |
| with: | |
| python-version: 3.12 | |
| - name: Generate Containerfile to build an image from an arbitrary llama-stack commit (workflow_dispatch) | |
| if: github.event_name == 'workflow_dispatch' | |
| env: | |
| LLAMA_STACK_VERSION: ${{ env.LLAMA_STACK_COMMIT_SHA }} | |
| run: | | |
| tmp_build_dir=$(mktemp -d) | |
| git clone --filter=blob:none --no-checkout https://github.com/opendatahub-io/llama-stack.git "$tmp_build_dir" | |
| cd "$tmp_build_dir" | |
| git checkout "$LLAMA_STACK_VERSION" | |
| python3 -m venv .venv | |
| source .venv/bin/activate | |
| pip install --no-cache -e . | |
| # now remove the install line from the Containerfile | |
| cd - | |
| python3 distribution/build.py | |
| sed -i '/^RUN pip install --no-cache llama-stack==/d' distribution/Containerfile | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 | |
| - name: Log in to Quay.io | |
| uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ secrets.QUAY_USERNAME }} | |
| password: ${{ secrets.QUAY_PASSWORD }} | |
| - name: Publish multi-arch image to Quay.io | |
| uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 | |
| with: | |
| context: . | |
| file: distribution/Containerfile | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| tags: ${{ github.event_name == 'workflow_dispatch' && format('{0}:source-{1}-{2}', env.IMAGE_NAME, env.LLAMA_STACK_COMMIT_SHA, github.sha) || format('{0}:{1}{2}', env.IMAGE_NAME, github.sha, github.ref == 'refs/heads/main' && format(',{0}:latest', env.IMAGE_NAME) || (startsWith(github.ref, 'refs/heads/rhoai-v') && format(',{0}:{1}-latest', env.IMAGE_NAME, github.ref_name)) || '') }} | |
| cache-from: | | |
| type=gha,scope=build-amd64 | |
| type=gha,scope=build-arm64 | |
| - name: Set Slack notify status | |
| if: failure() | |
| run: echo "NOTIFY_FAILURE=1" >> "$GITHUB_ENV" | |
| - name: Notify Slack | |
| if: always() | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.WH_SLACK_TEAM_LLS_CORE }} | |
| IMAGE_NAME: ${{ env.IMAGE_NAME }} | |
| IMAGE_TAG: ${{ github.event_name == 'workflow_dispatch' && format('source-{0}-{1}', env.LLAMA_STACK_COMMIT_SHA, github.sha) || github.sha }} | |
| COMMIT_SHA: ${{ github.sha }} | |
| WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| COMMIT_SHA_SHORT="${COMMIT_SHA:0:7}" | |
| TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ) | |
| if [[ "${NOTIFY_FAILURE:-0}" == "1" ]]; then | |
| TEXT=$(printf '%s\n%s\n%s' \ | |
| ":failed: *Build failed for Llama Stack* - [${TIMESTAMP}]" \ | |
| "Commit: ${COMMIT_SHA_SHORT}" \ | |
| "<${WORKFLOW_URL}|View workflow run>") | |
| bash .github/actions/notify-slack/notify.sh "$TEXT" "#d00000" | |
| else | |
| TEXT=$(printf '%s\n%s\n%s\n%s' \ | |
| ":greenchecked: *New image is available for Llama Stack* - [${TIMESTAMP}]" \ | |
| "Image: ${IMAGE_NAME}:${IMAGE_TAG}" \ | |
| "Commit: ${COMMIT_SHA_SHORT}" \ | |
| "<${WORKFLOW_URL}|View workflow run>") | |
| bash .github/actions/notify-slack/notify.sh "$TEXT" | |
| fi |