Update docker-build.yaml to remove PR trigger #212
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Build and Push Docker Images | |
| on: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| runtime_version: | |
| description: 'Specific runtime version to build (e.g., "16.4 LTS"). Leave empty to build all LTS runtimes.' | |
| required: false | |
| type: string | |
| image_type: | |
| description: 'Image type: "all" builds all images (minimal/standard/python + GPU variants), "cpu" builds CPU-only, "gpu" builds GPU images only' | |
| required: false | |
| type: choice | |
| options: | |
| - all | |
| - cpu | |
| - gpu | |
| default: all | |
| all_lts: | |
| description: 'Build all LTS versions (default: latest 1 LTS only)' | |
| required: false | |
| type: boolean | |
| default: false | |
| lts_count: | |
| description: 'Number of latest LTS versions to build (default: 1)' | |
| required: false | |
| type: number | |
| default: 1 | |
| force_ubuntu_version: | |
| description: 'Force specific Ubuntu version for base images (e.g., "22.04" or "24.04")' | |
| required: false | |
| type: string | |
| include_ml_variants: | |
| description: 'Include ML runtime variants (default: false)' | |
| required: false | |
| type: boolean | |
| default: false | |
| push_images: | |
| description: 'Push images to registry' | |
| required: false | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_PREFIX: ${{ github.repository_owner }}/dbx-runtime | |
| jobs: | |
| generate-dockerfiles: | |
| name: Generate Dockerfiles | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| has_gpu_images: ${{ steps.check-images.outputs.has_gpu }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.11' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "**/uv.lock" | |
| - name: Install dependencies | |
| run: uv sync --all-groups --all-extras --frozen | |
| - name: Generate Dockerfiles | |
| run: | | |
| BUILD_CMD="uv run dbx-container build --output-dir data --registry '${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}'" | |
| # Runtime version filter | |
| if [ -n "${{ github.event.inputs.runtime_version }}" ]; then | |
| BUILD_CMD="$BUILD_CMD --runtime-version '${{ github.event.inputs.runtime_version }}'" | |
| fi | |
| # LTS filtering | |
| if [ "${{ github.event.inputs.all_lts }}" = "true" ]; then | |
| BUILD_CMD="$BUILD_CMD --all-lts" | |
| elif [ -n "${{ github.event.inputs.lts_count }}" ]; then | |
| BUILD_CMD="$BUILD_CMD --lts-count ${{ github.event.inputs.lts_count }}" | |
| fi | |
| # Image type filter | |
| if [ "${{ github.event.inputs.image_type }}" = "gpu" ]; then | |
| BUILD_CMD="$BUILD_CMD --image-type gpu" | |
| fi | |
| # Ubuntu version | |
| if [ -n "${{ github.event.inputs.force_ubuntu_version }}" ]; then | |
| BUILD_CMD="$BUILD_CMD --force-ubuntu-version '${{ github.event.inputs.force_ubuntu_version }}'" | |
| fi | |
| # ML variants | |
| if [ "${{ github.event.inputs.include_ml_variants }}" = "true" ]; then | |
| BUILD_CMD="$BUILD_CMD --include-ml-variants" | |
| fi | |
| eval $BUILD_CMD | |
| - name: Generate build matrix | |
| id: set-matrix | |
| run: | | |
| MATRIX_CMD="uv run dbx-container generate-matrix --only-lts" | |
| # Configure image types for the matrix | |
| if [ "${{ github.event.inputs.image_type }}" = "gpu" ]; then | |
| # Only GPU images | |
| MATRIX_CMD="$MATRIX_CMD --image-type python-gpu" | |
| elif [ "${{ github.event.inputs.image_type }}" = "cpu" ]; then | |
| # Only CPU images | |
| MATRIX_CMD="$MATRIX_CMD --image-type python" | |
| fi | |
| # For "all", don't filter - generate matrix will include both python and python-gpu | |
| # Apply LTS count filter | |
| if [ -n "${{ github.event.inputs.lts_count }}" ] && [ "${{ github.event.inputs.all_lts }}" != "true" ]; then | |
| MATRIX_CMD="$MATRIX_CMD --latest-lts-count ${{ github.event.inputs.lts_count }}" | |
| fi | |
| # Capture output and suppress stderr warnings (SSL warnings, uv messages, etc) | |
| MATRIX_JSON=$(eval $MATRIX_CMD 2>&1 | grep -v "WARNING" | grep -v "⚠" | grep -v "Skipping virtualenv" | grep "^{") | |
| # Validate JSON before outputting | |
| if echo "$MATRIX_JSON" | jq -e . >/dev/null 2>&1; then | |
| echo "matrix=$(echo $MATRIX_JSON | jq -c .)" >> $GITHUB_OUTPUT | |
| echo "Generated matrix with $(echo $MATRIX_JSON | jq '.include | length') entries" | |
| else | |
| echo "Error: Invalid JSON generated" | |
| echo "$MATRIX_JSON" | |
| exit 1 | |
| fi | |
| - name: Check for GPU images | |
| id: check-images | |
| run: | | |
| if [ -d "data/gpu" ] && [ "$(ls -A data/gpu 2>/dev/null)" ]; then | |
| echo "has_gpu=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "has_gpu=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Upload Dockerfiles | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: dockerfiles | |
| path: data/ | |
| retention-days: 7 | |
| # Build minimal image (base image for all variants) | |
| build-minimal: | |
| name: Build minimal | |
| needs: generate-dockerfiles | |
| if: github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Download Dockerfiles | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: dockerfiles | |
| path: data/ | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Log in to Container Registry | |
| if: github.event_name != 'pull_request' && (github.event.inputs.push_images == 'true' || github.ref == 'refs/heads/main') | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push minimal image | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: . | |
| file: data/minimal/latest/Dockerfile | |
| push: ${{ github.event_name != 'pull_request' && (github.event.inputs.push_images == 'true' || github.ref == 'refs/heads/main') }} | |
| tags: | | |
| ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}:minimal | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.html_url }} | |
| org.opencontainers.image.description=Minimal Databricks runtime container image | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| org.opencontainers.image.created=${{ github.event.head_commit.timestamp }} | |
| org.opencontainers.image.maintainer=${{ github.repository_owner }} | |
| provenance: false | |
| sbom: false | |
| cache-from: type=gha,scope=minimal | |
| cache-to: type=gha,mode=max,scope=minimal | |
| # Build python images using matrix (depends on standard or standard-gpu based on image type) | |
| build-python: | |
| name: Build ${{ matrix.image_type }}-${{ matrix.runtime }} | |
| needs: [generate-dockerfiles, build-standard, build-standard-gpu] | |
| if: github.ref == 'refs/heads/main' && always() && needs.generate-dockerfiles.result == 'success' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.generate-dockerfiles.outputs.matrix) }} | |
| steps: | |
| - name: Check dependencies | |
| id: check-deps | |
| run: | | |
| # Check if the required base image was built successfully | |
| if [ "${{ matrix.image_type }}" = "python" ]; then | |
| if [ "${{ needs.build-standard.result }}" != "success" ]; then | |
| echo "Standard image build failed, skipping python build" | |
| exit 1 | |
| fi | |
| elif [ "${{ matrix.image_type }}" = "python-gpu" ]; then | |
| if [ "${{ needs.build-standard-gpu.result }}" != "success" ]; then | |
| echo "Standard-gpu image build failed, skipping python-gpu build" | |
| exit 1 | |
| fi | |
| fi | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Download Dockerfiles | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: dockerfiles | |
| path: data/ | |
| - name: Determine runtime directory | |
| id: runtime-dir | |
| run: | | |
| # Find the directory matching this runtime version | |
| runtime_version="${{ matrix.runtime }}" | |
| # Convert "17.3 LTS" to directory pattern like "17.3-LTS-*" | |
| version_pattern=$(echo "$runtime_version" | sed 's/ /-/g') | |
| runtime_dir=$(ls -1 "data/${{ matrix.image_type }}" | grep -E "^${version_pattern}" | head -1) | |
| echo "Runtime directory: ${runtime_dir}" | |
| echo "dir=${runtime_dir}" >> $GITHUB_OUTPUT | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Log in to Container Registry | |
| if: github.event_name != 'pull_request' && (github.event.inputs.push_images == 'true' || github.ref == 'refs/heads/main') | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract version tag | |
| id: version | |
| run: | | |
| # Extract version number from runtime (e.g., "17.3 LTS" -> "17.3") | |
| version=$(echo "${{ matrix.runtime }}" | sed 's/ LTS//g') | |
| echo "version=${version}" >> $GITHUB_OUTPUT | |
| # Determine if this is the latest LTS (first in matrix) | |
| first_runtime=$(echo '${{ toJson(fromJson(needs.generate-dockerfiles.outputs.matrix).include) }}' | jq -r '.[0].runtime') | |
| first_type=$(echo '${{ toJson(fromJson(needs.generate-dockerfiles.outputs.matrix).include) }}' | jq -r '.[0].image_type') | |
| if [ "${{ matrix.runtime }}" = "$first_runtime" ] && [ "${{ matrix.image_type }}" = "$first_type" ]; then | |
| echo "is_latest=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "is_latest=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Prepare tags | |
| id: tags | |
| run: | | |
| # Always include the versioned tag | |
| TAGS="${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}:${{ matrix.image_type }}-${{ steps.version.outputs.version }}" | |
| echo "tags<<EOF" >> $GITHUB_OUTPUT | |
| echo "$TAGS" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| - name: Build and push python image | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: . | |
| file: data/${{ matrix.image_type }}/${{ steps.runtime-dir.outputs.dir }}/Dockerfile | |
| push: ${{ github.event_name != 'pull_request' && (github.event.inputs.push_images == 'true' || github.ref == 'refs/heads/main') }} | |
| tags: ${{ steps.tags.outputs.tags }} | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.html_url }} | |
| org.opencontainers.image.description=Databricks ${{ matrix.runtime }} runtime container image | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| org.opencontainers.image.created=${{ github.event.head_commit.timestamp }} | |
| org.opencontainers.image.maintainer=${{ github.repository_owner }} | |
| provenance: false | |
| sbom: false | |
| cache-from: type=gha,scope=${{ matrix.image_type }}-${{ steps.version.outputs.version }} | |
| cache-to: type=gha,mode=max,scope=${{ matrix.image_type }}-${{ steps.version.outputs.version }} | |
| # Build standard image (depends on minimal) | |
| build-standard: | |
| name: Build standard | |
| needs: [generate-dockerfiles, build-minimal] | |
| if: github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Download Dockerfiles | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: dockerfiles | |
| path: data/ | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Log in to Container Registry | |
| if: github.event_name != 'pull_request' && (github.event.inputs.push_images == 'true' || github.ref == 'refs/heads/main') | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push standard image | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: . | |
| file: data/standard/latest/Dockerfile | |
| push: ${{ github.event_name != 'pull_request' && (github.event.inputs.push_images == 'true' || github.ref == 'refs/heads/main') }} | |
| tags: | | |
| ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}:standard | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.html_url }} | |
| org.opencontainers.image.description=Standard Databricks runtime container image | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| org.opencontainers.image.created=${{ github.event.head_commit.timestamp }} | |
| org.opencontainers.image.maintainer=${{ github.repository_owner }} | |
| provenance: false | |
| sbom: false | |
| cache-from: type=gha,scope=standard | |
| cache-to: type=gha,mode=max,scope=standard | |
| # Build GPU base image | |
| build-gpu: | |
| name: Build GPU base | |
| needs: generate-dockerfiles | |
| if: github.ref == 'refs/heads/main' && github.event.inputs.image_type != 'cpu' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Download Dockerfiles | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: dockerfiles | |
| path: data/ | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Log in to Container Registry | |
| if: github.event_name != 'pull_request' && (github.event.inputs.push_images == 'true' || github.ref == 'refs/heads/main') | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push GPU base image | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: . | |
| file: data/gpu/latest/Dockerfile | |
| push: ${{ github.event_name != 'pull_request' && (github.event.inputs.push_images == 'true' || github.ref == 'refs/heads/main') }} | |
| tags: | | |
| ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}:gpu | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.html_url }} | |
| org.opencontainers.image.description=GPU base Databricks runtime container image | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| org.opencontainers.image.created=${{ github.event.head_commit.timestamp }} | |
| org.opencontainers.image.maintainer=${{ github.repository_owner }} | |
| provenance: false | |
| sbom: false | |
| cache-from: type=gha,scope=gpu | |
| cache-to: type=gha,mode=max,scope=gpu | |
| # Build minimal-gpu (depends on gpu base) | |
| build-minimal-gpu: | |
| name: Build minimal-gpu | |
| needs: [generate-dockerfiles, build-gpu] | |
| if: github.ref == 'refs/heads/main' && github.event.inputs.image_type != 'cpu' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Download Dockerfiles | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: dockerfiles | |
| path: data/ | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Log in to Container Registry | |
| if: github.event_name != 'pull_request' && (github.event.inputs.push_images == 'true' || github.ref == 'refs/heads/main') | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push minimal-gpu image | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: . | |
| file: data/minimal-gpu/latest/Dockerfile | |
| push: ${{ github.event_name != 'pull_request' && (github.event.inputs.push_images == 'true' || github.ref == 'refs/heads/main') }} | |
| tags: | | |
| ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}:minimal-gpu | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.html_url }} | |
| org.opencontainers.image.description=Minimal GPU Databricks runtime container image | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| org.opencontainers.image.created=${{ github.event.head_commit.timestamp }} | |
| org.opencontainers.image.maintainer=${{ github.repository_owner }} | |
| provenance: false | |
| sbom: false | |
| cache-from: type=gha,scope=minimal-gpu | |
| cache-to: type=gha,mode=max,scope=minimal-gpu | |
| # Build standard-gpu (depends on minimal-gpu) | |
| build-standard-gpu: | |
| name: Build standard-gpu | |
| needs: [generate-dockerfiles, build-minimal-gpu] | |
| if: github.ref == 'refs/heads/main' && github.event.inputs.image_type != 'cpu' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Download Dockerfiles | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: dockerfiles | |
| path: data/ | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Log in to Container Registry | |
| if: github.event_name != 'pull_request' && (github.event.inputs.push_images == 'true' || github.ref == 'refs/heads/main') | |
| uses: docker/login-action@v4 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push standard-gpu image | |
| uses: docker/build-push-action@v7 | |
| with: | |
| context: . | |
| file: data/standard-gpu/latest/Dockerfile | |
| push: ${{ github.event_name != 'pull_request' && (github.event.inputs.push_images == 'true' || github.ref == 'refs/heads/main') }} | |
| tags: | | |
| ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}:standard-gpu | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.html_url }} | |
| org.opencontainers.image.description=Standard GPU Databricks runtime container image | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| org.opencontainers.image.created=${{ github.event.head_commit.timestamp }} | |
| org.opencontainers.image.maintainer=${{ github.repository_owner }} | |
| provenance: false | |
| sbom: false | |
| cache-from: type=gha,scope=standard-gpu | |
| cache-to: type=gha,mode=max,scope=standard-gpu |