Build Expt Images Or Runtimes #32
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Expt Images Or Runtimes | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| tag: | |
| description: 'Tag for the Docker image' | |
| required: true | |
| default: 'latest' | |
| kind: | |
| description: 'Type of packages to build' | |
| required: false | |
| type: choice | |
| options: | |
| - operating-systems | |
| - languages | |
| - frameworks | |
| default: 'operating-systems' | |
| name: | |
| required: false | |
| description: 'Specific package folder; leave empty to build all packages of the selected kind' | |
| type: string | |
| default: '' | |
| build_type: | |
| description: 'Type of build to perform' | |
| required: false | |
| type: choice | |
| options: | |
| - images | |
| - runtimes | |
| default: 'images' | |
| tools_version: | |
| description: 'Base tools version (optional, defaults to tag if not specified)' | |
| required: false | |
| type: string | |
| default: '' | |
| os_version: | |
| description: 'OS image version (optional, defaults to tag if not specified)' | |
| required: false | |
| type: string | |
| default: '' | |
| framework_image_version: | |
| description: 'Framework image version (optional, defaults to tag if not specified)' | |
| required: false | |
| type: string | |
| default: '' | |
| l10n: | |
| description: 'Localization setting (choose one or both)' | |
| required: false | |
| type: choice | |
| options: | |
| - en_US | |
| - zh_CN | |
| - both | |
| default: 'en_US' | |
| arch: | |
| description: 'System architecture (choose one or both)' | |
| required: false | |
| type: choice | |
| options: | |
| - amd64 | |
| - arm64 | |
| - both | |
| default: 'amd64' | |
| aliyun_enabled: | |
| description: 'Enable Aliyun ACR builds' | |
| required: false | |
| type: boolean | |
| default: false | |
| workflow_call: | |
| inputs: | |
| tag: | |
| description: 'Tag for the Docker image' | |
| required: true | |
| type: string | |
| kind: | |
| description: 'Type of packages to build' | |
| required: false | |
| type: string | |
| default: 'operating-systems' | |
| name: | |
| description: 'Specific package folder; leave empty to build all packages of the selected kind' | |
| required: false | |
| type: string | |
| default: '' | |
| build_type: | |
| description: 'Type of build to perform' | |
| required: false | |
| type: string | |
| default: 'images' | |
| tools_version: | |
| description: 'Base tools version (optional, defaults to tag if not specified)' | |
| required: false | |
| type: string | |
| default: '' | |
| os_version: | |
| description: 'OS image version (optional, defaults to tag if not specified)' | |
| required: false | |
| type: string | |
| default: '' | |
| framework_image_version: | |
| description: 'Framework image version (optional, defaults to tag if not specified)' | |
| required: false | |
| type: string | |
| default: '' | |
| l10n: | |
| description: 'Localization setting (choose one or both)' | |
| required: false | |
| type: string | |
| default: 'en_US' | |
| arch: | |
| description: 'System architecture (choose one or both)' | |
| required: false | |
| type: string | |
| default: 'amd64' | |
| aliyun_enabled: | |
| description: 'Enable Aliyun ACR builds' | |
| required: false | |
| type: boolean | |
| default: false | |
| secrets: | |
| ALIYUN_REGISTRY: | |
| required: false | |
| ALIYUN_USERNAME: | |
| required: false | |
| ALIYUN_PASSWORD: | |
| required: false | |
| ALIYUN_NAMESPACE: | |
| required: false | |
| jobs: | |
| # Define expt image matrix | |
| define-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tag: ${{ steps.set_tag.outputs.tag }} | |
| framework_image_version: ${{ steps.set_tag.outputs.framework_image_version }} | |
| tools_version: ${{ steps.set_tag.outputs.tools_version }} | |
| os_version: ${{ steps.set_tag.outputs.os_version }} | |
| l10n_matrix: ${{ steps.set_matrix.outputs.l10n_matrix }} | |
| arch_matrix: ${{ steps.set_matrix.outputs.arch_matrix }} | |
| platforms: ${{ steps.set_matrix.outputs.platforms }} | |
| platforms_include_amd64: ${{ steps.set_matrix.outputs.platforms_include_amd64 }} | |
| packages: ${{ steps.get_packages.outputs.packages }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up tag | |
| id: set_tag | |
| run: | | |
| if [ -n "${{ inputs.tag }}" ]; then | |
| tag=${{ inputs.tag }} | |
| else | |
| tag=$(echo "${{ github.sha }}" | cut -c1-7) | |
| fi | |
| # Set tools_version: use inputs.tools_version if provided, otherwise use tag | |
| if [ -n "${{ inputs.tools_version }}" ]; then | |
| tools_version=${{ inputs.tools_version }} | |
| else | |
| tools_version=$tag | |
| fi | |
| # Set os_version: use inputs.os_version if provided, otherwise use tag | |
| if [ -n "${{ inputs.os_version }}" ]; then | |
| os_version=${{ inputs.os_version }} | |
| else | |
| os_version=$tag | |
| fi | |
| # Set framework_image_version: use inputs.framework_image_version if provided, otherwise use tag | |
| if [ -n "${{ inputs.framework_image_version }}" ]; then | |
| framework_image_version=${{ inputs.framework_image_version }} | |
| else | |
| framework_image_version=$tag | |
| fi | |
| echo "tag=$tag" >> $GITHUB_OUTPUT | |
| echo "framework_image_version=$framework_image_version" >> $GITHUB_OUTPUT | |
| echo "tools_version=$tools_version" >> $GITHUB_OUTPUT | |
| echo "os_version=$os_version" >> $GITHUB_OUTPUT | |
| - name: Build localization and architecture matrix | |
| id: set_matrix | |
| run: | | |
| l10n_values=() | |
| case "${{ inputs.l10n }}" in | |
| en_US) | |
| l10n_values+=('{"display":"en_US","normalized":"en-us"}') | |
| ;; | |
| zh_CN) | |
| l10n_values+=('{"display":"zh_CN","normalized":"zh-cn"}') | |
| ;; | |
| both) | |
| l10n_values+=('{"display":"en_US","normalized":"en-us"}') | |
| l10n_values+=('{"display":"zh_CN","normalized":"zh-cn"}') | |
| ;; | |
| *) | |
| echo "Error: Unknown l10n option '${{ inputs.l10n }}'." >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| l10n_matrix=$(printf '%s\n' "${l10n_values[@]}" | jq -s -c '.') | |
| # Single platforms string for buildx (one job builds all requested arches, manifest list) | |
| case "${{ inputs.arch }}" in | |
| amd64) | |
| platforms="linux/amd64" | |
| platforms_include_amd64="true" | |
| ;; | |
| arm64) | |
| platforms="linux/arm64" | |
| platforms_include_amd64="false" | |
| ;; | |
| both) | |
| platforms="linux/amd64,linux/arm64" | |
| platforms_include_amd64="true" | |
| ;; | |
| *) | |
| echo "Error: Unknown arch option '${{ inputs.arch }}'." >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| echo "platforms=$platforms" >> $GITHUB_OUTPUT | |
| echo "platforms_include_amd64=$platforms_include_amd64" >> $GITHUB_OUTPUT | |
| # Keep arch_matrix for any consumers (e.g. step summary); single arch or both | |
| arch_values=() | |
| case "${{ inputs.arch }}" in | |
| amd64) arch_values+=("amd64") ;; | |
| arm64) arch_values+=("arm64") ;; | |
| both) arch_values+=("amd64") ; arch_values+=("arm64") ;; | |
| esac | |
| arch_matrix=$(printf '%s\n' "${arch_values[@]}" | jq -R -s -c 'split("\n")[:-1]') | |
| echo "arch_matrix=$arch_matrix" >> $GITHUB_OUTPUT | |
| echo "l10n_matrix=$l10n_matrix" >> $GITHUB_OUTPUT | |
| - name: Get packages | |
| id: get_packages | |
| run: | | |
| # Get all packages | |
| kind=${{ inputs.kind }} | |
| name=${{ inputs.name }} | |
| build_type=${{ inputs.build_type }} | |
| if [ -n "$name" ]; then | |
| echo "Looking for specific package: $name" | |
| target_dockerfiles=$(find experimental/$build_type/$kind/$name -name "Dockerfile") | |
| if [ -z "$target_dockerfiles" ]; then | |
| echo "Error: No Dockerfile found for package '$name' in kind '$kind'" >&2 | |
| exit 1 | |
| fi | |
| else | |
| echo "No specific package provided, gathering all packages of kind: $kind" | |
| target_dockerfiles=$(find experimental/$build_type/$kind -name "Dockerfile") | |
| fi | |
| # Convert to JSON array | |
| packages=$(echo "$target_dockerfiles" | jq -R -s 'split("\n")[:-1]') | |
| echo "packages<<EOF" >> $GITHUB_OUTPUT | |
| echo "$packages" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| package_count=$(echo "$packages" | jq 'length // 0') | |
| echo "Found $package_count packages to build" | |
| # Build expt images | |
| build-expt-images: | |
| runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} | |
| needs: define-matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| packages: ${{ fromJson(needs.define-matrix.outputs.packages) }} | |
| l10n: ${{ fromJson(needs.define-matrix.outputs.l10n_matrix) }} | |
| arch: ${{ fromJson(needs.define-matrix.outputs.arch_matrix) }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Generate image names (standard) | |
| id: generate-standard | |
| uses: ./.github/actions/generate-image-names | |
| with: | |
| dockerfile: ${{ matrix.packages }} | |
| tag: ${{ format('{0}-{1}-{2}', needs.define-matrix.outputs.tag, matrix.l10n.normalized, matrix.arch) }} | |
| ghcr_credentials: ${{ format('{{"registry":"{0}","username":"{1}","password":"{2}"}}', 'ghcr.io', github.repository_owner, secrets.GITHUB_TOKEN) || '{}' }} | |
| aliyun_credentials: ${{ inputs.aliyun_enabled == 'true' && format('{{"registry":"{0}","username":"{1}","password":"{2}", "namespace":"{3}"}}', secrets.ALIYUN_REGISTRY, secrets.ALIYUN_USERNAME, secrets.ALIYUN_PASSWORD, secrets.ALIYUN_NAMESPACE) || '{}' }} | |
| expt_image_naming: ${{ inputs.build_type }} | |
| - name: Build and push standard images | |
| uses: ./.github/actions/build-and-push | |
| with: | |
| platforms: ${{ format('linux/{0}', matrix.arch) }} | |
| # Note: NODE_IMAGE_VERSION and RUNTIME_IMAGE_VERSION are the same for now, | |
| # but we keep them separate for future flexibility; | |
| # OR: we need change the logic to avoid duplicate build arguments | |
| build_args: | | |
| REPO=${{ github.repository_owner }}/devbox-base-expt | |
| L10N=${{ matrix.l10n.display }} | |
| L10N_NORMALIZED=${{ matrix.l10n.normalized }} | |
| BASE_TOOLS_VERSION=${{ needs.define-matrix.outputs.tools_version }} | |
| OS_IMAGE_VERSION=${{ needs.define-matrix.outputs.os_version }}-${{ matrix.l10n.normalized }} | |
| FRAMEWORK_IMAGE_VERSION=${{ needs.define-matrix.outputs.framework_image_version }}-${{ matrix.l10n.normalized }} | |
| NODE_IMAGE_VERSION=${{ format('{0}-{1}', needs.define-matrix.outputs.tag, matrix.l10n.normalized) }} | |
| RUNTIME_IMAGE_VERSION=${{ format('{0}-{1}', needs.define-matrix.outputs.tag, matrix.l10n.normalized) }} | |
| dockerfile: ${{ matrix.packages }} | |
| ghcr_credentials: ${{ format('{{"registry":"{0}","username":"{1}","password":"{2}"}}', 'ghcr.io', github.repository_owner, secrets.GITHUB_TOKEN) || '{}' }} | |
| ghcr_image_name: ${{ steps.generate-standard.outputs.ghcr_image_name }} | |
| aliyun_credentials: ${{ inputs.aliyun_enabled == 'true' && format('{{"registry":"{0}","username":"{1}","password":"{2}", "namespace":"{3}"}}', secrets.ALIYUN_REGISTRY, secrets.ALIYUN_USERNAME, secrets.ALIYUN_PASSWORD, secrets.ALIYUN_NAMESPACE) || '{}' }} | |
| acr_image_name: ${{ steps.generate-standard.outputs.acr_image_name }} | |
| - name: Runtime smoke test (en_US, amd64) | |
| if: ${{ inputs.build_type == 'runtimes' && matrix.arch == 'amd64' && matrix.l10n.normalized == 'en-us' }} | |
| continue-on-error: true | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| image="${{ steps.generate-standard.outputs.ghcr_image_name }}" | |
| dockerfile="${{ matrix.packages }}" | |
| rel_path="${dockerfile#experimental/runtimes/}" | |
| rel_path="${rel_path%/Dockerfile}" | |
| test_dir="$GITHUB_WORKSPACE/experimental/tests/runtime-smoke/$rel_path" | |
| test_script="$test_dir/smoke.sh" | |
| if [ ! -f "$test_script" ]; then | |
| echo "Smoke script not found: $test_script" >&2 | |
| exit 1 | |
| fi | |
| echo "Smoke test: $dockerfile -> $image" | |
| docker pull "$image" | |
| docker run --rm \ | |
| -e L10N=en_US \ | |
| -e SMOKE_DEBUG=1 \ | |
| --entrypoint /bin/bash \ | |
| -v "$test_dir:/tests:ro" \ | |
| "$image" -lc "cp /tests/smoke.sh /tmp/smoke.sh && chmod +x /tmp/smoke.sh && su - devbox -c '/tmp/smoke.sh'" | |
| - name: Clean up images (standard) | |
| if: always() | |
| run: | | |
| set -eux | |
| # Remove GHCR image by tag if present | |
| if [ -n "${{ steps.generate-standard.outputs.ghcr_image_name }}" ]; then | |
| echo "Removing local image: ${{ steps.generate-standard.outputs.ghcr_image_name }} (if exists)" | |
| docker image rm -f "${{ steps.generate-standard.outputs.ghcr_image_name }}" || true | |
| fi | |
| # Remove Aliyun/ACR image by tag if present | |
| if [ -n "${{ steps.generate-standard.outputs.acr_image_name }}" ]; then | |
| echo "Removing local image: ${{ steps.generate-standard.outputs.acr_image_name }} (if exists)" | |
| docker image rm -f "${{ steps.generate-standard.outputs.acr_image_name }}" || true | |
| fi | |
| # Remove dangling images and builder cache to free space | |
| docker image prune -af || true | |
| # If docker buildx was used, prune builder cache as well | |
| docker builder prune -af || true | |
| - name: Record built image names (standard) | |
| id: record-standard | |
| if: always() | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| out="$RUNNER_TEMP/image-list-standard.txt" | |
| : > "$out" | |
| if [ -n "${{ steps.generate-standard.outputs.ghcr_image_name }}" ]; then | |
| echo "${{ steps.generate-standard.outputs.ghcr_image_name }}" >> "$out" | |
| fi | |
| if [ -n "${{ steps.generate-standard.outputs.acr_image_name }}" ]; then | |
| echo "${{ steps.generate-standard.outputs.acr_image_name }}" >> "$out" | |
| fi | |
| if [ ! -s "$out" ]; then | |
| rm -f "$out" | |
| fi | |
| slug=$(echo -n "${{ matrix.packages }}" | tr '/:' '-' | tr -cd '[:alnum:]._-') | |
| echo "artifact_name=image-list-standard-${{ inputs.kind }}-${{ inputs.build_type }}-${{ matrix.l10n.normalized }}-${{ matrix.arch }}-${slug}" >> "$GITHUB_OUTPUT" | |
| - name: Upload image list (standard) | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.record-standard.outputs.artifact_name }} | |
| path: ${{ runner.temp }}/image-list-standard.txt | |
| if-no-files-found: ignore | |
| - name: Output built image names (standard) | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| run: | | |
| if [ -n "${{ steps.generate-standard.outputs.ghcr_image_name }}" ]; then | |
| echo "- ${{ steps.generate-standard.outputs.ghcr_image_name }}" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| if [ -n "${{ steps.generate-standard.outputs.acr_image_name }}" ]; then | |
| echo "- ${{ steps.generate-standard.outputs.acr_image_name }}" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| create-manifests: | |
| runs-on: ubuntu-latest | |
| needs: [define-matrix, build-expt-images] | |
| permissions: | |
| contents: read | |
| packages: write | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| packages: ${{ fromJson(needs.define-matrix.outputs.packages) }} | |
| l10n: ${{ fromJson(needs.define-matrix.outputs.l10n_matrix) }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Login to GHCR | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Login to Aliyun ACR | |
| if: ${{ inputs.aliyun_enabled == 'true' }} | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ secrets.ALIYUN_REGISTRY }} | |
| username: ${{ secrets.ALIYUN_USERNAME }} | |
| password: ${{ secrets.ALIYUN_PASSWORD }} | |
| - name: Generate base image names (manifest) | |
| id: generate-base | |
| uses: ./.github/actions/generate-image-names | |
| with: | |
| dockerfile: ${{ matrix.packages }} | |
| tag: ${{ format('{0}-{1}', needs.define-matrix.outputs.tag, matrix.l10n.normalized) }} | |
| ghcr_credentials: ${{ format('{{"registry":"{0}","username":"{1}","password":"{2}"}}', 'ghcr.io', github.repository_owner, secrets.GITHUB_TOKEN) || '{}' }} | |
| aliyun_credentials: ${{ inputs.aliyun_enabled == 'true' && format('{{"registry":"{0}","username":"{1}","password":"{2}", "namespace":"{3}"}}', secrets.ALIYUN_REGISTRY, secrets.ALIYUN_USERNAME, secrets.ALIYUN_PASSWORD, secrets.ALIYUN_NAMESPACE) || '{}' }} | |
| expt_image_naming: ${{ inputs.build_type }} | |
| - name: Create multi-arch manifests | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| arches=$(echo '${{ needs.define-matrix.outputs.arch_matrix }}' | jq -r '.[]') | |
| ghcr_base="${{ steps.generate-base.outputs.ghcr_image_name }}" | |
| ghcr_imgs=() | |
| for a in $arches; do | |
| ghcr_imgs+=("${ghcr_base}-${a}") | |
| done | |
| echo "Creating GHCR manifest: $ghcr_base <- ${ghcr_imgs[*]}" | |
| docker buildx imagetools create -t "$ghcr_base" "${ghcr_imgs[@]}" | |
| if [ "${{ steps.generate-base.outputs.acr_image_name }}" != "" ]; then | |
| acr_base="${{ steps.generate-base.outputs.acr_image_name }}" | |
| acr_imgs=() | |
| for a in $arches; do | |
| acr_imgs+=("${acr_base}-${a}") | |
| done | |
| echo "Creating ACR manifest: $acr_base <- ${acr_imgs[*]}" | |
| docker buildx imagetools create -t "$acr_base" "${acr_imgs[@]}" | |
| fi | |
| - name: Record manifest image names | |
| id: record-manifest | |
| if: always() | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| out="$RUNNER_TEMP/image-list-manifest.txt" | |
| : > "$out" | |
| if [ -n "${{ steps.generate-base.outputs.ghcr_image_name }}" ]; then | |
| echo "${{ steps.generate-base.outputs.ghcr_image_name }}" >> "$out" | |
| fi | |
| if [ -n "${{ steps.generate-base.outputs.acr_image_name }}" ]; then | |
| echo "${{ steps.generate-base.outputs.acr_image_name }}" >> "$out" | |
| fi | |
| if [ ! -s "$out" ]; then | |
| rm -f "$out" | |
| fi | |
| slug=$(echo -n "${{ matrix.packages }}" | tr '/:' '-' | tr -cd '[:alnum:]._-') | |
| echo "artifact_name=image-list-manifest-${{ inputs.kind }}-${{ inputs.build_type }}-${{ matrix.l10n.normalized }}-${slug}" >> "$GITHUB_OUTPUT" | |
| - name: Upload image list (manifest) | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.record-manifest.outputs.artifact_name }} | |
| path: ${{ runner.temp }}/image-list-manifest.txt | |
| if-no-files-found: ignore | |
| - name: Output manifest image names | |
| if: ${{ github.event_name == 'workflow_dispatch' }} | |
| run: | | |
| if [ -n "${{ steps.generate-base.outputs.ghcr_image_name }}" ]; then | |
| echo "- ${{ steps.generate-base.outputs.ghcr_image_name }}" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| if [ -n "${{ steps.generate-base.outputs.acr_image_name }}" ]; then | |
| echo "- ${{ steps.generate-base.outputs.acr_image_name }}" >> $GITHUB_STEP_SUMMARY | |
| fi |