diff --git a/.github/workflows/docker-build-and-push-new.yaml b/.github/workflows/docker-build-and-push-new.yaml new file mode 100644 index 0000000000..8c7048e209 --- /dev/null +++ b/.github/workflows/docker-build-and-push-new.yaml @@ -0,0 +1,46 @@ +name: docker-build-and-push-new + +on: + push: + branches: + - main + tags: + - "*.*.*" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + changed-files: + runs-on: ubuntu-24.04 + outputs: + should-build: ${{ steps.check.outputs.any_changed == 'true' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref_type == 'tag') }} + steps: + - uses: actions/checkout@v6 + - id: check + uses: step-security/changed-files@v47 + with: + files: | + docker-new/** + ansible/** + ansible-galaxy-requirements.yaml + repositories/*.repos + .github/workflows/docker-build-new.yaml + .github/workflows/docker-build-pipeline-new.yaml + .github/workflows/docker-build-and-push-new.yaml + + humble: + needs: changed-files + if: needs.changed-files.outputs.should-build == 'true' + uses: ./.github/workflows/docker-build-pipeline-new.yaml + with: + ros-distro: humble + + jazzy: + needs: changed-files + if: needs.changed-files.outputs.should-build == 'true' + uses: ./.github/workflows/docker-build-pipeline-new.yaml + with: + ros-distro: jazzy diff --git a/.github/workflows/docker-build-new.yaml b/.github/workflows/docker-build-new.yaml new file mode 100644 index 0000000000..9d87e35ee5 --- /dev/null +++ b/.github/workflows/docker-build-new.yaml @@ -0,0 +1,220 @@ +name: docker-build-new-reusable + +on: + workflow_call: + inputs: + bake-group: + description: HCL group to build (e.g. ci-base, ci-core) + type: string + required: true + runner-amd64: + description: Runner for amd64 builds + type: string + default: ubuntu-24.04 + runner-arm64: + description: Runner for arm64 builds + type: string + default: ubuntu-24.04-arm + ros-distro: + description: ROS distribution + type: string + default: jazzy + cache-key-prefix: + description: Extra prefix for cache key separation + type: string + default: "" + target-image: + description: Registry image name + type: string + default: autoware + needs-source: + description: Whether to clone source repos via vcs import + type: boolean + default: false + free-disk-space: + description: Whether to free disk space before building + type: boolean + default: true +jobs: + build: + strategy: + fail-fast: false + matrix: + platform: [amd64, arm64] + include: + - platform: amd64 + arch-platform: linux/amd64 + - platform: arm64 + arch-platform: linux/arm64 + runs-on: ${{ matrix.platform == 'amd64' && inputs.runner-amd64 || inputs.runner-arm64 }} + outputs: + target-tags: ${{ steps.compute-tags.outputs.tags }} + steps: + - name: ๐Ÿ’ป Runner specs + run: | + echo "::group::CPU" + lscpu | grep -E "^(Architecture|CPU\(s\)|Model name|Thread|Core|Socket)" + echo "::endgroup::" + echo "::group::Memory" + free -h + echo "::endgroup::" + echo "::group::Disk" + df -h / + echo "::endgroup::" + + - name: ๐Ÿ”ง Change permission of workspace + run: sudo chown -R $USER:$USER ${{ github.workspace }} + + - name: ๐Ÿ“ฅ Check out repository + uses: actions/checkout@v6 + + - name: ๐Ÿท๏ธ Compute manifest tags from HCL + id: compute-tags + run: | + targets=$(docker buildx bake -f docker-new/docker-bake.hcl --print ${{ inputs.bake-group }} 2>/dev/null \ + | jq -r '.group."${{ inputs.bake-group }}".targets[]') + tags="" + for t in $targets; do + tags="${tags:+$tags }${t}-${{ inputs.ros-distro }}" + done + echo "tags=${tags}" >> "$GITHUB_OUTPUT" + echo "::notice::Manifest tags: ${tags}" + + - name: ๐Ÿงน Free disk space + if: inputs.free-disk-space && runner.environment == 'github-hosted' + uses: ./.github/actions/free-disk-space + + - name: ๐Ÿ“ฆ Import source repos + if: inputs.needs-source + run: | + pipx install vcs2l + mkdir -p src + vcs import --shallow src < repositories/autoware.repos + + - name: ๐Ÿณ Setup Docker Buildx + uses: docker/setup-buildx-action@v4 + with: + buildkitd-config-inline: | + [worker.oci] + max-parallelism = 2 + + - name: ๐Ÿ”‘ Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: ๐Ÿ“‹ Prepare build metadata + id: meta + run: | + ref_name="${GITHUB_REF_NAME//\//-}" + echo "ref_name=${ref_name}" >> "$GITHUB_OUTPUT" + echo "date=$(date +'%Y%m%d')" >> "$GITHUB_OUTPUT" + + if [[ "${{ github.ref_type }}" == "tag" ]]; then + echo "version=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT" + fi + + if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ github.ref_name }}" != "main" ]]; then + echo "ref=${ref_name}" >> "$GITHUB_OUTPUT" + fi + + echo "::group::Build info" + echo " Group: ${{ inputs.bake-group }}" + echo " Distro: ${{ inputs.ros-distro }}" + echo " Platform: ${{ matrix.platform }}" + echo " Ref: ${GITHUB_REF_NAME}" + echo " Event: ${{ github.event_name }}" + echo "::endgroup::" + + - name: ๐Ÿ”จ Build and push [${{ inputs.bake-group }}] (${{ matrix.platform }}) + uses: docker/bake-action@v7 + env: + ROS_DISTRO: ${{ inputs.ros-distro }} + REGISTRY: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} + PLATFORM: ${{ matrix.platform }} + TAG_DATE: ${{ steps.meta.outputs.date }} + TAG_VERSION: ${{ steps.meta.outputs.version }} + TAG_REF: ${{ steps.meta.outputs.ref }} + USE_REGISTRY_CONTEXTS: ${{ inputs.bake-group != 'ci-base' }} + with: + source: . + push: true + targets: ${{ inputs.bake-group }} + files: docker-new/docker-bake.hcl + provenance: false + set: | + *.platform=${{ matrix.arch-platform }} + *.args.ROS_DISTRO=${{ inputs.ros-distro }} + *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache-new:${{ inputs.cache-key-prefix }}${{ matrix.platform }}-${{ steps.meta.outputs.ref_name }} + *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache-new:${{ matrix.platform }}-${{ steps.meta.outputs.ref_name }} + *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache-new:${{ inputs.cache-key-prefix }}${{ matrix.platform }}-main + *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache-new:${{ matrix.platform }}-main + *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache-new:${{ inputs.cache-key-prefix }}${{ matrix.platform }}-${{ steps.meta.outputs.ref_name }},mode=max,ignore-error=true + + - name: ๐Ÿ“Š Build summary + if: always() + env: + IMAGE: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} + run: | + { + echo "### ๐Ÿณ Docker Build: \`${{ inputs.bake-group }}\` (${{ matrix.platform }})" + echo "" + echo "| Key | Value |" + echo "|-----|-------|" + echo "| **ROS Distro** | \`${{ inputs.ros-distro }}\` |" + echo "| **Platform** | \`${{ matrix.arch-platform }}\` |" + echo "| **Registry** | \`${IMAGE}\` |" + echo "| **Tags** | \`${{ steps.compute-tags.outputs.tags }}\` |" + echo "" + echo "#### ๐Ÿ’พ Disk space" + echo "\`\`\`" + df -h / | head -2 + echo "\`\`\`" + } >> "$GITHUB_STEP_SUMMARY" + + manifest: + needs: build + if: needs.build.result == 'success' + runs-on: ubuntu-24.04 + steps: + - name: ๐Ÿ”‘ Login to GitHub Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: ๐Ÿ”— Create multi-arch manifests + env: + IMAGE: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} + REF_SUFFIX: ${{ github.event_name == 'workflow_dispatch' && github.ref_name != 'main' && format('-{0}', github.ref_name) || '' }} + run: | + ref_suffix="${REF_SUFFIX//\//-}" + for tag in ${{ needs.build.outputs.target-tags }}; do + manifest_tag="${tag}${ref_suffix}" + echo "::group::๐Ÿ“ฆ ${manifest_tag}" + docker manifest create "${IMAGE}:${manifest_tag}" \ + "${IMAGE}:${manifest_tag}-amd64" \ + "${IMAGE}:${manifest_tag}-arm64" + docker manifest push "${IMAGE}:${manifest_tag}" + echo "::endgroup::" + done + + - name: ๐Ÿ“Š Manifest summary + env: + IMAGE: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }} + REF_SUFFIX: ${{ github.event_name == 'workflow_dispatch' && github.ref_name != 'main' && format('-{0}', github.ref_name) || '' }} + run: | + ref_suffix="${REF_SUFFIX//\//-}" + { + echo "### ๐Ÿ”— Multi-arch Manifests" + echo "" + echo "| Image | Platforms |" + echo "|-------|-----------|" + for tag in ${{ needs.build.outputs.target-tags }}; do + manifest_tag="${tag}${ref_suffix}" + echo "| \`${IMAGE}:${manifest_tag}\` | amd64, arm64 |" + done + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/docker-build-pipeline-new.yaml b/.github/workflows/docker-build-pipeline-new.yaml new file mode 100644 index 0000000000..5a8593832e --- /dev/null +++ b/.github/workflows/docker-build-pipeline-new.yaml @@ -0,0 +1,51 @@ +name: docker-build-pipeline-new + +on: + workflow_call: + inputs: + ros-distro: + description: ROS distribution + type: string + required: true + target-image: + description: Registry image name + type: string + default: autoware-new + +jobs: + build-base: + uses: ./.github/workflows/docker-build-new.yaml + with: + bake-group: ci-base + ros-distro: ${{ inputs.ros-distro }} + target-image: ${{ inputs.target-image }} + free-disk-space: false + + build-core: + needs: build-base + uses: ./.github/workflows/docker-build-new.yaml + with: + bake-group: ci-core + ros-distro: ${{ inputs.ros-distro }} + target-image: ${{ inputs.target-image }} + needs-source: true + free-disk-space: false + + build-universe: + needs: build-core + uses: ./.github/workflows/docker-build-new.yaml + with: + bake-group: ci-universe + ros-distro: ${{ inputs.ros-distro }} + target-image: ${{ inputs.target-image }} + needs-source: true + + build-universe-cuda: + needs: build-core + uses: ./.github/workflows/docker-build-new.yaml + with: + bake-group: ci-universe-cuda + ros-distro: ${{ inputs.ros-distro }} + target-image: ${{ inputs.target-image }} + needs-source: true + cache-key-prefix: cuda- diff --git a/docker-new/README.md b/docker-new/README.md index 8ca152ec31..f49f940a10 100644 --- a/docker-new/README.md +++ b/docker-new/README.md @@ -44,6 +44,40 @@ graph TD | `universe` | Runtime image with compiled autoware (no CUDA) | Deployment without GPU | | `universe-cuda` | Runtime image with compiled autoware + CUDA runtime libs | Deployment with GPU | +## Pull from GHCR + +Pre-built multi-arch images (amd64 + arm64) are available on GHCR: + +```bash +# Pull a specific image +docker pull ghcr.io/autowarefoundation/autoware-new:base-jazzy +docker pull ghcr.io/autowarefoundation/autoware-new:base-humble + +# Pull a dated version (for pinning) +docker pull ghcr.io/autowarefoundation/autoware-new:base-jazzy-20260407 + +# Pull a release version +docker pull ghcr.io/autowarefoundation/autoware-new:base-jazzy-1.2.3 +``` + +Tag pattern: `-[-|-]` + +Available images (replace `jazzy` with `humble` for other distros): + +| Tag | Description | +| ------------------------------------- | ------------------------------------------------- | +| `base-jazzy` | ROS base + ansible + user aw | +| `core-dependencies-jazzy` | Build deps + core packages (except autoware_core) | +| `core-devel-jazzy` | Full core development image | +| `core-jazzy` | Lightweight core runtime | +| `universe-dependencies-jazzy` | Universe build dependencies | +| `universe-dependencies-cuda-jazzy` | Universe + CUDA dev libs | +| `universe-devel-jazzy` | Full universe development (no CUDA) | +| `universe-devel-cuda-jazzy` | Full universe development with CUDA | +| `universe-runtime-dependencies-jazzy` | Universe runtime dependencies | +| `universe-jazzy` | Runtime without GPU | +| `universe-cuda-jazzy` | Runtime with GPU | + ## Build locally From the repository root. Targets beyond `base` require source repositories under `src/`: diff --git a/docker-new/docker-bake.hcl b/docker-new/docker-bake.hcl index 406ee4e5a2..49984c329d 100644 --- a/docker-new/docker-bake.hcl +++ b/docker-new/docker-bake.hcl @@ -2,9 +2,48 @@ variable "ROS_DISTRO" { default = "jazzy" } +// CI variables: set via environment in GitHub Actions, empty for local builds +variable "REGISTRY" { + default = "" +} +variable "PLATFORM" { + default = "" +} +variable "TAG_DATE" { + default = "" +} +variable "TAG_VERSION" { + default = "" +} +variable "TAG_REF" { + default = "" +} + +// When true, cross-group context references pull pre-built images from the +// registry instead of rebuilding via target:. Set in CI where each group +// builds in a separate job and upstream images are already pushed. +variable "USE_REGISTRY_CONTEXTS" { + default = false +} + +// IMPORTANT: The first element must always be the plain name-distro-platform tag +// because ctx() uses tags(name)[0] to construct docker-image:// references. function "tags" { params = [name] - result = ["autoware:${name}-${ROS_DISTRO}"] + result = compact(concat( + REGISTRY == "" ? ["autoware:${name}-${ROS_DISTRO}"] : [], + REGISTRY != "" && TAG_REF == "" ? ["${REGISTRY}:${name}-${ROS_DISTRO}-${PLATFORM}"] : [], + REGISTRY != "" && TAG_DATE != "" && TAG_REF == "" ? ["${REGISTRY}:${name}-${ROS_DISTRO}-${TAG_DATE}-${PLATFORM}"] : [], + REGISTRY != "" && TAG_VERSION != "" ? ["${REGISTRY}:${name}-${ROS_DISTRO}-${TAG_VERSION}-${PLATFORM}"] : [], + REGISTRY != "" && TAG_REF != "" ? ["${REGISTRY}:${name}-${ROS_DISTRO}-${TAG_REF}-${PLATFORM}"] : [], + )) +} + +// Returns "docker-image://..." when USE_REGISTRY_CONTEXTS is true (CI), +// or "target:..." when false (local builds). +function "ctx" { + params = [name] + result = USE_REGISTRY_CONTEXTS ? "docker-image://${tags(name)[0]}" : "target:${name}" } group "default" { @@ -46,7 +85,7 @@ target "core-dependencies" { target = "core-dependencies" tags = tags("core-dependencies") contexts = { - autoware-base = "target:base" + autoware-base = ctx("base") } args = { BASE_IMAGE = "autoware-base" @@ -58,7 +97,7 @@ target "core-devel" { target = "core-devel" tags = tags("core-devel") contexts = { - autoware-base = "target:base" + autoware-base = ctx("base") } args = { BASE_IMAGE = "autoware-base" @@ -70,7 +109,7 @@ target "core" { target = "core" tags = tags("core") contexts = { - autoware-base = "target:base" + autoware-base = ctx("base") } args = { BASE_IMAGE = "autoware-base" @@ -80,8 +119,8 @@ target "core" { target "_universe-base" { dockerfile = "docker-new/universe.Dockerfile" contexts = { - autoware-core-devel = "target:core-devel" - autoware-core = "target:core" + autoware-core-devel = ctx("core-devel") + autoware-core = ctx("core") } args = { CORE_DEVEL_IMAGE = "autoware-core-devel"