Update docker images for use with docker buildx

.github/workflows/test.yml

@@ -0,0 +1,122 @@
+name: CI
+
+on: [push, pull_request]
+
+env:
+  PREVIOUS_CRYSTAL_VERSION: 1.6.0
+  CRYSTAL_VERSION: 1.6.2
+  PACKAGE_VERSION: 1
+
+jobs:
+  linux-build:
+    # TODO: set architecture based off matrix value when ARM runner available
+    name: Build linux binaries
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        architecture:
+          - x86_64
+          - aarch64
+    steps:
+      -
+        name: Checkout the distribution scripts
+        uses: actions/checkout@v3
+
+      -
+        name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      -
+        name: Build the linux binaries
+        run: |
+          cd linux
+          make \
+            arch=${{ matrix.architecture }} \
+            CRYSTAL_VERSION=${{ env.CRYSTAL_VERSION }} \
+            PACKAGE_VERSION=${{ env.PACKAGE_VERSION }} \
+            PREVIOUS_CRYSTAL_VERSION=${{ env.PREVIOUS_CRYSTAL_VERSION }}
+
+      -
+        name: Upload the binaries for use in the next step
+        uses: actions/upload-artifact@v3
+        with:
+          name: linux-${{ env.CRYSTAL_VERSION }}-{{ matrix.architecture }}-${{ env.PACKAGE_VERSION }}
+          path: linux/build/*.gz
+
+
+  docker-build:
+    name: Build and test the docker images
+    runs-on: ubuntu-latest
+    needs: linux-build
+    strategy:
+      fail-fast: false
+      matrix:
+        architecture:
+          - x86_64
+          - aarch64
+    steps:
+      -
+        name: Checkout the distribution scripts
+        uses: actions/checkout@v3
+
+      -
+        name: Download the linux binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: linux-${{ env.CRYSTAL_VERSION }}-{{ matrix.architecture }}-${{ env.PACKAGE_VERSION }}
+          path: docker/build-context
+
+      -
+        name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      -
+        name: Build the docker image
+        run: |
+          cd docker
+          make \
+            arch=${{ matrix.architecture }} \
+            CRYSTAL_VERSION=${{ env.CRYSTAL_VERSION }} \
+            PACKAGE_VERSION=${{ env.PACKAGE_VERSION }} \
+
+      -
+        name: Ensure images are well formed
+        run: |
+          # NOTE: Hack for GHA
+          sudo mkdir -p /var/lib/docker/tmp
+
+          cd docker
+          make test \
+            arch=${{ matrix.architecture }} \
+            CRYSTAL_VERSION=${{ env.CRYSTAL_VERSION }} \
+            PACKAGE_VERSION=${{ env.PACKAGE_VERSION }}
+
+      -
+        name: Upload the docker images
+        uses: actions/upload-artifact@v3
+        with:
+          name: docker-crystal-${{ env.CRYSTAL_VERSION }}-${{ env.PACKAGE_VERSION }}-{{ matrix.architecture }}
+          path: docker/build/*.gz

.gitignore

@@ -4,6 +4,8 @@ omnibus/bin
 omnibus/crystal-darwin-*
 omnibus/shards-darwin-*
 
+build
+
 docs/build/
 
 darwin/build/

darwin/Makefile

@@ -8,7 +8,12 @@ CRYSTAL_SHA1 ?= $(CRYSTAL_VERSION) ## Git tag/branch/sha1 to checkout and build
 PACKAGE_ITERATION ?= 1
 FORCE_GIT_TAGGED ?= 1              ## Require build to be based on git tag/branch
 
-PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ ?= ## url to crystal-{version}-{package}-darwin-x86_64.tar.gz
+PREVIOUS_CRYSTAL_VERSION ?=    ## Version of the bootstrap compiler
+PREVIOUS_PACKAGE_ITERATION ?= 1## Package iteration of the bootstrap compiler
+
+CRYSTAL_REPOSITORY ?= crystal-lang/crystal
+
+PREVIOUS_CRYSTAL_RELEASE_DARWIN_TARGZ ?= https://github.com/$(CRYSTAL_REPOSITORY)/releases/download/$(PREVIOUS_CRYSTAL_VERSION)/crystal-$(PREVIOUS_CRYSTAL_VERSION)-$(PREVIOUS_PACKAGE_ITERATION)-darwin-universal.tar.gz## Url to crystal-{version}-{package}-darwin-universal.tar.gz
 
 OUTPUT_DIR = build
 
@@ -49,10 +54,11 @@ $(CURDIR)/../omnibus/crystal-darwin-x86_64/embedded/bin/crystal:
 	&& rm /tmp/crystal-darwin-x86_64.tar.gz \
 	&& chmod +x $(CURDIR)/../omnibus/crystal-darwin-x86_64/embedded/bin/crystal
 
+# TODO: Shallow clone
 $(OUTPUT_DIR)/$(DARWIN_NAME) $(OUTPUT_DIR)/$(DARWIN_PKG_NAME): ## Build omnibus crystal project
 ifeq ($(FORCE_GIT_TAGGED), 0)
 	rm -Rf $(CURDIR)/tmp && mkdir -p $(CURDIR)/tmp && cd $(CURDIR)/tmp \
-	&& git clone https://github.com/crystal-lang/crystal \
+	&& git clone https://github.com/$(CRYSTAL_REPOSITORY) \
 	&& cd crystal \
 	&& git checkout $(CRYSTAL_SHA1) \
 	&& git checkout -b $(CRYSTAL_VERSION)

docker/Makefile

@@ -1,78 +1,127 @@
 # Recipies for this Makefile
 
-## Build all docker images
-##   $ make DOCKER_TAG=nightly CRYSTAL_VERSION=0.xy.z CRYSTAL_DEB=... CRYSTAL_TARGZ=...
-## Build ubuntu64 docker images
-##   $ make ubuntu64 DOCKER_TAG=nightly CRYSTAL_VERSION=0.xy.z CRYSTAL_TARGZ=...
-## Build alpine docker images
-##   $ make alpine DOCKER_TAG=nightly CRYSTAL_VERSION=0.xy.z CRYSTAL_TARGZ=...
+## Build all docker images for x86_64
+##   $ make arch=x86_64 DOCKER_TAG=nightly CRYSTAL_VERSION=0.xy.z CRYSTAL_TARGZ=...
+## Build all docker images for aarch64
+##   $ make arch=aarch64 DOCKER_TAG=nightly CRYSTAL_VERSION=0.xy.z CRYSTAL_TARGZ=...
+## Build ubuntu docker images for x86_64
+##   $ make ubuntu arch=x86_64 DOCKER_TAG=nightly CRYSTAL_VERSION=0.xy.z CRYSTAL_TARGZ=...
+## Build alpine docker images for aarch64
+##   $ make alpine arch=aarch64 DOCKER_TAG=nightly CRYSTAL_VERSION=0.xy.z CRYSTAL_TARGZ=...
+
+arch ?=        ## Architecture to build for (x86_64, aarch64)
+no_cache ?=    ## Disable the docker build cache
+pull_images ?= ## Always pull docker images to ensure they're up to date
 
 CRYSTAL_VERSION ?=   ## How the binaries should be branded
-CRYSTAL_DEB ?=       ## Which crystal.deb file to install in debian based docker images (ubuntu32)
-CRYSTAL_TARGZ ?=     ## Which crystal.tar.gz file to install in docker images (ubuntu64, alpine)
-DOCKER_TAG ?= $(CRYSTAL_VERSION) ## How to tag the docker image (examples: `0.27.2`, `nightly20190307`). `-build` will be appended for build images.
-DOCKER_REPOSITORY ?= crystallang/crystal ## Docker hub repository to commit image
+PACKAGE_ITERATION ?= 1
+PACKAGE_MAINTAINER = Crystal Team <[email protected]>
 
-GC_VERSION = v8.2.0
+DOCKER_TAG ?= $(CRYSTAL_VERSION)## How to tag the docker image (examples: `0.27.2`, `nightly20190307`). `-build` will be appended for build images.
+DOCKER_REPOSITORY ?= crystallang/crystal## Docker hub repository to commit image
+
+GC_VERSION = v8.2.2
+
+ifeq ($(arch),x86_64)
+  DOCKER_BUILD_PLATFORM = linux/amd64
+else ifeq ($(arch),aarch64)
+  DOCKER_BUILD_PLATFORM = linux/arm64
+else
+  $(error Expected arch to be one of 'x86_64', or 'aarch64', got: $(arch))
+endif
+
+BASE_UBUNTU_DOCKER_IMAGE = ubuntu:focal
 
 OUTPUT_DIR := build
+
 BUILD_CONTEXT := $(CURDIR)/build-context
-BUILD_ARGS_UBUNTU64 := --build-arg crystal_targz=crystal.tar.gz $(BUILD_CONTEXT)/ubuntu64 --build-arg base_docker_image=ubuntu:focal
-BUILD_ARGS_ALPINE := --build-arg crystal_targz=crystal.tar.gz $(BUILD_CONTEXT)/alpine --build-arg gc_version=$(GC_VERSION)
+
+CRYSTAL_TARGZ ?= $(BUILD_CONTEXT)/crystal-$(CRYSTAL_VERSION)-$(PACKAGE_ITERATION)-linux-$(arch).tar.gz## Which crystal.tar.gz file to install in docker images (ubuntu, alpine)
+
+BUILD_ARGS_COMMON := \
+                      $(if $(pull_images),--pull) \
+                      $(if $(no_cache),--no-cache) \
+                      $(if $(GITHUB_ACTIONS),--cache-from type=gha,mode=max,scope=$(arch) --cache-to type=gha,mode=max,scope=$(arch)) \
+                      --platform $(DOCKER_BUILD_PLATFORM) \
+                      --label "crystal_version=$(CRYSTAL_VERSION)" \
+                      --label "package_iteration=$(PACKAGE_ITERATION)" \
+                      --label "package_maintainer=$(PACKAGE_MAINTAINER)" \
+                      --build-arg llvm_version=$(LLVM_VERSION)
+
+BUILD_ARGS_UBUNTU := $(BUILD_ARGS_COMMON) \
+                      --build-arg crystal_targz=crystal.tar.gz \
+                      --build-arg base_docker_image=$(BASE_UBUNTU_DOCKER_IMAGE) \
+                      $(BUILD_CONTEXT)/ubuntu
+
+BUILD_ARGS_ALPINE := $(BUILD_ARGS_COMMON) \
+                     --build-arg crystal_targz=crystal.tar.gz \
+                     --build-arg gc_version=$(GC_VERSION) \
+                     $(BUILD_CONTEXT)/alpine
+
 DOCKER_TAG_UBUNTU := $(DOCKER_REPOSITORY):$(DOCKER_TAG)
 DOCKER_TAG_ALPINE := $(DOCKER_REPOSITORY):$(DOCKER_TAG)-alpine
 
-.PHONY: all64
-all64: ubuntu64 alpine ## Build all x86_64 images
+.PHONY: all
+all: ubuntu alpine ## Build all images
 
 .PHONY: all_ubuntu
-all_ubuntu: ubuntu64  ## Build all ubuntu images
+all_ubuntu: ubuntu  ## Build all ubuntu images
 
-.PHONY: ubuntu64
-ubuntu64: ## Build ubuntu x86_64 images
-ubuntu64: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-ubuntu-x86_64.tar.gz
-ubuntu64: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-ubuntu-x86_64-build.tar.gz
+.PHONY: ubuntu
+ubuntu: ## Build ubuntu images
+ubuntu: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-ubuntu-$(arch).tar.gz
+ubuntu: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-ubuntu-$(arch)-build.tar.gz
 
 .PHONY: alpine
 alpine: ## Build alpine images
-alpine: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine.tar.gz
-alpine: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine-build.tar.gz
+alpine: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine-$(arch).tar.gz
+alpine: $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine-$(arch)-build.tar.gz
 
-$(BUILD_CONTEXT)/ubuntu64: ubuntu.Dockerfile $(BUILD_CONTEXT)/ubuntu64/crystal.tar.gz
+$(BUILD_CONTEXT)/ubuntu: ubuntu.Dockerfile $(BUILD_CONTEXT)/ubuntu/crystal.tar.gz
 	cp ubuntu.Dockerfile $@/Dockerfile
 
 $(BUILD_CONTEXT)/alpine: alpine.Dockerfile $(BUILD_CONTEXT)/alpine/crystal.tar.gz
 	cp alpine.Dockerfile $@/Dockerfile
 	mkdir $@/files/
 
-%/crystal.deb:
-	mkdir -p $(shell dirname $@)
-	cp $(CRYSTAL_DEB) $@
-
 %/crystal.tar.gz:
 	mkdir -p $(shell dirname $@)
 	cp $(CRYSTAL_TARGZ) $@
 
 $(OUTPUT_DIR):
-	mkdir -p $(OUTPUT_DIR)
+	mkdir -p $(@)
+
+$(BUILD_CONTEXT):
+	mkdir -p $(@)
+
+.PHONY: scripts
+scripts: $(BUILD_CONTEXT)
+	cp -r ../scripts $(BUILD_CONTEXT)/alpine
 
-# Ubuntu x86_64
-$(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-ubuntu-x86_64.tar.gz: $(BUILD_CONTEXT)/ubuntu64 $(OUTPUT_DIR)
-	docker build -t $(DOCKER_TAG_UBUNTU) --target runtime $(BUILD_ARGS_UBUNTU64)
-	docker save $(DOCKER_TAG_UBUNTU) | gzip > $@
+.PHONY: test
+test: test_ubuntu test_alpine
 
-$(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-ubuntu-x86_64-build.tar.gz: $(BUILD_CONTEXT)/ubuntu64 $(OUTPUT_DIR)
-	docker build -t $(DOCKER_TAG_UBUNTU)-build --target build  $(BUILD_ARGS_UBUNTU64)
-	docker save $(DOCKER_TAG_UBUNTU)-build | gzip > $@
+.PHONY: test_ubuntu
+test_ubuntu: ubuntu
+	$(call test_crystal_image, $(DOCKER_TAG_UBUNTU), $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-ubuntu-$(arch).tar.gz)
 
-# Alpine x86_64
-$(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine.tar.gz: $(BUILD_CONTEXT)/alpine $(OUTPUT_DIR)
-	docker build -t $(DOCKER_TAG_ALPINE) --target runtime $(BUILD_ARGS_ALPINE)
-	docker save $(DOCKER_TAG_ALPINE) | gzip > $@
+.PHONY: test_alpine
+test_alpine: alpine
+	$(call test_crystal_image, $(DOCKER_TAG_ALPINE), $(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine-$(arch).tar.gz)
 
-$(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine-build.tar.gz: $(BUILD_CONTEXT)/alpine $(OUTPUT_DIR)
-	docker build -t $(DOCKER_TAG_ALPINE)-build --target build $(BUILD_ARGS_ALPINE)
-	docker save $(DOCKER_TAG_ALPINE)-build | gzip > $@
+# Ubuntu
+$(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-ubuntu-$(arch).tar.gz: $(BUILD_CONTEXT)/ubuntu $(OUTPUT_DIR)
+	docker buildx build --tag $(DOCKER_TAG_UBUNTU) --target runtime -o type=docker,dest=- $(BUILD_ARGS_UBUNTU) | gzip > $@
+
+$(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-ubuntu-$(arch)-build.tar.gz: $(BUILD_CONTEXT)/ubuntu $(OUTPUT_DIR)
+	docker buildx build --tag $(DOCKER_TAG_UBUNTU)-build --target build -o type=docker,dest=- $(BUILD_ARGS_UBUNTU) | gzip > $@
+
+# Alpine
+$(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine-$(arch).tar.gz: $(BUILD_CONTEXT)/alpine $(OUTPUT_DIR) scripts
+	docker buildx build --tag $(DOCKER_TAG_ALPINE) --target runtime -o type=docker,dest=- $(BUILD_ARGS_ALPINE) | gzip > $@
+
+$(OUTPUT_DIR)/docker-$(CRYSTAL_VERSION)-alpine-$(arch)-build.tar.gz: $(BUILD_CONTEXT)/alpine $(OUTPUT_DIR) scripts
+	docker buildx build --tag $(DOCKER_TAG_ALPINE)-build --target build -o type=docker,dest=- $(BUILD_ARGS_ALPINE) | gzip > $@
 
 .PHONY: clean
 clean: ## Clean up build and output directories
@@ -95,3 +144,19 @@ help: ## Show this help
 	@printf '\033[34mrecipes:\033[0m\n'
 	@grep -hE '^##.*$$' $(MAKEFILE_LIST) |\
 awk 'BEGIN {FS = "## "}; /^## [a-zA-Z_-]/ {printf "  \033[36m%s\033[0m\n", $$2}; /^##  / {printf "  %s\n", $$2}'
+
+# Load the docker image and run `crystal` and `shards`
+# Arguments:
+#  $1: The docker image tag to load
+#  $2: The docker tar ball to load
+define test_crystal_image
+  docker load -i $(2) \
+  && \
+  docker run --rm \
+    $(1) \
+    crystal --version \
+  && \
+  docker run --rm \
+    $(1) \
+    shards --version
+endef

docker/README.md

@@ -0,0 +1,15 @@
+# Docker builds
+
+Crystal binaries are [built separately](../linux/README.md), then packaged into Docker images.
+
+## Dependencies
+
+- Crystal binaries for `x86_64` and `aarch64` (targz)
+- `docker`
+
+## Environment
+
+- `CRYSTAL_VERSION`: How the binaries should be branded
+- `CRYSTAL_TARGZ`: Which crystal.tar.gz file to install in docker images (ubuntu64, alpine)
+- `DOCKER_TAG`: How to tag the docker image (examples: `0.27.2`, `nightly20190307`). `-build` will be appended for build images.
+- `DOCKER_REPOSITORY`: Docker hub repository to commit image