Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

split packages #716

Draft
wants to merge 6 commits into
base: master
Choose a base branch
from
Draft

split packages #716

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8cd69c5
static: Makefile cleanup and reformatting
thaJeztah Jul 6, 2022
58e329a
static: Makefile: add BUILD_DIR var
thaJeztah Jul 6, 2022
59c7a85
static: Makefile: add CLI_VERSION
thaJeztah Jul 6, 2022
02c24cb
static: split packages, add plugins, split versions
thaJeztah Jul 4, 2022
595b0ee
Jenkinsfile: show built artifacts for debugging
thaJeztah Jul 4, 2022
c47111f
Jenkinsfile: disable stages
thaJeztah Jul 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ def build_package_steps = [
checkout scm
sh "make REF=$branch DOCKER_BUILD_PKGS='static-linux' static"
} finally {
sh "ls -lR static/build"
sh "make clean"
}
}
Expand All @@ -93,6 +94,7 @@ def build_package_steps = [
checkout scm
sh "make REF=$branch DOCKER_BUILD_PKGS='cross-mac' static"
} finally {
sh "ls -lR static/build"
sh "make clean"
}
}
Expand All @@ -114,6 +116,7 @@ def build_package_steps = [
checkout scm
sh "make REF=$branch DOCKER_BUILD_PKGS='cross-win' static"
} finally {
sh "ls -lR static/build"
sh "make clean"
}
}
Expand All @@ -127,6 +130,6 @@ def genPackageSteps(opts) {
}
}

build_package_steps << pkgs.collectEntries { genPackageSteps(it) }
// build_package_steps << pkgs.collectEntries { genPackageSteps(it) }

parallel(build_package_steps)
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ debian-% raspbian-% ubuntu-%: checkout ## build deb packages for the specified d
static: DOCKER_BUILD_PKGS:=static-linux cross-mac cross-win cross-arm
static: checkout ## build static-compiled packages
for p in $(DOCKER_BUILD_PKGS); do \
$(MAKE) -C $@ VERSION=$(VERSION) GO_VERSION=$(GO_VERSION) TARGETPLATFORM=$(TARGETPLATFORM) CONTAINERD_VERSION=$(CONTAINERD_VERSION) RUNC_VERSION=$(RUNC_VERSION) $${p}; \
$(MAKE) -C $@ VERSION=$(VERSION) GO_VERSION=$(GO_VERSION) CONTAINERD_VERSION=$(CONTAINERD_VERSION) RUNC_VERSION=$(RUNC_VERSION) $${p}; \
done

.PHONY: verify
Expand Down
201 changes: 163 additions & 38 deletions static/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
include ../common.mk

CLI_DIR=$(realpath $(CURDIR)/../src/github.com/docker/cli)
ENGINE_DIR=$(realpath $(CURDIR)/../src/github.com/docker/docker)
BUILDX_DIR=$(realpath $(CURDIR)/../src/github.com/docker/buildx)
CLI_DIR = $(realpath $(CURDIR)/../src/github.com/docker/cli)
ENGINE_DIR = $(realpath $(CURDIR)/../src/github.com/docker/docker)
COMPOSE_DIR = $(realpath $(CURDIR)/../src/github.com/docker/compose)
BUILDX_DIR = $(realpath $(CURDIR)/../src/github.com/docker/buildx)
SCAN_DIR = $(realpath $(CURDIR)/../src/github.com/docker/scan-cli-plugin)

CLI_VERSION = $(shell ./gen-static-ver $(CLI_DIR) $(VERSION))
ENGINE_VERSION = $(shell ./gen-static-ver $(ENGINE_DIR) $(VERSION))
CONTAINERD_PKG_VER = $(shell ./gen-containerd-ver $(ENGINE_DIR) $(CONTAINERD_VERSION))

GEN_STATIC_VER=$(shell ./gen-static-ver $(CLI_DIR) $(VERSION))
HASH_CMD=docker run -v $(CURDIR):/sum -w /sum debian:jessie bash hash_files
DIR_TO_HASH:=build/linux
DOCKER_CLI_GOLANG_IMG=golang:$(GO_VERSION)
Expand Down Expand Up @@ -39,25 +44,48 @@ clean: ## remove build artifacts
static: static-linux cross-mac cross-win cross-arm ## create all static packages

.PHONY: static-linux
static-linux: static-cli static-engine static-buildx-plugin ## create tgz
mkdir -p build/linux/docker
cp $(CLI_DIR)/build/docker build/linux/docker/
for f in dockerd containerd ctr containerd-shim containerd-shim-runc-v2 docker-init docker-proxy runc; do \
cp -L $(ENGINE_DIR)/bundles/binary-daemon/$$f build/linux/docker/$$f; \
static-linux: BUILD_DIR=build/linux
static-linux: static-cli static-engine static-buildx-plugin static-compose-plugin docker-scan-plugin ## create tgz
# docker cli binaries
mkdir -p $(BUILD_DIR)/docker-cli
cp $(CLI_DIR)/build/docker $(BUILD_DIR)/docker-cli/
tar -C $(BUILD_DIR) -c -z -f $(BUILD_DIR)/docker-cli-$(CLI_VERSION).tgz docker-cli
$(RM) -r $(BUILD_DIR)/docker-cli

# docker engine binaries
mkdir -p $(BUILD_DIR)/docker-engine
for f in dockerd docker-init docker-proxy; do \
cp -L $(ENGINE_DIR)/bundles/binary-daemon/$$f $(BUILD_DIR)/docker-engine/$$f; \
done
tar -C $(BUILD_DIR) -c -z -f $(BUILD_DIR)/docker-engine-$(ENGINE_VERSION).tgz docker-engine
$(RM) -r $(BUILD_DIR)/docker-engine

# containerd binaries
mkdir -p $(BUILD_DIR)/containerd
for f in containerd ctr containerd-shim-runc-v2 runc; do \
cp -L $(ENGINE_DIR)/bundles/binary-daemon/$$f $(BUILD_DIR)/containerd/$$f; \
done
tar -C build/linux -c -z -f build/linux/docker-$(GEN_STATIC_VER).tgz docker
tar -C $(BUILD_DIR) -c -z -f $(BUILD_DIR)/containerd-$(CONTAINERD_PKG_VER).tgz containerd
$(RM) -r $(BUILD_DIR)/containerd

# extra binaries for running rootless
mkdir -p build/linux/docker-rootless-extras
mkdir -p $(BUILD_DIR)/docker-rootless-extras
for f in rootlesskit rootlesskit-docker-proxy dockerd-rootless.sh dockerd-rootless-setuptool.sh vpnkit; do \
if [ -f $(ENGINE_DIR)/bundles/binary-daemon/$$f ]; then \
cp -L $(ENGINE_DIR)/bundles/binary-daemon/$$f build/linux/docker-rootless-extras/$$f; \
cp -L $(ENGINE_DIR)/bundles/binary-daemon/$$f $(BUILD_DIR)/docker-rootless-extras/$$f; \
fi \
done
tar -C build/linux -c -z -f build/linux/docker-rootless-extras-$(GEN_STATIC_VER).tgz docker-rootless-extras
tar -C $(BUILD_DIR) -c -z -f $(BUILD_DIR)/docker-rootless-extras-$(ENGINE_VERSION).tgz docker-rootless-extras
$(RM) -r $(BUILD_DIR)/docker-rootless-extras

# docker compose
tar -C $(COMPOSE_DIR)/bin -c -z -f $(BUILD_DIR)/docker-compose-plugin-$(DOCKER_COMPOSE_REF:v%=%).tgz docker-compose

# docker buildx
tar -C $(BUILDX_DIR)/bin -c -z -f $(BUILD_DIR)/docker-buildx-plugin-$(DOCKER_BUILDX_REF:v%=%).tgz docker-buildx

# buildx
tar -C $(BUILDX_DIR)/bin -c -z -f build/linux/docker-buildx-plugin-$(DOCKER_BUILDX_REF:v%=%).tgz docker-buildx
# docker scan
tar -C $(SCAN_DIR)/bin -c -z -f $(BUILD_DIR)/docker-scan-plugin-$(DOCKER_SCAN_REF:v%=%).tgz docker-scan

.PHONY: hash_files
hash_files:
Expand All @@ -69,47 +97,144 @@ buildx:
docker buildx inspect | grep -q 'Driver: docker-container' || docker buildx create --use

.PHONY: cross-mac
cross-mac: buildx
cd $(CLI_DIR) && VERSION=$(GEN_STATIC_VER) docker buildx bake --set binary.platform=darwin/amd64,darwin/arm64 binary
dest=$$PWD/build/mac; cd $(CLI_DIR)/build && for platform in *; do \
arch=$$(echo $$platform | cut -d_ -f2); \
mkdir -p $$dest/$$arch/docker; \
cp $$platform/docker-darwin-* $$dest/$$arch/docker/docker && \
tar -C $$dest/$$arch -c -z -f $$dest/$$arch/docker-$(GEN_STATIC_VER).tgz docker; \
done
cross-mac: cross-mac-amd64 cross-mac-arm64
@echo "Cross-compiling mac binaries"

.PHONY: cross-mac-amd64
cross-mac-amd64: TARGETPLATFORM=darwin/amd64
cross-mac-amd64: BUILD_DIR=build/mac/amd64
cross-mac-amd64: buildx static-cli static-buildx-plugin static-compose-plugin cross-scan-plugin
# docker cli binaries
mkdir -p $(BUILD_DIR)/docker-cli
cp -L $(CLI_DIR)/build/docker $(BUILD_DIR)/docker-cli/docker
tar -C $(BUILD_DIR)/ -c -z -f $(BUILD_DIR)/docker-cli-$(CLI_VERSION).tgz docker-cli
$(RM) -r $(BUILD_DIR)/docker-cli

# docker compose
tar -C $(COMPOSE_DIR)/bin -c -z -f $(BUILD_DIR)/docker-compose-plugin-$(DOCKER_COMPOSE_REF:v%=%).tgz docker-compose

# docker buildx
tar -C $(BUILDX_DIR)/bin -c -z -f $(BUILD_DIR)/docker-buildx-plugin-$(DOCKER_BUILDX_REF:v%=%).tgz docker-buildx

# docker scan
mkdir -p $(BUILD_DIR)/docker-scan
cp -L $(SCAN_DIR)/dist/docker-scan_darwin_amd64 $(BUILD_DIR)/docker-scan/docker-scan
tar -C $(BUILD_DIR)/ -c -z -f $(BUILD_DIR)/docker-scan-plugin-$(DOCKER_SCAN_REF:v%=%).tgz docker-scan
$(RM) -r $(BUILD_DIR)/docker-scan

.PHONY: cross-mac-arm64
cross-mac-arm64: TARGETPLATFORM=darwin/arm64
cross-mac-arm64: BUILD_DIR=build/mac/arm64
cross-mac-arm64: buildx static-cli static-buildx-plugin static-compose-plugin cross-scan-plugin
# docker cli binaries
mkdir -p $(BUILD_DIR)/docker-cli
cp -L $(CLI_DIR)/build/docker $(BUILD_DIR)/docker-cli/docker
tar -C $(BUILD_DIR)/ -c -z -f $(BUILD_DIR)/docker-cli-$(CLI_VERSION).tgz docker-cli
$(RM) -r $(BUILD_DIR)/docker-cli

# docker compose
tar -C $(COMPOSE_DIR)/bin -c -z -f $(BUILD_DIR)/docker-compose-plugin-$(DOCKER_COMPOSE_REF:v%=%).tgz docker-compose

# docker buildx
tar -C $(BUILDX_DIR)/bin -c -z -f $(BUILD_DIR)/docker-buildx-plugin-$(DOCKER_BUILDX_REF:v%=%).tgz docker-buildx

# docker scan
mkdir -p $(BUILD_DIR)/docker-scan
cp -L $(SCAN_DIR)/dist/docker-scan_darwin_arm64 $(BUILD_DIR)/docker-scan/docker-scan
tar -C $(BUILD_DIR)/ -c -z -f $(BUILD_DIR)/docker-scan-plugin-$(DOCKER_SCAN_REF:v%=%).tgz docker-scan
$(RM) -r $(BUILD_DIR)/docker-scan

.PHONY: cross-win
cross-win: cross-win-engine
cd $(CLI_DIR) && VERSION=$(GEN_STATIC_VER) docker buildx bake --set binary.platform=windows/amd64 binary
mkdir -p build/win/amd64/docker
cp $(CLI_DIR)/build/docker-windows-amd64.exe build/win/amd64/docker/docker.exe
cp $(ENGINE_DIR)/bundles/cross/windows/amd64-daemon/dockerd.exe build/win/amd64/docker/dockerd.exe
cp $(ENGINE_DIR)/bundles/cross/windows/amd64-daemon/docker-proxy.exe build/win/amd64/docker/docker-proxy.exe
docker run --rm -v $(CURDIR)/build/win/amd64:/v -w /v alpine sh -c 'apk update&&apk add zip&&zip -r docker-$(GEN_STATIC_VER).zip docker'
cross-win: TARGETPLATFORM=windows/amd64
cross-win: EXT=.exe
cross-win: BUILD_DIR=build/win/amd64
cross-win: cross-win-engine static-cli static-buildx-plugin static-compose-plugin cross-scan-plugin
# docker cli binaries
mkdir -p $(BUILD_DIR)/docker-cli
cp -L $(CLI_DIR)/build/docker $(BUILD_DIR)/docker-cli/docker.exe

# docker engine binaries
mkdir -p $(BUILD_DIR)/docker-engine
for f in dockerd.exe docker-proxy.exe; do \
cp -L $(ENGINE_DIR)/bundles/cross/windows/amd64-daemon/$$f $(BUILD_DIR)/docker-engine/$$f; \
done

# docker compose, docker buildx
mkdir -p $(BUILD_DIR)/docker-compose
cp $(COMPOSE_DIR)/bin/docker-compose/docker-compose.exe $(BUILD_DIR)/docker-compose/docker-compose.exe

# docker buildx
mkdir -p $(BUILD_DIR)/docker-buildx
cp $(BUILDX_DIR)/bin/docker-buildx/docker-buildx.exe $(BUILD_DIR)/docker-buildx/docker-buildx.exe

# docker scan
mkdir -p $(BUILD_DIR)/docker-scan
cp $(SCAN_DIR)/dist/docker-scan_windows_amd64.exe $(BUILD_DIR)/docker-scan/docker-scan.exe

docker run --rm -v $(CURDIR)/$(BUILD_DIR):/v -w /v alpine sh -c 'apk add --no-cache zip \
&& zip -r docker-cli-$(CLI_VERSION).zip docker-cli \
&& zip -r docker-engine-$(ENGINE_VERSION).zip docker-engine \
&& zip -r docker-compose-plugin-$(DOCKER_COMPOSE_REF:v%=%).zip docker-compose \
&& zip -r docker-buildx-plugin-$(DOCKER_BUILDX_REF:v%=%).zip docker-buildx '
$(CHOWN) -R $(shell id -u):$(shell id -g) build

# clean up temporary directories
$(RM) -r $(BUILD_DIR)/docker-cli $(BUILD_DIR)/docker-engine $(BUILD_DIR)/docker-compose $(BUILD_DIR)/docker-buildx $(BUILD_DIR)/docker-scan

.PHONY: cross-arm
cross-arm: BUILD_DIR=build/arm
cross-arm: cross-all-cli ## create tgz with linux armhf client only
mkdir -p build/arm/docker
cp $(CLI_DIR)/build/docker-linux-arm build/arm/docker/docker
tar -C build/arm -c -z -f build/arm/docker-$(GEN_STATIC_VER).tgz docker
mkdir -p $(BUILD_DIR)/docker-cli
cp $(CLI_DIR)/build/docker-linux-arm $(BUILD_DIR)/docker-cli/docker
tar -C $(BUILD_DIR) -c -z -f $(BUILD_DIR)/docker-$(CLI_VERSION).tgz docker-cli
$(RM) -r $(BUILD_DIR)/docker-cli

.PHONY: static-cli
static-cli:
cd $(CLI_DIR) && VERSION=$(GEN_STATIC_VER) docker buildx bake --set binary.platform=$(TARGETPLATFORM) --set binary.args.CGO_ENABLED=$(CGO_ENABLED) binary
# TODO(thaJeztah) check if CGO_ENABLED variable is correct here
cd $(CLI_DIR) && VERSION=$(CLI_VERSION) docker buildx bake --set binary.platform=$(TARGETPLATFORM) --set binary.args.CGO_ENABLED=$(CGO_ENABLED) binary

.PHONY: static-engine
static-engine:
$(MAKE) -C $(ENGINE_DIR) VERSION=$(GEN_STATIC_VER) DOCKER_BUILD_OPTS="$(DOCKER_BUILD_OPTS)" binary
$(MAKE) -C $(ENGINE_DIR) VERSION=$(ENGINE_VERSION) DOCKER_BUILD_OPTS="$(DOCKER_BUILD_OPTS)" binary

.PHONY: static-buildx-plugin
static-buildx-plugin:
cd $(BUILDX_DIR) && docker buildx bake --set binaries.platform=$(TARGETPLATFORM) binaries && mv ./bin/buildx ./bin/docker-buildx
cd $(BUILDX_DIR) && docker buildx bake --set binaries.platform=$(TARGETPLATFORM) binaries && mkdir -p ./bin/docker-buildx/ && mv ./bin/buildx$(EXT) ./bin/docker-buildx/docker-buildx$(EXT)

.PHONY: static-compose-plugin
static-compose-plugin:
$(RM) -r "$(COMPOSE_DIR)"/bin
docker buildx build "$(COMPOSE_DIR)" \
--target compose-plugin \
--platform "$(or $(TARGETPLATFORM),local)" \
--build-arg GIT_TAG="$(DOCKER_COMPOSE_REF)" \
--output $(COMPOSE_DIR)/bin/docker-compose/

.PHONY: docker-scan-plugin
docker-scan-plugin:
# Build the scan-plugin
# TODO this should be skipped on non-x86 because there's no image for non-x86?
# TODO change once we support scan-plugin on other architectures
$(MAKE) -C $(SCAN_DIR) GIT_TAG_NAME="$(DOCKER_SCAN_REF)" build

# TODO(thaJeztah) upstream Makefile should pass PLATFORM_BINARY
# Binary is named (e.g.) docker-scan_linux_amd64. Rename it to have something consistent
mv $(SCAN_DIR)/bin/docker-scan_linux_* $(SCAN_DIR)/bin/docker-scan

ls -laR $(SCAN_DIR)/bin

.PHONY: cross-scan-plugin
cross-scan-plugin:
# TODO: Add TARGETPLATFORM support on scan-cli-plugin repo to build efficiently with --platform
$(MAKE) -C $(SCAN_DIR) GIT_TAG_NAME="$(DOCKER_SCAN_REF)" cross

ls -laR $(SCAN_DIR)/dist

.PHONY: cross-all-cli
cross-all-cli:
$(MAKE) -C $(CLI_DIR) -f docker.Makefile VERSION=$(GEN_STATIC_VER) cross
$(MAKE) -C $(CLI_DIR) -f docker.Makefile VERSION=$(CLI_VERSION) cross

.PHONY: cross-win-engine
cross-win-engine:
$(MAKE) -C $(ENGINE_DIR) VERSION=$(GEN_STATIC_VER) DOCKER_CROSSPLATFORMS=windows/amd64 DOCKER_BUILD_OPTS="$(DOCKER_BUILD_OPTS)" cross
$(MAKE) -C $(ENGINE_DIR) VERSION=$(ENGINE_VERSION) DOCKER_CROSSPLATFORMS=windows/amd64 DOCKER_BUILD_OPTS="$(DOCKER_BUILD_OPTS)" cross
15 changes: 15 additions & 0 deletions static/gen-containerd-ver
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/usr/bin/env bash

engine_dir="$1"
# containerd_pkg_version is used to name the containerd-<version>.tgz file.
containerd_pkg_version="$2"

if [ -z "${containerd_pkg_version}" ]; then
# If no custom version is set for containerd, we select the default version of
# containerd based on the docker engine source.
# which is needed for naming the produced .tgz file.
# TODO containerd binaries should be built as part of containerd-packaging, not as part of docker/docker-ce-packaging
awk -F'=' '$1 == "ARG CONTAINERD_VERSION" {{sub("v","")}; print $2 }' "${engine_dir}/Dockerfile.windows"
else
echo "${containerd_pkg_version#v}"
fi