Skip to content

Commit 77df2c7

Browse files
MizuxMizux
committed
Add tools
1 parent c126929 commit 77df2c7

7 files changed

+1033
-0
lines changed

‎tools/Makefile

+322
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,322 @@
1+
PROJECT := python_native
2+
BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
3+
SHA1 := $(shell git rev-parse --verify HEAD)
4+
5+
# General commands
6+
.PHONY: help
7+
BOLD=\e[1m
8+
RESET=\e[0m
9+
10+
help:
11+
@echo -e "${BOLD}SYNOPSIS${RESET}"
12+
@echo -e "\tmake <TARGET> [NOCACHE=1]"
13+
@echo
14+
@echo -e "${BOLD}DESCRIPTION${RESET}"
15+
@echo -e "\tTools to generate various wheel packages."
16+
@echo
17+
@echo -e "${BOLD}MAIN TARGETS${RESET}"
18+
@echo -e "\t${BOLD}help${RESET}: display this help and exit."
19+
@echo -e "\t${BOLD}python${RESET}: Build musllinux and manylinux python '${PROJECT}' wheel packages (3.8+)."
20+
@echo -e "\t${BOLD}python_<platform>${RESET}: Build all python '${PROJECT}' wheel packages (3.8+) for a specific platform."
21+
@echo -e "\t${BOLD}python_<platform>_<step>${RESET}: Build all python '${PROJECT}' wheel packages (3.8+) for a specific platform."
22+
@echo -e "\t${BOLD}python_<target>_<step>${RESET}: Build python '${PROJECT}' wheel packages (3.8+) for a specific target."
23+
@echo -e "\t${BOLD}save_python_<target>${RESET}: Save python '${PROJECT}' image."
24+
@echo -e "\t${BOLD}clean_python_<target>${RESET}: Clean manylinux and musllinux python '${PROJECT}' wheel packages."
25+
@echo -e "\t${BOLD}sh_python_<target>${RESET}: Run a container using the python '${PROJECT}' image."
26+
@echo
27+
@echo -e "\t${BOLD}<platform>${RESET}:"
28+
@echo -e "\t\t${BOLD}amd64${RESET}"
29+
@echo -e "\t\t${BOLD}arm64${RESET}"
30+
@echo
31+
@echo -e "\t${BOLD}<target>${RESET}:"
32+
@echo -e "\t\t${BOLD}<platform>_<distro>${RESET}"
33+
@echo -e "\t\t${BOLD}<platform>_manylinux_cp<version>${RESET}"
34+
@echo
35+
@echo -e "\t${BOLD}<distro>${RESET}:"
36+
@echo -e "\t\t${BOLD}manylinux${RESET} (manylinux_2_28)"
37+
@echo -e "\t\t${BOLD}musllinux${RESET} (musllinux_1_2)"
38+
@echo
39+
@echo -e "\t${BOLD}<version>${RESET}:"
40+
@echo -e "\t\t${BOLD}38${RESET} Python3.8"
41+
@echo -e "\t\t${BOLD}39${RESET} Python3.9"
42+
@echo -e "\t\t${BOLD}310${RESET} Python3.10"
43+
@echo -e "\t\t${BOLD}311${RESET} Python3.11"
44+
@echo -e "\t\t${BOLD}312${RESET} Python3.12"
45+
@echo -e "\t\t${BOLD}313${RESET} Python3.13"
46+
@echo
47+
@echo -e "\t${BOLD}<step>${RESET}:"
48+
@echo -e "\t\t${BOLD}env${RESET}"
49+
@echo -e "\t\t${BOLD}devel${RESET}"
50+
@echo -e "\t\t${BOLD}build${RESET}"
51+
@echo -e "\t\t${BOLD}test${RESET}"
52+
@echo -e "\t\t${BOLD}export${RESET}"
53+
@echo -e "\te.g. 'make python_amd64_manylinux_cp39_export'"
54+
@echo -e "\te.g. 'make python_arm64_musllinux_export'"
55+
@echo
56+
@echo -e "\t${BOLD}NOCACHE=1${RESET}: use 'docker build --no-cache' when building container (default use cache)."
57+
@echo -e "\t${BOLD}VERBOSE=1${RESET}: use 'docker build --progress=plain' when building container."
58+
@echo
59+
@echo -e "${BOLD}NOTES${RESET}"
60+
@echo -e "\tAll generated code will be located in the export/ folder, use target ${BOLD}distclean${RESET} to remove it."
61+
@echo
62+
63+
# Delete all implicit rules to speed up makefile
64+
MAKEFLAGS += --no-builtin-rules
65+
.SUFFIXES:
66+
# Remove some rules from gmake that .SUFFIXES does not remove.
67+
SUFFIXES :=
68+
# keep all intermediate files e.g. export/docker_*.tar
69+
# src: https://www.gnu.org/software/make/manual/html_node/Special-Targets.html
70+
.SECONDARY:
71+
72+
$(info branch: ${BRANCH})
73+
$(info SHA1: ${SHA1})
74+
75+
DOCKER_BUILD_CMD := docker build
76+
DOCKER_BUILDX_CMD := docker buildx build
77+
ifdef NOCACHE
78+
DOCKER_BUILD_CMD := ${DOCKER_BUILD_CMD} --no-cache
79+
DOCKER_BUILDX_CMD := ${DOCKER_BUILDX_CMD} --no-cache
80+
endif
81+
ifdef VERBOSE
82+
DOCKER_BUILD_CMD := ${DOCKER_BUILD_CMD} --progress=plain
83+
DOCKER_BUILDX_CMD := ${DOCKER_BUILDX_CMD} --progress=plain
84+
endif
85+
DOCKER_RUN_CMD := docker run --rm --init --net=host
86+
87+
###############
88+
### PYTHON ##
89+
###############
90+
# $* stem
91+
# $< first prerequist
92+
# $@ target name
93+
PYTHON_PLATFORMS := amd64 arm64
94+
PYTHON_DISTROS := manylinux musllinux
95+
PYTHON_STAGES := env devel build test
96+
97+
export:
98+
-mkdir $@
99+
100+
cache:
101+
-mkdir $@
102+
103+
## MANYLINUX ##
104+
PYTHON_VERSIONS := 38 39 310 311 312 313
105+
106+
export/manylinux: | export
107+
-mkdir -p $@
108+
109+
export/manylinux/build-manylinux.sh: build-manylinux.sh | export/manylinux
110+
cp $< $@
111+
112+
define manylinux_inner =
113+
#$$(info manylinux_inner: PLATFORM:'$1' VERSION:'$2' STAGE:'$3')
114+
115+
.PHONY: python_$1_manylinux_cp$2_$3
116+
python_$1_manylinux_cp$2_$3: $1/manylinux.Dockerfile export/manylinux/build-manylinux.sh
117+
@docker image rm -f ${PROJECT}:$$@ 2>/dev/null
118+
${DOCKER_BUILDX_CMD} --platform linux/$1 \
119+
--tag ${PROJECT}:$$@ \
120+
--build-arg GIT_BRANCH=${BRANCH} \
121+
--build-arg GIT_SHA1=${SHA1} \
122+
--build-arg PYTHON_VERSION=$2 \
123+
--target=$3 \
124+
-f $$< \
125+
export/manylinux
126+
127+
.PHONY: save_python_$1_manylinux_cp$2_$3
128+
save_python_$1_manylinux_cp$2_$3: cache/docker_$1_manylinux_cp$2_$3.tar
129+
cache/docker_$1_manylinux_cp$2_$3.tar: python_$1_manylinux_cp$2_$3 | cache
130+
@rm -f $$@
131+
docker save ${PROJECT}:$$< -o $$@
132+
133+
.PHONY: clean_python_$1_manylinux_cp$2_$3
134+
clean_python_$1_manylinux_cp$2_$3: $1/manylinux.Dockerfile export/manylinux/build-manylinux.sh
135+
docker image rm -f ${PROJECT}:python_$1_manylinux_cp$2_$3 2>/dev/null
136+
rm -f cache/docker_$1_manylinux_cp$2_$3.tar
137+
138+
# Debug purpose
139+
.PHONY: sh_python_$1_manylinux_cp$2_$3
140+
sh_python_$1_manylinux_cp$2_$3: python_$1_manylinux_cp$2_$3
141+
${DOCKER_RUN_CMD} \
142+
-v `pwd`/export:/export \
143+
-it \
144+
--name ${PROJECT}_$$< \
145+
${PROJECT}:$$<
146+
endef
147+
148+
define manylinux_outer =
149+
#$$(info manylinux_outer: PLATFORM: '$1' VERSION: '$2')
150+
151+
$$(foreach stage,${PYTHON_STAGES},$$(eval $$(call manylinux_inner,$1,$2,$${stage})))
152+
153+
.PHONY: python_$1_manylinux_cp$2_export
154+
python_$1_manylinux_cp$2_export: python_$1_manylinux_cp$2_build
155+
${DOCKER_RUN_CMD} \
156+
-v `pwd`/export:/export \
157+
-it \
158+
--name ${PROJECT}_$$< \
159+
${PROJECT}:$$< \
160+
"cp build*/python/dist/*-many*.whl /export"
161+
endef
162+
163+
$(foreach version,${PYTHON_VERSIONS},$(eval $(call manylinux_outer,amd64,${version})))
164+
$(foreach version,${PYTHON_VERSIONS},$(eval $(call manylinux_outer,arm64,${version})))
165+
166+
# Merge
167+
define manylinux_merge =
168+
#$$(info manylinux_merge: PLATFORM:'$1' STAGE:'$2')
169+
170+
.PHONY: python_$1_manylinux_$2
171+
python_$1_manylinux_$2: $(addprefix python_$1_manylinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS}))
172+
.PHONY: save_python_$1_manylinux_$2
173+
save_python_$1_manylinux_$2: $(addprefix save_python_$1_manylinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS}))
174+
.PHONY: clean_python_$1_manylinux_$2
175+
clean_python_$1_manylinux_$2: $(addprefix clean_python_$1_manylinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS}))
176+
endef
177+
178+
$(foreach stage,${PYTHON_STAGES} export,$(eval $(call manylinux_merge,amd64,${stage})))
179+
$(foreach stage,${PYTHON_STAGES} export,$(eval $(call manylinux_merge,arm64,${stage})))
180+
181+
## MUSLLINUX ##
182+
export/musllinux: | export
183+
-mkdir -p $@
184+
185+
export/musllinux/build-musllinux.sh: build-musllinux.sh | export/musllinux
186+
cp $< $@
187+
188+
define musllinux_inner =
189+
#$$(info musllinux_inner: PLATFORM:'$1' VERSION:'$2' STAGE:'$3')
190+
191+
.PHONY: python_$1_musllinux_cp$2_$3
192+
python_$1_musllinux_cp$2_$3: $1/musllinux.Dockerfile | export/musllinux/build-musllinux.sh
193+
@docker image rm -f ${PROJECT}:$$@ 2>/dev/null
194+
${DOCKER_BUILDX_CMD} --platform linux/$1 \
195+
--tag ${PROJECT}:$$@ \
196+
--build-arg GIT_BRANCH=${BRANCH} \
197+
--build-arg GIT_SHA1=${SHA1} \
198+
--build-arg PYTHON_VERSION=$2 \
199+
--target=$3 \
200+
-f $$< \
201+
export/musllinux
202+
203+
.PHONY: save_python_$1_musllinux_cp$2_$3
204+
save_python_$1_musllinux_cp$2_$3: cache/docker_$1_musllinux_cp$2_$3.tar
205+
cache/docker_$1_musllinux_cp$2_$3.tar: python_$1_musllinux_cp$2_$3 | cache
206+
@rm -f $$@
207+
docker save ${PROJECT}:$$< -o $$@
208+
209+
.PHONY: clean_python_$1_musllinux_cp$2_$3
210+
clean_python_$1_musllinux_cp$2_$3: $1/musllinux.Dockerfile | export/musllinux/build-musllinux.sh
211+
docker image rm -f ${PROJECT}:python_$1_musllinux_cp$2_$3 2>/dev/null
212+
rm -f cache/docker_$1_musllinux_cp$2_$3.tar
213+
214+
# Debug purpose
215+
.PHONY: sh_python_$1_musllinux_cp$2_$3
216+
sh_python_$1_musllinux_cp$2_$3: python_$1_musllinux_cp$2_$3
217+
${DOCKER_RUN_CMD} \
218+
-v `pwd`/export:/export \
219+
-it \
220+
--name ${PROJECT}_$$< \
221+
${PROJECT}:$$<
222+
endef
223+
224+
define musllinux_outer =
225+
#$$(info musllinux_outer: PLATFORM: '$1' VERSION: '$2')
226+
227+
$$(foreach stage,${PYTHON_STAGES},$$(eval $$(call musllinux_inner,$1,$2,$${stage})))
228+
229+
.PHONY: python_$1_musllinux_cp$2_export
230+
python_$1_musllinux_cp$2_export: python_$1_musllinux_cp$2_build
231+
${DOCKER_RUN_CMD} \
232+
-v `pwd`/export:/export \
233+
-it \
234+
--name ${PROJECT}_$$< \
235+
${PROJECT}:$$< \
236+
"cp build*/python/dist/*-musl*.whl /export"
237+
endef
238+
239+
$(foreach version,${PYTHON_VERSIONS},$(eval $(call musllinux_outer,amd64,${version})))
240+
$(foreach version,${PYTHON_VERSIONS},$(eval $(call musllinux_outer,arm64,${version})))
241+
242+
# Merge
243+
define musllinux_merge =
244+
#$$(info musllinux_merge: PLATFORM:'$1' STAGE:'$2')
245+
246+
.PHONY: python_$1_musllinux_$2
247+
python_$1_musllinux_$2: $(addprefix python_$1_musllinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS}))
248+
.PHONY: save_python_$1_musllinux_$2
249+
save_python_$1_musllinux_$2: $(addprefix save_python_$1_musllinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS}))
250+
.PHONY: clean_python_$1_musllinux_$2
251+
clean_python_$1_musllinux_$2: $(addprefix clean_python_$1_musllinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS}))
252+
endef
253+
254+
$(foreach stage,${PYTHON_STAGES} export,$(eval $(call musllinux_merge,amd64,${stage})))
255+
$(foreach stage,${PYTHON_STAGES} export,$(eval $(call musllinux_merge,arm64,${stage})))
256+
257+
## MERGE DISTRO ##
258+
define python_distro_merge =
259+
#$$(info python_distro_merge: PLATFORM:'$1' STAGE:'$2')
260+
261+
.PHONY: python_$1_$2
262+
python_$1_$2: $(addprefix python_$1_, $(addsuffix _$2, ${PYTHON_DISTROS}))
263+
.PHONY: save_python_$1_$2
264+
save_python_$1_$2: $(addprefix save_python_$1_, $(addsuffix _$2, ${PYTHON_DISTROS}))
265+
.PHONY: clean_python_$1_$2
266+
clean_python_$1_$2: $(addprefix clean_python_$1_, $(addsuffix _$2, ${PYTHON_DISTROS}))
267+
endef
268+
269+
$(foreach stage,${PYTHON_STAGES} export,$(eval $(call python_distro_merge,amd64,${stage})))
270+
$(foreach stage,${PYTHON_STAGES} export,$(eval $(call python_distro_merge,arm64,${stage})))
271+
272+
## MERGE PLATFORM ##
273+
define clean_python_platform =
274+
#$$(info clean_python_platform: PLATFORM:'$1')
275+
276+
.PHONY: clean_python_$1
277+
clean_python_$1: $(addprefix clean_python_$1_, ${PYTHON_STAGES})
278+
endef
279+
280+
$(foreach platform,${PYTHON_PLATFORMS},$(eval $(call clean_python_platform,${platform})))
281+
282+
283+
define python_platform_merge =
284+
#$$(info python_platform_merge: STAGE:'$1')
285+
286+
.PHONY: python_$1
287+
python_$1: $(addprefix python_, $(addsuffix _$1, ${PYTHON_PLATFORMS}))
288+
.PHONY: save_python_$1
289+
save_python_$1: $(addprefix save_python_, $(addsuffix _$1, ${PYTHON_PLATFORMS}))
290+
.PHONY: clean_python_$1
291+
clean_python_$1: $(addprefix clean_python_, $(addsuffix _$1, ${PYTHON_PLATFORMS}))
292+
endef
293+
294+
$(foreach stage,${PYTHON_STAGES} export,$(eval $(call python_platform_merge,${stage})))
295+
296+
# Alias
297+
.PHONY: python
298+
python: python_amd64_export
299+
300+
.PHONY: clean_python
301+
clean_python: $(addprefix clean_python_, ${PYTHON_PLATFORMS})
302+
-rm -rf cache/*
303+
-rm -rf export/*
304+
305+
#############
306+
## CLEAN ##
307+
#############
308+
.PHONY: clean
309+
clean: clean_python
310+
-docker container prune -f
311+
-docker image prune -f
312+
-rm -rf cache
313+
314+
.PHONY: distclean
315+
distclean: clean
316+
-docker container ls -a
317+
-docker container prune -f
318+
-docker image ls -a
319+
-docker image prune -a -f
320+
-docker system df
321+
-docker system prune -a -f
322+
-rm -rf export

‎tools/amd64/manylinux.Dockerfile

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# ref: https://github.com/pypa/manylinux
2+
FROM quay.io/pypa/manylinux_2_28_x86_64:latest AS env
3+
# note: Almalinux:8 based image with
4+
# CMake 3.31.2 and SWIG 4.3.0 already installed
5+
6+
RUN dnf -y update \
7+
&& dnf -y install \
8+
curl wget \
9+
git patch \
10+
which pkgconfig autoconf libtool \
11+
make gcc-c++ \
12+
redhat-lsb openssl-devel pcre2-devel \
13+
zlib-devel unzip zip \
14+
&& dnf clean all \
15+
&& rm -rf /var/cache/dnf
16+
ENTRYPOINT ["/usr/bin/bash", "-c"]
17+
CMD ["/usr/bin/bash"]
18+
19+
#####################
20+
## PYTHON-NATIVE ##
21+
#####################
22+
FROM env AS devel
23+
ENV GIT_URL=https://github.com/Mizux/python-native
24+
25+
ARG GIT_BRANCH
26+
ENV GIT_BRANCH=${GIT_BRANCH:-main}
27+
ARG GIT_SHA1
28+
ENV GIT_SHA1=${GIT_SHA1:-unknown}
29+
30+
# Download sources
31+
# use GIT_SHA1 to modify the command
32+
# i.e. avoid docker reusing the cache when new commit is pushed
33+
RUN git clone -b "${GIT_BRANCH}" --single-branch "$GIT_URL" /project \
34+
&& cd /project \
35+
&& git reset --hard "${GIT_SHA1}"
36+
WORKDIR /project
37+
38+
# Copy build script and setup env
39+
ENV PLATFORM=x86_64
40+
ARG PYTHON_VERSION
41+
ENV PYTHON_VERSION=${PYTHON_VERSION:-3}
42+
COPY build-manylinux.sh .
43+
RUN chmod a+x "build-manylinux.sh"
44+
45+
FROM devel AS build
46+
RUN ./build-manylinux.sh build
47+
48+
FROM build AS test
49+
RUN ./build-manylinux.sh test

0 commit comments

Comments
 (0)