Skip to content

Commit 9fde7d2

Browse files
mahdiyehmrmahdiyehmr
committed
Support importing preinstallimage as container image
Implement the option to user preinstallimage as base container image for podman and docker build environments. This speeds up the build process since there would be no need to untar the preinstallimage and install the packages inside it. It would only import the preinstallimage if it doesn't already find it implemented in preinstall_container_image(). For finding the package id and HDRMD5 of packages in preinstallimage it prioritizes the preinstallimageinfo and if not found it would fallback to previous method of reading unpacked preinstallimage. Adding randomization for container name in both podman and docker to avoid issues in concurrent builds.
1 parent 277ee4c commit 9fde7d2

File tree

5 files changed

+177
-33
lines changed

5 files changed

+177
-33
lines changed

‎build‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,7 @@ Known Parameters:
346346
347347
--vm-type TYPE
348348
Use virtual machine instead of chroot
349-
TYPE is one of xen|kvm|uml|qemu|lxc|zvm|openstack|ec2|docker|pvm|nspawn
349+
TYPE is one of xen|kvm|uml|qemu|lxc|zvm|openstack|ec2|docker|pvm|nspawn|podman
350350
351351
--vm-args ARGS
352352
Pass extra arguments to virtual machine
@@ -426,6 +426,11 @@ Known Parameters:
426426
Available options are -net, -netdev, -device
427427
(This options in kvm can not guarantee reproducible builds)
428428
429+
--vm-use-container-preinstallimage
430+
Use preinstall container image as base image in containers instead of
431+
using preinstall image tar file. This option is only available when vm-type
432+
is one of podman|docker
433+
429434
--debuginfo
430435
Enable creation of debuginfo packages
431436

‎build-vm‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ VM_TYPE=
3535
declare -a VM_ARGS
3636
VM_TYPE_PRIVILEGED=
3737
VM_TYPE_CONTAINER=
38+
VM_USE_CONTAINER_PREINSTALLIMAGE=false
39+
VM_CONTAINER_IMAGE=
3840
VM_ROOT=
3941
VM_SWAP=
4042
VM_ROOT_TYPE=
@@ -304,6 +306,13 @@ vm_parse_options() {
304306
VM_OPENSTACK_FLAVOR="$ARG"
305307
shift
306308
;;
309+
-vm-use-container-preinstallimage)
310+
if test "$VM_TYPE" == "docker" -o "$VM_TYPE" == "podman"; then
311+
VM_USE_CONTAINER_PREINSTALLIMAGE=true
312+
else
313+
cleanup_and_exit 1 "Option 'vm-use-container-preinstallimage' is not supported"
314+
fi
315+
;;
307316
-*)
308317
return 1
309318
;;
@@ -984,11 +993,16 @@ vm_first_stage() {
984993
if test "$DO_INIT" = true ; then
985994
# do first stage of init_buildsystem
986995
rm -f $BUILD_ROOT/.build.success
987-
set -- init_buildsystem --configdir "$CONFIG_DIR" --cachedir "$CACHE_DIR" --prepare "${initbuildsysstuff[@]}" "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$RECIPEPATH" $ADDITIONAL_PACKS
996+
if test "$VM_USE_CONTAINER_PREINSTALLIMAGE" = "true"; then
997+
set -- init_buildsystem --use-container-preinstallimage "$VM_TYPE" --configdir "$CONFIG_DIR" --cachedir "$CACHE_DIR" --prepare "${initbuildsysstuff[@]}" "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$RECIPEPATH" $ADDITIONAL_PACKS
998+
else
999+
set -- init_buildsystem --configdir "$CONFIG_DIR" --cachedir "$CACHE_DIR" --prepare "${initbuildsysstuff[@]}" "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$RECIPEPATH" $ADDITIONAL_PACKS
1000+
fi
9881001
echo "$* ..."
9891002
start_time=$SECONDS
9901003
"$@" || cleanup_and_exit 1
9911004
check_exit
1005+
test -f "$BUILD_ROOT/.build/build.data" && source "$BUILD_ROOT/.build/build.data"
9921006
TIME_PREINSTALL=$(( $SECONDS - $start_time ))
9931007
unset start_time
9941008
if test ! -w /root ; then
@@ -1087,6 +1101,9 @@ vm_first_stage() {
10871101
echo ")" >> $BUILD_ROOT/.build/build.data
10881102
echo "VM_TYPE='$VM_TYPE'" >> $BUILD_ROOT/.build/build.data
10891103
echo "VM_TYPE_CONTAINER='$VM_TYPE_CONTAINER'" >> $BUILD_ROOT/.build/build.data
1104+
test -n "$CONTAINER_PREINSTALLIMAGE" && echo "CONTAINER_PREINSTALLIMAGE='$CONTAINER_PREINSTALLIMAGE'" >> $BUILD_ROOT/.build/build.data
1105+
test "$VM_TYPE" = "docker" && echo "CONTAINER_NAME='obsbuild.${BUILD_ROOT##*/}.$(date +%s).${RANDOM}'" >> $BUILD_ROOT/.build/build.data
1106+
test "$VM_TYPE" = "podman" && echo "CONTAINER_NAME='build_${RECIPEFILE//:/-}.$(date +%s).${RANDOM}'" >> $BUILD_ROOT/.build/build.data
10901107
echo "VM_NETWORK='$VM_NETWORK'" >> $BUILD_ROOT/.build/build.data
10911108
echo "RUN_SHELL='$RUN_SHELL'" >> $BUILD_ROOT/.build/build.data
10921109
echo "RUN_SHELL_AFTER_FAIL='$RUN_SHELL_AFTER_FAIL'" >> $BUILD_ROOT/.build/build.data
@@ -1274,6 +1291,11 @@ vm_save_statistics() {
12741291
# args: resultdirs
12751292
vm_wrapup_build() {
12761293
test "$DO_STATISTICS" = 1 && vm_save_statistics
1294+
if test "$BUILDTYPE" = "preinstallimage"; then
1295+
case "$VM_TYPE" in
1296+
docker|podman) preinstallimage_compress "$BUILD_ROOT/$TOPDIR/OTHER" ;;
1297+
esac
1298+
fi
12771299
if test -n "$VM_SWAP"; then
12781300
echo "... saving built packages"
12791301
swapoff "$VM_SWAP"

‎build-vm-docker‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,24 +28,48 @@ vm_verify_options_docker() {
2828
}
2929

3030
vm_startup_docker() {
31-
local name="obsbuild.${BUILD_ROOT##*/}"
31+
source $BUILD_ROOT/.build/build.data
32+
local name=$CONTAINER_NAME
3233
docker rm "$name" >/dev/null 2>&1 || true
3334
local docker_opts=
35+
local container_root="/mnt"
36+
local build_command=(chroot /mnt "$vm_init_script")
37+
local mounts=("--mount" "type=bind,source=$BUILD_ROOT,destination=/mnt")
3438
test -n "$VM_TYPE_PRIVILEGED" && docker_opts="--privileged --cap-add=SYS_ADMIN --cap-add=MKNOD"
3539
test -n "$RUN_SHELL" -o -n "$RUN_SHELL_AFTER_FAIL" && docker_opts="$docker_opts -it"
40+
local docker_image="busybox"
41+
if test -n "$VM_CONTAINER_IMAGE" ; then
42+
mkdir -p $BUILD_ROOT/home/abuild
43+
# create .build.packages link
44+
rm -rf $BUILD_ROOT/.build.packages
45+
ln -s home/abuild/rpmbuild $BUILD_ROOT/.build.packages
46+
docker_image="$VM_CONTAINER_IMAGE"
47+
container_root=
48+
mounts+=("--mount" "type=bind,source=$BUILD_ROOT/home/abuild,destination=/home/abuild")
49+
local build_command=(bash -c "
50+
# Create symlinks for /mnt hidden directories that include build script and data
51+
for item in /mnt/.[^.]*; do
52+
[ -d \"\$item\" ] && ln -sf \"\$item\" \"/\$(basename \"\$item\")\"
53+
done
54+
$vm_init_script
55+
")
56+
fi
57+
mounts+=(
58+
"--mount" "type=bind,source=/proc,destination=$container_root/proc"
59+
"--mount" "type=bind,source=/dev/pts,destination=$container_root/dev/pts"
60+
"--mount" "type=bind,source=/dev/null,destination=$container_root/dev/null"
61+
)
3662
docker run \
3763
--rm --name "$name" --net=none $docker_opts \
38-
--mount "type=bind,source=$BUILD_ROOT,destination=/mnt" \
39-
--mount "type=bind,source=/proc,destination=/mnt/proc" \
40-
--mount "type=bind,source=/dev/pts,destination=/mnt/dev/pts" \
41-
"$@" busybox chroot /mnt "$vm_init_script"
64+
"${mounts[@]}" "$@" "$docker_image" "${build_command[@]}"
4265
BUILDSTATUS="$?"
4366
test "$BUILDSTATUS" != 255 || BUILDSTATUS=3
4467
cleanup_and_exit "$BUILDSTATUS"
4568
}
4669

4770
vm_kill_docker() {
48-
local name="obsbuild.${BUILD_ROOT##*/}"
71+
source $BUILD_ROOT/.build/build.data
72+
local name=$CONTAINER_NAME
4973
docker stop -t 2 "$name" || true
5074
}
5175

‎build-vm-podman‎

Lines changed: 35 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,37 @@ vm_verify_options_podman() {
2828
}
2929

3030
vm_startup_podman() {
31-
local name="build_${RECIPEFILE//:/-}"
31+
source $BUILD_ROOT/.build/build.data
32+
local name=$CONTAINER_NAME
3233
podman rm "$name" >/dev/null 2>&1 || true
3334
local podman_opts=
35+
local container_root="/"
36+
local build_command=("$vm_init_script")
37+
local mounts=()
3438
test -n "$VM_TYPE_PRIVILEGED" && podman_opts="--privileged --cap-add=SYS_ADMIN --cap-add=MKNOD"
3539
test -n "$RUN_SHELL" -o -n "$RUN_SHELL_AFTER_FAIL" && podman_opts="$podman_opts -it"
40+
local podman_image="build-scratch:latest"
41+
if test -n "$VM_CONTAINER_IMAGE" ; then
42+
mkdir -p $BUILD_ROOT/home/abuild
43+
# create .build.packages link
44+
rm -rf $BUILD_ROOT/.build.packages
45+
ln -s home/abuild/rpmbuild $BUILD_ROOT/.build.packages
46+
podman_image="$VM_CONTAINER_IMAGE"
47+
container_root="/mnt"
48+
mounts+=("--mount" "type=bind,source=$BUILD_ROOT/home/abuild,destination=/home/abuild")
49+
local build_command=(bash -c "
50+
# Create symlinks for /mnt hidden directories that include build script and data
51+
for item in /mnt/.[^.]*; do
52+
[ -d \"\$item\" ] && ln -sf \"\$item\" \"/\$(basename \"\$item\")\"
53+
done
54+
$vm_init_script
55+
")
56+
fi
57+
mounts+=("--mount" "type=bind,source=$BUILD_ROOT,destination=$container_root")
3658
podman run \
3759
--runtime=runc \
3860
--rm --name "$name" --net=none $podman_opts \
39-
--mount "type=bind,source=$BUILD_ROOT,destination=/" \
40-
"$@" build-scratch:latest "$vm_init_script"
61+
"${mounts[@]}" "$@" $podman_image "${build_command[@]}"
4162
BUILDSTATUS="$?"
4263
test "$BUILDSTATUS" != 255 || BUILDSTATUS=3
4364
cleanup_and_exit "$BUILDSTATUS"
@@ -49,9 +70,15 @@ vm_kill_podman() {
4970
}
5071

5172
vm_fixup_podman() {
52-
# create a scratch image for our build environment
53-
if ! podman image exists build-scratch:latest ; then
54-
echo "FROM scratch" | podman build -t build-scratch:latest -
73+
if test -n "$VM_CONTAINER_IMAGE"; then
74+
if ! podman image exists "$VM_CONTAINER_IMAGE" ; then
75+
cleanup_and_exit 1 "ERROR: Unable to find container preinstallimage ${VM_CONTAINER_IMAGE}"
76+
fi
77+
else
78+
# create a scratch image for our build environment
79+
if ! podman image exists build-scratch:latest ; then
80+
echo "FROM scratch" | podman build -t build-scratch:latest -
81+
fi
5582
fi
5683
}
5784

@@ -80,7 +107,8 @@ vm_sysrq_podman() {
80107
}
81108

82109
vm_wipe_podman() {
83-
local name="build_${RECIPEFILE//:/-}"
110+
source $BUILD_ROOT/.build/build.data
111+
local name=$CONTAINER_NAME
84112
podman rm "$name" >/dev/null 2>&1 || true
85113

86114
echo "Wiping build root: '$BUILD_ROOT'"

‎init_buildsystem‎

Lines changed: 83 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ DLNOSIGNATURE=
6363
CACHE_DIR=
6464
OBSURL=
6565
BUILD_SYSROOT=
66+
CONTAINER=
6667
test -z "$CONFIG_DIR" && CONFIG_DIR="$BUILD_DIR/configs"
6768

6869
# default repository type
@@ -132,6 +133,10 @@ while test -n "$1" ; do
132133
OBSURL="$2"
133134
shift 2
134135
;;
136+
--use-container-preinstallimage)
137+
CONTAINER="$2"
138+
shift 2
139+
;;
135140
*)
136141
break
137142
;;
@@ -226,10 +231,21 @@ unsafe_preinstall_check() {
226231
}
227232

228233
preinstall_image_filter() {
229-
for PKG in "$@" ; do
230-
test -e "$BUILD_ROOT/.preinstall_image/$PKG" && continue
231-
echo $PKG
232-
done
234+
# If we have preinstall image info file, use it to filter packages
235+
if test -n "$PREINSTALL_IMAGE_INFO" -a -f "$PREINSTALL_IMAGE_INFO" ; then
236+
for PKG in "$@" ; do
237+
if grep -q "$PKG$" "$PREINSTALL_IMAGE_INFO" 2>/dev/null ; then
238+
continue
239+
fi
240+
echo $PKG
241+
done
242+
else
243+
# Check unpacked preinstallimage if no info file available
244+
for PKG in "$@" ; do
245+
test -e "$BUILD_ROOT/.preinstall_image/$PKG" && continue
246+
echo $PKG
247+
done
248+
fi
233249
}
234250

235251
# call with cmd [OPTIONS] -- ARGS...
@@ -309,6 +325,27 @@ preinstall_image() {
309325
preinstall_integrate
310326
}
311327

328+
preinstall_container_image() {
329+
check_exit
330+
IMAGE_NAME="${2##*/}"
331+
IMAGE_NAME="${IMAGE_NAME%% \[*}"
332+
IMAGE_TAG=$(echo "$2" | awk -F'[][]' '{print $2}')
333+
echo "Using preinstall image${2:+ $2} as container image"
334+
preinstall_setup
335+
# Check if the image exists
336+
if $CONTAINER images --format "{{.Repository}}:{{.Tag}}" | grep -q "${IMAGE_NAME}:${IMAGE_TAG}"; then
337+
echo "Preinstall container image ${IMAGE_NAME}:${IMAGE_TAG} exists for ${CONTAINER}."
338+
else
339+
echo "Importing preinstall container image ${IMAGE_NAME}:${IMAGE_TAG}"
340+
if ! $CONTAINER import "$BUILD_INIT_CACHE/rpms/$1" "${IMAGE_NAME}:${IMAGE_TAG}"; then
341+
echo "ERROR: failed to import preinstall container image."
342+
cleanup_and_exit 1
343+
fi
344+
fi
345+
export VM_CONTAINER_IMAGE="${IMAGE_NAME}:${IMAGE_TAG}"
346+
echo "VM_CONTAINER_IMAGE='$VM_CONTAINER_IMAGE'" >> $BUILD_ROOT/.build/build.data
347+
}
348+
312349
preinstall() {
313350
local PKG="$1"
314351
check_exit
@@ -757,6 +794,10 @@ else
757794
PREINSTALL_IMAGE_SOURCE="$SRC"
758795
continue
759796
fi
797+
if test "$PKG" = "preinstallimageinfo:" ; then
798+
PREINSTALL_IMAGE_INFO="$SRC"
799+
continue
800+
fi
760801
if test "$PKG" = "runscripts:" ; then
761802
RUNSCRIPTS_SEEN=true
762803
PACKAGES_TO_RUNSCRIPTS=$SRC
@@ -918,29 +959,37 @@ if test ! -e $BUILD_ROOT/installed-pkg -a ! -e $BUILD_ROOT/.build/init_buildsyst
918959
PACKAGES_TO_VMINSTALL_FILTERED="$PACKAGES_TO_VMINSTALL"
919960
rm -f "$BUILD_ROOT/.preinstall_image"/*
920961
if test -n "$PREINSTALL_IMAGE" ; then
921-
preinstall_image "$PREINSTALL_IMAGE" "$PREINSTALL_IMAGE_SOURCE"
962+
if test -n "$CONTAINER"; then
963+
preinstall_container_image "$PREINSTALL_IMAGE" "$PREINSTALL_IMAGE_SOURCE"
964+
else
965+
preinstall_image "$PREINSTALL_IMAGE" "$PREINSTALL_IMAGE_SOURCE"
966+
fi
922967
PACKAGES_TO_PREINSTALL_FILTERED=`preinstall_image_filter $PACKAGES_TO_PREINSTALL_FILTERED`
923968
PACKAGES_TO_VMINSTALL_FILTERED=`preinstall_image_filter $PACKAGES_TO_VMINSTALL_FILTERED`
924969
fi
925970
PACKAGES_TO_PREINSTALL_FILTERED=`reorder $PACKAGES_TO_PREINSTALL_FILTERED`
926971
if test -n "$PREPARE_VM" ; then
927972
PACKAGES_TO_VMINSTALL_FILTERED=`reorder $PACKAGES_TO_VMINSTALL_FILTERED`
928973
fi
929-
progress_setup PACKAGES_TO_PREINSTALL_FILTERED
930-
for PKG in $PACKAGES_TO_PREINSTALL_FILTERED ; do
931-
progress_step PACKAGES_TO_PREINSTALL_FILTERED
932-
preinstall ${PKG##*/}
933-
done
934-
if test -n "$PREPARE_VM" ; then
935-
echo
936-
progress_setup PACKAGES_TO_VMINSTALL_FILTERED
937-
for PKG in $PACKAGES_TO_VMINSTALL_FILTERED ; do
938-
progress_step PACKAGES_TO_VMINSTALL_FILTERED
939-
preinstall ${PKG##*/}
940-
done
974+
# no need to preinstall packages if we are importing them in preinstall container image
975+
if test -z "$VM_CONTAINER_IMAGE"; then
976+
progress_setup PACKAGES_TO_PREINSTALL_FILTERED
977+
for PKG in $PACKAGES_TO_PREINSTALL_FILTERED ; do
978+
progress_step PACKAGES_TO_PREINSTALL_FILTERED
979+
preinstall ${PKG##*/}
980+
done
981+
if test -n "$PREPARE_VM" ; then
982+
echo
983+
progress_setup PACKAGES_TO_VMINSTALL_FILTERED
984+
for PKG in $PACKAGES_TO_VMINSTALL_FILTERED ; do
985+
progress_step PACKAGES_TO_VMINSTALL_FILTERED
986+
preinstall ${PKG##*/}
987+
done
988+
fi
941989
fi
942990
# for reorder
943991
check_exit
992+
mkdir -p "$BUILD_ROOT/etc"
944993
if test -w /root ; then
945994
test -c $BUILD_ROOT/dev/null || create_devs
946995
fi
@@ -961,6 +1010,10 @@ if test ! -e $BUILD_ROOT/installed-pkg -a ! -e $BUILD_ROOT/.build/init_buildsyst
9611010
rm -rf "$BUILD_ROOT/installed-pkg"
9621011
mkdir -p "$BUILD_ROOT/installed-pkg"
9631012
else
1013+
# we still need preinstall image name even if it was already done once
1014+
if test -n "$CONTAINER"; then
1015+
preinstall_container_image "$PREINSTALL_IMAGE" "$PREINSTALL_IMAGE_SOURCE"
1016+
fi
9641017
# preinstall was already done
9651018
if test -z "$PREPARE_VM" ; then
9661019
mount_stuff
@@ -1114,7 +1167,19 @@ for PKG in $MAIN_LIST ; do
11141167
if ! test -e $BUILD_ROOT/.preinstall_image/$PKG ; then
11151168
cleanup_and_exit 1 "Package $PKG is missing from the preinstall image"
11161169
fi
1117-
read PKG_HDRMD5 PKGID < $BUILD_ROOT/.preinstall_image/$PKG
1170+
1171+
# Try to read PKG_HDRMD5 and PKGID from preinstallimage info file first
1172+
if test -n "$PREINSTALL_IMAGE_INFO" -a -f "$PREINSTALL_IMAGE_INFO" ; then
1173+
PKG_INFO=$(grep "$PKG$" "$PREINSTALL_IMAGE_INFO" 2>/dev/null | head -n1)
1174+
if test -n "$PKG_INFO" ; then
1175+
PKG_HDRMD5=$(echo "$PKG_INFO" | awk '{print $1}')
1176+
PKGID=$(echo "$PKG_INFO" | awk '{print $2}')
1177+
fi
1178+
else
1179+
# Fallback to reading from unpacked preinstall image
1180+
read PKG_HDRMD5 PKGID < $BUILD_ROOT/.preinstall_image/$PKG
1181+
fi
1182+
11181183
echo "preinstalled ${PKGID%% *}"
11191184
echo "$PKGID" > $BUILD_ROOT/installed-pkg/$PKG
11201185
continue

0 commit comments

Comments
 (0)