Support importing preinstallimage as container image

Implement the option to user preinstallimage as base container image for podman and docker
build environments. This speeds up the build process since there would be no need to untar
the preinstallimage and install the packages inside it. It would only import the preinstallimage
if it doesn't already find it implemented in preinstall_container_image().
For finding the package id and HDRMD5 of packages in preinstallimage it prioritizes the preinstallimageinfo
and if not found it would fallback to previous method of reading unpacked preinstallimage.
Adding randomization for container name in both podman and docker to avoid issues in concurrent builds.

Author: mahdiyehmr

build

@@ -346,7 +346,7 @@ Known Parameters:
 
   --vm-type TYPE
               Use virtual machine instead of chroot
-              TYPE is one of xen|kvm|uml|qemu|lxc|zvm|openstack|ec2|docker|pvm|nspawn
+              TYPE is one of xen|kvm|uml|qemu|lxc|zvm|openstack|ec2|docker|pvm|nspawn|podman
 
   --vm-args ARGS
               Pass extra arguments to virtual machine
@@ -426,6 +426,11 @@ Known Parameters:
               Available options are -net, -netdev, -device
 	      (This options in kvm can not guarantee reproducible builds)
 
+   --vm-use-container-preinstallimage
+              Use preinstall container image as base image in containers instead of
+              using preinstall image tar file. This option is only available when vm-type
+              is one of podman|docker
+
   --debuginfo
               Enable creation of debuginfo packages
 

build-vm

@@ -35,6 +35,8 @@ VM_TYPE=
 declare -a VM_ARGS
 VM_TYPE_PRIVILEGED=
 VM_TYPE_CONTAINER=
+VM_USE_CONTAINER_PREINSTALLIMAGE=false
+VM_CONTAINER_IMAGE=
 VM_ROOT=
 VM_SWAP=
 VM_ROOT_TYPE=
@@ -305,6 +307,13 @@ vm_parse_options() {
         VM_OPENSTACK_FLAVOR="$ARG"
         shift
       ;;
+      -vm-use-container-preinstallimage)
+        if test "$VM_TYPE" == "docker" -o "$VM_TYPE" == "podman"; then
+            VM_USE_CONTAINER_PREINSTALLIMAGE=true
+        else
+            cleanup_and_exit 1 "Option 'vm-use-container-preinstallimage' is not supported"
+        fi
+      ;;
       -*)
 	return 1
       ;;
@@ -990,11 +999,16 @@ vm_first_stage() {
     if test "$DO_INIT" = true ; then
 	# do first stage of init_buildsystem
 	rm -f $BUILD_ROOT/.build.success
-	set -- init_buildsystem --configdir "$CONFIG_DIR" --cachedir "$CACHE_DIR" --prepare "${initbuildsysstuff[@]}" "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$RECIPEPATH" $ADDITIONAL_PACKS
+	if test "$VM_USE_CONTAINER_PREINSTALLIMAGE" = "true"; then
+	    set -- init_buildsystem --use-container-preinstallimage "$VM_TYPE" --configdir "$CONFIG_DIR" --cachedir "$CACHE_DIR" --prepare "${initbuildsysstuff[@]}" "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$RECIPEPATH" $ADDITIONAL_PACKS
+	else
+	    set -- init_buildsystem --configdir "$CONFIG_DIR" --cachedir "$CACHE_DIR" --prepare "${initbuildsysstuff[@]}" "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$RECIPEPATH" $ADDITIONAL_PACKS
+	fi
 	echo "$* ..."
 	start_time=$SECONDS
 	"$@" || cleanup_and_exit 1
 	check_exit
+	test -f "$BUILD_ROOT/.build/build.data" && source "$BUILD_ROOT/.build/build.data"
 	TIME_PREINSTALL=$(( $SECONDS - $start_time ))
 	unset start_time
 	if test ! -w /root ; then
@@ -1095,6 +1109,9 @@ vm_first_stage() {
     echo ")" >> $BUILD_ROOT/.build/build.data
     echo "VM_TYPE='$VM_TYPE'" >> $BUILD_ROOT/.build/build.data
     echo "VM_TYPE_CONTAINER='$VM_TYPE_CONTAINER'" >> $BUILD_ROOT/.build/build.data
+    test -n "$CONTAINER_PREINSTALLIMAGE" && echo "CONTAINER_PREINSTALLIMAGE='$CONTAINER_PREINSTALLIMAGE'" >> $BUILD_ROOT/.build/build.data
+    test "$VM_TYPE" = "docker" && echo "CONTAINER_NAME='obsbuild.${BUILD_ROOT##*/}.$(date +%s).${RANDOM}'" >> $BUILD_ROOT/.build/build.data
+    test "$VM_TYPE" = "podman" && echo "CONTAINER_NAME='build_${RECIPEFILE//:/-}.$(date +%s).${RANDOM}'" >> $BUILD_ROOT/.build/build.data
     echo "VM_NETWORK='$VM_NETWORK'" >> $BUILD_ROOT/.build/build.data
     echo "RUN_SHELL='$RUN_SHELL'" >> $BUILD_ROOT/.build/build.data
     echo "RUN_SHELL_AFTER_FAIL='$RUN_SHELL_AFTER_FAIL'" >> $BUILD_ROOT/.build/build.data
@@ -1283,6 +1300,11 @@ vm_save_statistics() {
 # args: resultdirs
 vm_wrapup_build() {
     test "$DO_STATISTICS" = 1 && vm_save_statistics
+    if test "$BUILDTYPE" = "preinstallimage"; then
+        case "$VM_TYPE" in
+            docker|podman) preinstallimage_compress "$BUILD_ROOT/$TOPDIR/OTHER" ;;
+        esac
+    fi
     if test -n "$VM_SWAP"; then
         echo "... saving built packages"
         swapoff "$VM_SWAP"

build-vm-docker

@@ -28,24 +28,48 @@ vm_verify_options_docker() {
 }
 
 vm_startup_docker() {
-    local name="obsbuild.${BUILD_ROOT##*/}"
+    source $BUILD_ROOT/.build/build.data
+    local name=$CONTAINER_NAME
     docker rm "$name" >/dev/null 2>&1 || true
     local docker_opts=
+    local container_root="/mnt"
+    local build_command=(chroot /mnt "$vm_init_script")
+    local mounts=("--mount" "type=bind,source=$BUILD_ROOT,destination=/mnt")
     test -n "$VM_TYPE_PRIVILEGED" && docker_opts="--privileged --cap-add=SYS_ADMIN --cap-add=MKNOD"
     test -n "$RUN_SHELL" -o -n "$RUN_SHELL_AFTER_FAIL" && docker_opts="$docker_opts -it"
+    local docker_image="busybox"
+    if test -n "$VM_CONTAINER_IMAGE" ; then
+        mkdir -p $BUILD_ROOT/home/abuild
+        # create .build.packages link
+        rm -rf $BUILD_ROOT/.build.packages
+        ln -s home/abuild/rpmbuild $BUILD_ROOT/.build.packages
+        docker_image="$VM_CONTAINER_IMAGE"
+        container_root=
+        mounts+=("--mount" "type=bind,source=$BUILD_ROOT/home/abuild,destination=/home/abuild")
+        local build_command=(bash -c "
+            # Create symlinks for /mnt hidden directories that include build script and data
+            for item in /mnt/.[^.]*; do
+                [ -d \"\$item\" ] && ln -sf \"\$item\" \"/\$(basename \"\$item\")\"
+            done
+            $vm_init_script
+        ")
+    fi
+    mounts+=(
+        "--mount" "type=bind,source=/proc,destination=$container_root/proc"
+        "--mount" "type=bind,source=/dev/pts,destination=$container_root/dev/pts"
+        "--mount" "type=bind,source=/dev/null,destination=$container_root/dev/null"
+    )
     docker run \
         --rm --name "$name" --net=none $docker_opts \
-        --mount "type=bind,source=$BUILD_ROOT,destination=/mnt" \
-        --mount "type=bind,source=/proc,destination=/mnt/proc" \
-        --mount "type=bind,source=/dev/pts,destination=/mnt/dev/pts" \
-        "$@" busybox chroot /mnt "$vm_init_script"
+        "${mounts[@]}" "$@" "$docker_image" "${build_command[@]}"
     BUILDSTATUS="$?"
     test "$BUILDSTATUS" != 255 || BUILDSTATUS=3
     cleanup_and_exit "$BUILDSTATUS"
 }
 
 vm_kill_docker() {
-    local name="obsbuild.${BUILD_ROOT##*/}"
+    source $BUILD_ROOT/.build/build.data
+    local name=$CONTAINER_NAME
     docker stop -t 2 "$name" || true
 }
 

build-vm-podman

@@ -28,16 +28,37 @@ vm_verify_options_podman() {
 }
 
 vm_startup_podman() {
-    local name="build_${RECIPEFILE//:/-}"
+    source $BUILD_ROOT/.build/build.data
+    local name=$CONTAINER_NAME
     podman rm "$name" >/dev/null 2>&1 || true
     local podman_opts=
+    local container_root="/"
+    local build_command=("$vm_init_script")
+    local mounts=()
     test -n "$VM_TYPE_PRIVILEGED" && podman_opts="--privileged --cap-add=SYS_ADMIN --cap-add=MKNOD"
     test -n "$RUN_SHELL" -o -n "$RUN_SHELL_AFTER_FAIL" && podman_opts="$podman_opts -it"
+    local podman_image="build-scratch:latest"
+    if test -n "$VM_CONTAINER_IMAGE" ; then
+        mkdir -p $BUILD_ROOT/home/abuild
+        # create .build.packages link
+        rm -rf $BUILD_ROOT/.build.packages
+        ln -s home/abuild/rpmbuild $BUILD_ROOT/.build.packages
+        podman_image="$VM_CONTAINER_IMAGE"
+        container_root="/mnt"
+        mounts+=("--mount" "type=bind,source=$BUILD_ROOT/home/abuild,destination=/home/abuild")
+        local build_command=(bash -c "
+            # Create symlinks for /mnt hidden directories that include build script and data
+            for item in /mnt/.[^.]*; do
+                [ -d \"\$item\" ] && ln -sf \"\$item\" \"/\$(basename \"\$item\")\"
+            done
+            $vm_init_script
+        ")
+    fi
+    mounts+=("--mount" "type=bind,source=$BUILD_ROOT,destination=$container_root")
     podman run \
         --runtime=runc \
         --rm --name "$name" --net=none $podman_opts \
-        --mount "type=bind,source=$BUILD_ROOT,destination=/" \
-        "$@" build-scratch:latest "$vm_init_script"
+        "${mounts[@]}" "$@" $podman_image "${build_command[@]}"
     BUILDSTATUS="$?"
     test "$BUILDSTATUS" != 255 || BUILDSTATUS=3
     cleanup_and_exit "$BUILDSTATUS"
@@ -49,9 +70,15 @@ vm_kill_podman() {
 }
 
 vm_fixup_podman() {
-    # create a scratch image for our build environment
-    if ! podman image exists build-scratch:latest ; then
-        echo "FROM scratch"  | podman build -t build-scratch:latest  -
+    if  test -n "$VM_CONTAINER_IMAGE"; then
+        if ! podman image exists "$VM_CONTAINER_IMAGE" ; then
+            cleanup_and_exit 1 "ERROR: Unable to find container preinstallimage ${VM_CONTAINER_IMAGE}"
+        fi
+    else
+        # create a scratch image for our build environment
+        if ! podman image exists build-scratch:latest ; then
+            echo "FROM scratch"  | podman build -t build-scratch:latest  -
+        fi
     fi
 }
 
@@ -80,7 +107,8 @@ vm_sysrq_podman() {
 }
 
 vm_wipe_podman() {
-    local name="build_${RECIPEFILE//:/-}"
+    source $BUILD_ROOT/.build/build.data
+    local name=$CONTAINER_NAME
     podman rm "$name" >/dev/null 2>&1 || true
 
     echo "Wiping build root: '$BUILD_ROOT'"

init_buildsystem

@@ -63,6 +63,7 @@ DLNOSIGNATURE=
 CACHE_DIR=
 OBSURL=
 BUILD_SYSROOT=
+CONTAINER=
 test -z "$CONFIG_DIR" && CONFIG_DIR="$BUILD_DIR/configs"
 
 # default repository type
@@ -132,6 +133,10 @@ while test -n "$1" ; do
 	    OBSURL="$2"
 	    shift 2
 	    ;;
+	--use-container-preinstallimage)
+		CONTAINER="$2"
+		shift 2
+		;;
 	*)
 	    break
 	    ;;
@@ -226,10 +231,21 @@ unsafe_preinstall_check() {
 }
 
 preinstall_image_filter() {
-    for PKG in "$@" ; do
-	test -e "$BUILD_ROOT/.preinstall_image/$PKG" && continue
-	echo $PKG
-    done
+    # If we have preinstall image info file, use it to filter packages
+    if test -n "$PREINSTALL_IMAGE_INFO" -a -f "$PREINSTALL_IMAGE_INFO" ; then
+        for PKG in "$@" ; do
+            if grep -q "$PKG$" "$PREINSTALL_IMAGE_INFO" 2>/dev/null ; then
+                continue
+            fi
+            echo $PKG
+        done
+    else
+        # Check unpacked preinstallimage if no info file available
+        for PKG in "$@" ; do
+            test -e "$BUILD_ROOT/.preinstall_image/$PKG" && continue
+            echo $PKG
+        done
+    fi
 }
 
 # call with cmd [OPTIONS] -- ARGS...
@@ -310,6 +326,27 @@ preinstall_image() {
     preinstall_integrate
 }
 
+preinstall_container_image() {
+	check_exit
+	IMAGE_NAME="${2##*/}"
+	IMAGE_NAME="${IMAGE_NAME%% \[*}"
+	IMAGE_TAG=$(echo "$2" | awk -F'[][]' '{print $2}')
+	echo "Using preinstall image${2:+ $2} as container image"
+	preinstall_setup
+	# Check if the image exists
+	if $CONTAINER images --format "{{.Repository}}:{{.Tag}}" | grep -q "${IMAGE_NAME}:${IMAGE_TAG}"; then
+		echo "Preinstall container image ${IMAGE_NAME}:${IMAGE_TAG} exists for ${CONTAINER}."
+	else
+		echo "Importing preinstall container image ${IMAGE_NAME}:${IMAGE_TAG}"
+		if ! $CONTAINER import "$BUILD_INIT_CACHE/rpms/$1" "${IMAGE_NAME}:${IMAGE_TAG}"; then
+			echo "ERROR: failed to import preinstall container image."
+			cleanup_and_exit 1
+		fi
+	fi
+	export VM_CONTAINER_IMAGE="${IMAGE_NAME}:${IMAGE_TAG}"
+	echo "VM_CONTAINER_IMAGE='$VM_CONTAINER_IMAGE'" >> $BUILD_ROOT/.build/build.data
+}
+
 preinstall() {
     local PKG="$1"
     check_exit
@@ -758,6 +795,10 @@ else
 	    PREINSTALL_IMAGE_SOURCE="$SRC"
 	    continue
 	fi
+	if test "$PKG" = "preinstallimageinfo:" ; then
+	    PREINSTALL_IMAGE_INFO="$SRC"
+	    continue
+	fi
 	if test "$PKG" = "runscripts:" ; then
 	    RUNSCRIPTS_SEEN=true
 	    PACKAGES_TO_RUNSCRIPTS=$SRC
@@ -919,29 +960,37 @@ if test ! -e $BUILD_ROOT/installed-pkg -a ! -e $BUILD_ROOT/.build/init_buildsyst
     PACKAGES_TO_VMINSTALL_FILTERED="$PACKAGES_TO_VMINSTALL"
     rm -f "$BUILD_ROOT/.preinstall_image"/*
     if test -n "$PREINSTALL_IMAGE" ; then
-	preinstall_image "$PREINSTALL_IMAGE" "$PREINSTALL_IMAGE_SOURCE"
+	if test -n "$CONTAINER"; then
+		preinstall_container_image "$PREINSTALL_IMAGE" "$PREINSTALL_IMAGE_SOURCE"
+	else
+		preinstall_image "$PREINSTALL_IMAGE" "$PREINSTALL_IMAGE_SOURCE"
+	fi
 	PACKAGES_TO_PREINSTALL_FILTERED=`preinstall_image_filter $PACKAGES_TO_PREINSTALL_FILTERED`
 	PACKAGES_TO_VMINSTALL_FILTERED=`preinstall_image_filter $PACKAGES_TO_VMINSTALL_FILTERED`
     fi
     PACKAGES_TO_PREINSTALL_FILTERED=`reorder $PACKAGES_TO_PREINSTALL_FILTERED`
     if test -n "$PREPARE_VM" ; then
         PACKAGES_TO_VMINSTALL_FILTERED=`reorder $PACKAGES_TO_VMINSTALL_FILTERED`
     fi
-    progress_setup PACKAGES_TO_PREINSTALL_FILTERED
-    for PKG in $PACKAGES_TO_PREINSTALL_FILTERED ; do
-	progress_step PACKAGES_TO_PREINSTALL_FILTERED
-	preinstall ${PKG##*/}
-    done
-    if test -n "$PREPARE_VM" ; then
-	echo
-	progress_setup PACKAGES_TO_VMINSTALL_FILTERED
-	for PKG in $PACKAGES_TO_VMINSTALL_FILTERED ; do
-	    progress_step PACKAGES_TO_VMINSTALL_FILTERED
-	    preinstall ${PKG##*/}
-	done
+    # no need to preinstall packages if we are importing them in preinstall container image
+    if test -z "$VM_CONTAINER_IMAGE"; then
+	    progress_setup PACKAGES_TO_PREINSTALL_FILTERED
+	    for PKG in $PACKAGES_TO_PREINSTALL_FILTERED ; do
+	        progress_step PACKAGES_TO_PREINSTALL_FILTERED
+	        preinstall ${PKG##*/}
+	    done
+	    if test -n "$PREPARE_VM" ; then
+	        echo
+	        progress_setup PACKAGES_TO_VMINSTALL_FILTERED
+	        for PKG in $PACKAGES_TO_VMINSTALL_FILTERED ; do
+		        progress_step PACKAGES_TO_VMINSTALL_FILTERED
+		        preinstall ${PKG##*/}
+	        done
+	    fi
     fi
     # for reorder
     check_exit
+    mkdir -p "$BUILD_ROOT/etc"
     if test -w /root ; then
 	test -c $BUILD_ROOT/dev/null || create_devs
     fi
@@ -962,6 +1011,10 @@ if test ! -e $BUILD_ROOT/installed-pkg -a ! -e $BUILD_ROOT/.build/init_buildsyst
     rm -rf "$BUILD_ROOT/installed-pkg"
     mkdir -p "$BUILD_ROOT/installed-pkg"
 else
+    # we still need preinstall image name even if it was already done once
+    if test -n "$CONTAINER"; then
+        preinstall_container_image "$PREINSTALL_IMAGE" "$PREINSTALL_IMAGE_SOURCE"
+    fi
     # preinstall was already done
     if test -z "$PREPARE_VM" ; then
 	mount_stuff
@@ -1115,7 +1168,19 @@ for PKG in $MAIN_LIST ; do
 	if ! test -e $BUILD_ROOT/.preinstall_image/$PKG ; then
 	    cleanup_and_exit 1 "Package $PKG is missing from the preinstall image"
 	fi
-	read PKG_HDRMD5 PKGID < $BUILD_ROOT/.preinstall_image/$PKG
+
+	# Try to read PKG_HDRMD5 and PKGID from preinstallimage info file first
+	if test -n "$PREINSTALL_IMAGE_INFO" -a -f "$PREINSTALL_IMAGE_INFO" ; then
+	    PKG_INFO=$(grep "$PKG$" "$PREINSTALL_IMAGE_INFO" 2>/dev/null | head -n1)
+	    if test -n "$PKG_INFO" ; then
+	        PKG_HDRMD5=$(echo "$PKG_INFO" | awk '{print $1}')
+	        PKGID=$(echo "$PKG_INFO" | awk '{print $2}')
+	    fi
+	else
+	    # Fallback to reading from unpacked preinstall image
+	    read PKG_HDRMD5 PKGID < $BUILD_ROOT/.preinstall_image/$PKG
+	fi
+
 	echo "preinstalled ${PKGID%% *}"
 	echo "$PKGID" > $BUILD_ROOT/installed-pkg/$PKG
 	continue