Support importing preinstallimage as container image

build

@@ -346,7 +346,7 @@ Known Parameters:
 
   --vm-type TYPE
               Use virtual machine instead of chroot
-              TYPE is one of xen|kvm|uml|qemu|lxc|zvm|openstack|ec2|docker|pvm|nspawn
+              TYPE is one of xen|kvm|uml|qemu|lxc|zvm|openstack|ec2|docker|pvm|nspawn|podman
 
   --vm-args ARGS
               Pass extra arguments to virtual machine
@@ -426,6 +426,11 @@ Known Parameters:
               Available options are -net, -netdev, -device
 	      (This options in kvm can not guarantee reproducible builds)
 
+   --vm-use-container-preinstallimage
+              Use preinstall container image as base image in containers instead of
+              using preinstall image tar file. This option is only available when vm-type
+              is one of podman|docker
+
   --debuginfo
               Enable creation of debuginfo packages
 

build-vm

@@ -35,6 +35,8 @@ VM_TYPE=
 declare -a VM_ARGS
 VM_TYPE_PRIVILEGED=
 VM_TYPE_CONTAINER=
+VM_USE_CONTAINER_PREINSTALLIMAGE=false
+VM_CONTAINER_IMAGE=
 VM_ROOT=
 VM_SWAP=
 VM_ROOT_TYPE=
@@ -305,6 +307,13 @@ vm_parse_options() {
         VM_OPENSTACK_FLAVOR="$ARG"
         shift
       ;;
+      -vm-use-container-preinstallimage)
+        if test "$VM_TYPE" == "docker" -o "$VM_TYPE" == "podman"; then
+            VM_USE_CONTAINER_PREINSTALLIMAGE=true
+        else
+            cleanup_and_exit 1 "Option 'vm-use-container-preinstallimage' is not supported"
+        fi
+      ;;
       -*)
 	return 1
       ;;
@@ -990,11 +999,16 @@ vm_first_stage() {
     if test "$DO_INIT" = true ; then
 	# do first stage of init_buildsystem
 	rm -f $BUILD_ROOT/.build.success
-	set -- init_buildsystem --configdir "$CONFIG_DIR" --cachedir "$CACHE_DIR" --prepare "${initbuildsysstuff[@]}" "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$RECIPEPATH" $ADDITIONAL_PACKS
+	if test "$VM_USE_CONTAINER_PREINSTALLIMAGE" = "true"; then
+	    set -- init_buildsystem --use-container-preinstallimage "$VM_TYPE" --configdir "$CONFIG_DIR" --cachedir "$CACHE_DIR" --prepare "${initbuildsysstuff[@]}" "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$RECIPEPATH" $ADDITIONAL_PACKS
+	else
+	    set -- init_buildsystem --configdir "$CONFIG_DIR" --cachedir "$CACHE_DIR" --prepare "${initbuildsysstuff[@]}" "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$RECIPEPATH" $ADDITIONAL_PACKS
+	fi
 	echo "$* ..."
 	start_time=$SECONDS
 	"$@" || cleanup_and_exit 1
 	check_exit
+	test -f "$BUILD_ROOT/.build/build.data" && source "$BUILD_ROOT/.build/build.data"
 	TIME_PREINSTALL=$(( $SECONDS - $start_time ))
 	unset start_time
 	if test ! -w /root ; then
@@ -1095,6 +1109,9 @@ vm_first_stage() {
     echo ")" >> $BUILD_ROOT/.build/build.data
     echo "VM_TYPE='$VM_TYPE'" >> $BUILD_ROOT/.build/build.data
     echo "VM_TYPE_CONTAINER='$VM_TYPE_CONTAINER'" >> $BUILD_ROOT/.build/build.data
+    test -n "$CONTAINER_PREINSTALLIMAGE" && echo "CONTAINER_PREINSTALLIMAGE='$CONTAINER_PREINSTALLIMAGE'" >> $BUILD_ROOT/.build/build.data
+    test "$VM_TYPE" = "docker" && echo "CONTAINER_NAME='obsbuild.${BUILD_ROOT##*/}.$(date +%s).${RANDOM}'" >> $BUILD_ROOT/.build/build.data
+    test "$VM_TYPE" = "podman" && echo "CONTAINER_NAME='build_${RECIPEFILE//:/-}.$(date +%s).${RANDOM}'" >> $BUILD_ROOT/.build/build.data
     echo "VM_NETWORK='$VM_NETWORK'" >> $BUILD_ROOT/.build/build.data
     echo "RUN_SHELL='$RUN_SHELL'" >> $BUILD_ROOT/.build/build.data
     echo "RUN_SHELL_AFTER_FAIL='$RUN_SHELL_AFTER_FAIL'" >> $BUILD_ROOT/.build/build.data
@@ -1283,6 +1300,11 @@ vm_save_statistics() {
 # args: resultdirs
 vm_wrapup_build() {
     test "$DO_STATISTICS" = 1 && vm_save_statistics
+    if test "$BUILDTYPE" = "preinstallimage"; then
+        case "$VM_TYPE" in
+            docker|podman) preinstallimage_compress "$BUILD_ROOT/$TOPDIR/OTHER" ;;
+        esac
+    fi
     if test -n "$VM_SWAP"; then
         echo "... saving built packages"
         swapoff "$VM_SWAP"

build-vm-docker

@@ -28,24 +28,48 @@ vm_verify_options_docker() {
 }
 
 vm_startup_docker() {
-    local name="obsbuild.${BUILD_ROOT##*/}"
+    source $BUILD_ROOT/.build/build.data
+    local name=$CONTAINER_NAME
     docker rm "$name" >/dev/null 2>&1 || true
     local docker_opts=
+    local container_root="/mnt"
+    local build_command=(chroot /mnt "$vm_init_script")
+    local mounts=("--mount" "type=bind,source=$BUILD_ROOT,destination=/mnt")
     test -n "$VM_TYPE_PRIVILEGED" && docker_opts="--privileged --cap-add=SYS_ADMIN --cap-add=MKNOD"
     test -n "$RUN_SHELL" -o -n "$RUN_SHELL_AFTER_FAIL" && docker_opts="$docker_opts -it"
+    local docker_image="busybox"
+    if test -n "$VM_CONTAINER_IMAGE" ; then
+        mkdir -p $BUILD_ROOT/home/abuild
+        # create .build.packages link
+        rm -rf $BUILD_ROOT/.build.packages
+        ln -s home/abuild/rpmbuild $BUILD_ROOT/.build.packages
+        docker_image="$VM_CONTAINER_IMAGE"
+        container_root=
+        mounts+=("--mount" "type=bind,source=$BUILD_ROOT/home/abuild,destination=/home/abuild")
+        local build_command=(bash -c "
+            # Create symlinks for /mnt hidden directories that include build script and data
+            for item in /mnt/.[^.]*; do
+                [ -d \"\$item\" ] && ln -sf \"\$item\" \"/\$(basename \"\$item\")\"
+            done
+            $vm_init_script
+        ")
+    fi
+    mounts+=(
+        "--mount" "type=bind,source=/proc,destination=$container_root/proc"
+        "--mount" "type=bind,source=/dev/pts,destination=$container_root/dev/pts"
+        "--mount" "type=bind,source=/dev/null,destination=$container_root/dev/null"
+    )
     docker run \
         --rm --name "$name" --net=none $docker_opts \
-        --mount "type=bind,source=$BUILD_ROOT,destination=/mnt" \
-        --mount "type=bind,source=/proc,destination=/mnt/proc" \
-        --mount "type=bind,source=/dev/pts,destination=/mnt/dev/pts" \
-        "$@" busybox chroot /mnt "$vm_init_script"
+        "${mounts[@]}" "$@" "$docker_image" "${build_command[@]}"
     BUILDSTATUS="$?"
     test "$BUILDSTATUS" != 255 || BUILDSTATUS=3
     cleanup_and_exit "$BUILDSTATUS"
 }
 
 vm_kill_docker() {
-    local name="obsbuild.${BUILD_ROOT##*/}"
+    source $BUILD_ROOT/.build/build.data
+    local name=$CONTAINER_NAME
     docker stop -t 2 "$name" || true
 }
 

build-vm-podman

@@ -28,16 +28,37 @@ vm_verify_options_podman() {
 }
 
 vm_startup_podman() {
-    local name="build_${RECIPEFILE//:/-}"
+    source $BUILD_ROOT/.build/build.data
+    local name=$CONTAINER_NAME
     podman rm "$name" >/dev/null 2>&1 || true
     local podman_opts=
+    local container_root="/"
+    local build_command=("$vm_init_script")
+    local mounts=()
     test -n "$VM_TYPE_PRIVILEGED" && podman_opts="--privileged --cap-add=SYS_ADMIN --cap-add=MKNOD"
     test -n "$RUN_SHELL" -o -n "$RUN_SHELL_AFTER_FAIL" && podman_opts="$podman_opts -it"
+    local podman_image="build-scratch:latest"
+    if test -n "$VM_CONTAINER_IMAGE" ; then
+        mkdir -p $BUILD_ROOT/home/abuild
+        # create .build.packages link
+        rm -rf $BUILD_ROOT/.build.packages
+        ln -s home/abuild/rpmbuild $BUILD_ROOT/.build.packages
+        podman_image="$VM_CONTAINER_IMAGE"
+        container_root="/mnt"
+        mounts+=("--mount" "type=bind,source=$BUILD_ROOT/home/abuild,destination=/home/abuild")
+        local build_command=(bash -c "
+            # Create symlinks for /mnt hidden directories that include build script and data
+            for item in /mnt/.[^.]*; do
+                [ -d \"\$item\" ] && ln -sf \"\$item\" \"/\$(basename \"\$item\")\"
+            done
+            $vm_init_script
+        ")
+    fi
+    mounts+=("--mount" "type=bind,source=$BUILD_ROOT,destination=$container_root")
     podman run \
         --runtime=runc \
         --rm --name "$name" --net=none $podman_opts \
-        --mount "type=bind,source=$BUILD_ROOT,destination=/" \
-        "$@" build-scratch:latest "$vm_init_script"
+        "${mounts[@]}" "$@" $podman_image "${build_command[@]}"
     BUILDSTATUS="$?"
     test "$BUILDSTATUS" != 255 || BUILDSTATUS=3
     cleanup_and_exit "$BUILDSTATUS"
@@ -49,9 +70,15 @@ vm_kill_podman() {
 }
 
 vm_fixup_podman() {
-    # create a scratch image for our build environment
-    if ! podman image exists build-scratch:latest ; then
-        echo "FROM scratch"  | podman build -t build-scratch:latest  -
+    if  test -n "$VM_CONTAINER_IMAGE"; then
+        if ! podman image exists "$VM_CONTAINER_IMAGE" ; then
+            cleanup_and_exit 1 "ERROR: Unable to find container preinstallimage ${VM_CONTAINER_IMAGE}"
+        fi
+    else
+        # create a scratch image for our build environment
+        if ! podman image exists build-scratch:latest ; then
+            echo "FROM scratch"  | podman build -t build-scratch:latest  -
+        fi
     fi
 }
 
@@ -80,7 +107,8 @@ vm_sysrq_podman() {
 }
 
 vm_wipe_podman() {
-    local name="build_${RECIPEFILE//:/-}"
+    source $BUILD_ROOT/.build/build.data
+    local name=$CONTAINER_NAME
     podman rm "$name" >/dev/null 2>&1 || true
 
     echo "Wiping build root: '$BUILD_ROOT'"

init_buildsystem

@@ -63,6 +63,7 @@ DLNOSIGNATURE=
 CACHE_DIR=
 OBSURL=
 BUILD_SYSROOT=
+CONTAINER=
 test -z "$CONFIG_DIR" && CONFIG_DIR="$BUILD_DIR/configs"
 
 # default repository type
@@ -132,6 +133,10 @@ while test -n "$1" ; do
 	    OBSURL="$2"
 	    shift 2
 	    ;;
+	--use-container-preinstallimage)
+		CONTAINER="$2"
+		shift 2
+		;;
 	*)
 	    break
 	    ;;
@@ -226,10 +231,21 @@ unsafe_preinstall_check() {
 }
 
 preinstall_image_filter() {
-    for PKG in "$@" ; do
-	test -e "$BUILD_ROOT/.preinstall_image/$PKG" && continue
-	echo $PKG
-    done
+    # If we have preinstall image info file, use it to filter packages
+    if test -n "$PREINSTALL_IMAGE_INFO" -a -f "$PREINSTALL_IMAGE_INFO" ; then
+        for PKG in "$@" ; do
+            if grep -q "$PKG$" "$PREINSTALL_IMAGE_INFO" 2>/dev/null ; then
+                continue
+            fi
+            echo $PKG
+        done
+    else
+        # Check unpacked preinstallimage if no info file available
+        for PKG in "$@" ; do
+            test -e "$BUILD_ROOT/.preinstall_image/$PKG" && continue
+            echo $PKG
+        done
+    fi
 }
 
 # call with cmd [OPTIONS] -- ARGS...
@@ -310,6 +326,27 @@ preinstall_image() {
     preinstall_integrate
 }
 
+preinstall_container_image() {
+	check_exit
+	IMAGE_NAME="${2##*/}"
+	IMAGE_NAME="${IMAGE_NAME%% \[*}"
+	IMAGE_TAG=$(echo "$2" | awk -F'[][]' '{print $2}')
+	echo "Using preinstall image${2:+ $2} as container image"
+	preinstall_setup
+	# Check if the image exists
+	if $CONTAINER images --format "{{.Repository}}:{{.Tag}}" | grep -q "${IMAGE_NAME}:${IMAGE_TAG}"; then
+		echo "Preinstall container image ${IMAGE_NAME}:${IMAGE_TAG} exists for ${CONTAINER}."
+	else
+		echo "Importing preinstall container image ${IMAGE_NAME}:${IMAGE_TAG}"
+		if ! $CONTAINER import "$BUILD_INIT_CACHE/rpms/$1" "${IMAGE_NAME}:${IMAGE_TAG}"; then
+			echo "ERROR: failed to import preinstall container image."
+			cleanup_and_exit 1
+		fi
+	fi
+	export VM_CONTAINER_IMAGE="${IMAGE_NAME}:${IMAGE_TAG}"
+	echo "VM_CONTAINER_IMAGE='$VM_CONTAINER_IMAGE'" >> $BUILD_ROOT/.build/build.data
+}
+
 preinstall() {
     local PKG="$1"
     check_exit
@@ -758,6 +795,10 @@ else
 	    PREINSTALL_IMAGE_SOURCE="$SRC"
 	    continue
 	fi
+	if test "$PKG" = "preinstallimageinfo:" ; then
+	    PREINSTALL_IMAGE_INFO="$SRC"
+	    continue
+	fi
 	if test "$PKG" = "runscripts:" ; then
 	    RUNSCRIPTS_SEEN=true
 	    PACKAGES_TO_RUNSCRIPTS=$SRC
@@ -919,29 +960,37 @@ if test ! -e $BUILD_ROOT/installed-pkg -a ! -e $BUILD_ROOT/.build/init_buildsyst
     PACKAGES_TO_VMINSTALL_FILTERED="$PACKAGES_TO_VMINSTALL"
     rm -f "$BUILD_ROOT/.preinstall_image"/*
     if test -n "$PREINSTALL_IMAGE" ; then
-	preinstall_image "$PREINSTALL_IMAGE" "$PREINSTALL_IMAGE_SOURCE"
+	if test -n "$CONTAINER"; then
+		preinstall_container_image "$PREINSTALL_IMAGE" "$PREINSTALL_IMAGE_SOURCE"
+	else
+		preinstall_image "$PREINSTALL_IMAGE" "$PREINSTALL_IMAGE_SOURCE"
+	fi
 	PACKAGES_TO_PREINSTALL_FILTERED=`preinstall_image_filter $PACKAGES_TO_PREINSTALL_FILTERED`
 	PACKAGES_TO_VMINSTALL_FILTERED=`preinstall_image_filter $PACKAGES_TO_VMINSTALL_FILTERED`
     fi
     PACKAGES_TO_PREINSTALL_FILTERED=`reorder $PACKAGES_TO_PREINSTALL_FILTERED`
     if test -n "$PREPARE_VM" ; then
         PACKAGES_TO_VMINSTALL_FILTERED=`reorder $PACKAGES_TO_VMINSTALL_FILTERED`
     fi
-    progress_setup PACKAGES_TO_PREINSTALL_FILTERED
-    for PKG in $PACKAGES_TO_PREINSTALL_FILTERED ; do
-	progress_step PACKAGES_TO_PREINSTALL_FILTERED
-	preinstall ${PKG##*/}
-    done
-    if test -n "$PREPARE_VM" ; then
-	echo
-	progress_setup PACKAGES_TO_VMINSTALL_FILTERED
-	for PKG in $PACKAGES_TO_VMINSTALL_FILTERED ; do
-	    progress_step PACKAGES_TO_VMINSTALL_FILTERED
-	    preinstall ${PKG##*/}
-	done
+    # no need to preinstall packages if we are importing them in preinstall container image
+    if test -z "$VM_CONTAINER_IMAGE"; then
+	    progress_setup PACKAGES_TO_PREINSTALL_FILTERED
+	    for PKG in $PACKAGES_TO_PREINSTALL_FILTERED ; do
+	        progress_step PACKAGES_TO_PREINSTALL_FILTERED
+	        preinstall ${PKG##*/}
+	    done
+	    if test -n "$PREPARE_VM" ; then
+	        echo
+	        progress_setup PACKAGES_TO_VMINSTALL_FILTERED
+	        for PKG in $PACKAGES_TO_VMINSTALL_FILTERED ; do
+		        progress_step PACKAGES_TO_VMINSTALL_FILTERED
+		        preinstall ${PKG##*/}
+	        done
+	    fi
     fi
     # for reorder
     check_exit
+    mkdir -p "$BUILD_ROOT/etc"
     if test -w /root ; then
 	test -c $BUILD_ROOT/dev/null || create_devs
     fi
@@ -962,6 +1011,10 @@ if test ! -e $BUILD_ROOT/installed-pkg -a ! -e $BUILD_ROOT/.build/init_buildsyst
     rm -rf "$BUILD_ROOT/installed-pkg"
     mkdir -p "$BUILD_ROOT/installed-pkg"
 else
+    # we still need preinstall image name even if it was already done once
+    if test -n "$CONTAINER"; then
+        preinstall_container_image "$PREINSTALL_IMAGE" "$PREINSTALL_IMAGE_SOURCE"
+    fi
     # preinstall was already done
     if test -z "$PREPARE_VM" ; then
 	mount_stuff
@@ -1115,7 +1168,19 @@ for PKG in $MAIN_LIST ; do
 	if ! test -e $BUILD_ROOT/.preinstall_image/$PKG ; then
 	    cleanup_and_exit 1 "Package $PKG is missing from the preinstall image"
 	fi
-	read PKG_HDRMD5 PKGID < $BUILD_ROOT/.preinstall_image/$PKG
+
+	# Try to read PKG_HDRMD5 and PKGID from preinstallimage info file first
+	if test -n "$PREINSTALL_IMAGE_INFO" -a -f "$PREINSTALL_IMAGE_INFO" ; then
+	    PKG_INFO=$(grep "$PKG$" "$PREINSTALL_IMAGE_INFO" 2>/dev/null | head -n1)
+	    if test -n "$PKG_INFO" ; then
+	        PKG_HDRMD5=$(echo "$PKG_INFO" | awk '{print $1}')
+	        PKGID=$(echo "$PKG_INFO" | awk '{print $2}')
+	    fi
+	else
+	    # Fallback to reading from unpacked preinstall image
+	    read PKG_HDRMD5 PKGID < $BUILD_ROOT/.preinstall_image/$PKG
+	fi
+
 	echo "preinstalled ${PKGID%% *}"
 	echo "$PKGID" > $BUILD_ROOT/installed-pkg/$PKG
 	continue