diff --git a/build b/build index b1338153..9b7a0931 100755 --- a/build +++ b/build @@ -346,7 +346,7 @@ Known Parameters: --vm-type TYPE Use virtual machine instead of chroot - TYPE is one of xen|kvm|uml|qemu|lxc|zvm|openstack|ec2|docker|pvm|nspawn + TYPE is one of xen|kvm|uml|qemu|lxc|zvm|openstack|ec2|docker|pvm|nspawn|podman --vm-args ARGS Pass extra arguments to virtual machine @@ -426,6 +426,11 @@ Known Parameters: Available options are -net, -netdev, -device (This options in kvm can not guarantee reproducible builds) + --vm-use-container-preinstallimage + Use preinstall container image as base image in containers instead of + using preinstall image tar file. This option is only available when vm-type + is one of podman|docker + --debuginfo Enable creation of debuginfo packages diff --git a/build-vm b/build-vm index cb440bc1..e5cad7b0 100644 --- a/build-vm +++ b/build-vm @@ -35,6 +35,8 @@ VM_TYPE= declare -a VM_ARGS VM_TYPE_PRIVILEGED= VM_TYPE_CONTAINER= +VM_USE_CONTAINER_PREINSTALLIMAGE=false +VM_CONTAINER_IMAGE= VM_ROOT= VM_SWAP= VM_ROOT_TYPE= @@ -305,6 +307,13 @@ vm_parse_options() { VM_OPENSTACK_FLAVOR="$ARG" shift ;; + -vm-use-container-preinstallimage) + if test "$VM_TYPE" == "docker" -o "$VM_TYPE" == "podman"; then + VM_USE_CONTAINER_PREINSTALLIMAGE=true + else + cleanup_and_exit 1 "Option 'vm-use-container-preinstallimage' is not supported" + fi + ;; -*) return 1 ;; @@ -990,11 +999,16 @@ vm_first_stage() { if test "$DO_INIT" = true ; then # do first stage of init_buildsystem rm -f $BUILD_ROOT/.build.success - set -- init_buildsystem --configdir "$CONFIG_DIR" --cachedir "$CACHE_DIR" --prepare "${initbuildsysstuff[@]}" "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$RECIPEPATH" $ADDITIONAL_PACKS + if test "$VM_USE_CONTAINER_PREINSTALLIMAGE" = "true"; then + set -- init_buildsystem --use-container-preinstallimage "$VM_TYPE" --configdir "$CONFIG_DIR" --cachedir "$CACHE_DIR" --prepare "${initbuildsysstuff[@]}" "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$RECIPEPATH" $ADDITIONAL_PACKS + else + set -- init_buildsystem --configdir "$CONFIG_DIR" --cachedir "$CACHE_DIR" --prepare "${initbuildsysstuff[@]}" "${definesnstuff[@]}" "${repos[@]}" $CLEAN_BUILD $USEUSEDFORBUILD $RPMLIST "$RECIPEPATH" $ADDITIONAL_PACKS + fi echo "$* ..." start_time=$SECONDS "$@" || cleanup_and_exit 1 check_exit + test -f "$BUILD_ROOT/.build/build.data" && source "$BUILD_ROOT/.build/build.data" TIME_PREINSTALL=$(( $SECONDS - $start_time )) unset start_time if test ! -w /root ; then @@ -1095,6 +1109,9 @@ vm_first_stage() { echo ")" >> $BUILD_ROOT/.build/build.data echo "VM_TYPE='$VM_TYPE'" >> $BUILD_ROOT/.build/build.data echo "VM_TYPE_CONTAINER='$VM_TYPE_CONTAINER'" >> $BUILD_ROOT/.build/build.data + test -n "$CONTAINER_PREINSTALLIMAGE" && echo "CONTAINER_PREINSTALLIMAGE='$CONTAINER_PREINSTALLIMAGE'" >> $BUILD_ROOT/.build/build.data + test "$VM_TYPE" = "docker" && echo "CONTAINER_NAME='obsbuild.${BUILD_ROOT##*/}.$(date +%s).${RANDOM}'" >> $BUILD_ROOT/.build/build.data + test "$VM_TYPE" = "podman" && echo "CONTAINER_NAME='build_${RECIPEFILE//:/-}.$(date +%s).${RANDOM}'" >> $BUILD_ROOT/.build/build.data echo "VM_NETWORK='$VM_NETWORK'" >> $BUILD_ROOT/.build/build.data echo "RUN_SHELL='$RUN_SHELL'" >> $BUILD_ROOT/.build/build.data echo "RUN_SHELL_AFTER_FAIL='$RUN_SHELL_AFTER_FAIL'" >> $BUILD_ROOT/.build/build.data @@ -1283,6 +1300,11 @@ vm_save_statistics() { # args: resultdirs vm_wrapup_build() { test "$DO_STATISTICS" = 1 && vm_save_statistics + if test "$BUILDTYPE" = "preinstallimage"; then + case "$VM_TYPE" in + docker|podman) preinstallimage_compress "$BUILD_ROOT/$TOPDIR/OTHER" ;; + esac + fi if test -n "$VM_SWAP"; then echo "... saving built packages" swapoff "$VM_SWAP" diff --git a/build-vm-docker b/build-vm-docker index 752ad8e2..d3329289 100644 --- a/build-vm-docker +++ b/build-vm-docker @@ -28,24 +28,48 @@ vm_verify_options_docker() { } vm_startup_docker() { - local name="obsbuild.${BUILD_ROOT##*/}" + source $BUILD_ROOT/.build/build.data + local name=$CONTAINER_NAME docker rm "$name" >/dev/null 2>&1 || true local docker_opts= + local container_root="/mnt" + local build_command=(chroot /mnt "$vm_init_script") + local mounts=("--mount" "type=bind,source=$BUILD_ROOT,destination=/mnt") test -n "$VM_TYPE_PRIVILEGED" && docker_opts="--privileged --cap-add=SYS_ADMIN --cap-add=MKNOD" test -n "$RUN_SHELL" -o -n "$RUN_SHELL_AFTER_FAIL" && docker_opts="$docker_opts -it" + local docker_image="busybox" + if test -n "$VM_CONTAINER_IMAGE" ; then + mkdir -p $BUILD_ROOT/home/abuild + # create .build.packages link + rm -rf $BUILD_ROOT/.build.packages + ln -s home/abuild/rpmbuild $BUILD_ROOT/.build.packages + docker_image="$VM_CONTAINER_IMAGE" + container_root= + mounts+=("--mount" "type=bind,source=$BUILD_ROOT/home/abuild,destination=/home/abuild") + local build_command=(bash -c " + # Create symlinks for /mnt hidden directories that include build script and data + for item in /mnt/.[^.]*; do + [ -d \"\$item\" ] && ln -sf \"\$item\" \"/\$(basename \"\$item\")\" + done + $vm_init_script + ") + fi + mounts+=( + "--mount" "type=bind,source=/proc,destination=$container_root/proc" + "--mount" "type=bind,source=/dev/pts,destination=$container_root/dev/pts" + "--mount" "type=bind,source=/dev/null,destination=$container_root/dev/null" + ) docker run \ --rm --name "$name" --net=none $docker_opts \ - --mount "type=bind,source=$BUILD_ROOT,destination=/mnt" \ - --mount "type=bind,source=/proc,destination=/mnt/proc" \ - --mount "type=bind,source=/dev/pts,destination=/mnt/dev/pts" \ - "$@" busybox chroot /mnt "$vm_init_script" + "${mounts[@]}" "$@" "$docker_image" "${build_command[@]}" BUILDSTATUS="$?" test "$BUILDSTATUS" != 255 || BUILDSTATUS=3 cleanup_and_exit "$BUILDSTATUS" } vm_kill_docker() { - local name="obsbuild.${BUILD_ROOT##*/}" + source $BUILD_ROOT/.build/build.data + local name=$CONTAINER_NAME docker stop -t 2 "$name" || true } diff --git a/build-vm-podman b/build-vm-podman index 76e40c02..d2029f5f 100644 --- a/build-vm-podman +++ b/build-vm-podman @@ -28,16 +28,37 @@ vm_verify_options_podman() { } vm_startup_podman() { - local name="build_${RECIPEFILE//:/-}" + source $BUILD_ROOT/.build/build.data + local name=$CONTAINER_NAME podman rm "$name" >/dev/null 2>&1 || true local podman_opts= + local container_root="/" + local build_command=("$vm_init_script") + local mounts=() test -n "$VM_TYPE_PRIVILEGED" && podman_opts="--privileged --cap-add=SYS_ADMIN --cap-add=MKNOD" test -n "$RUN_SHELL" -o -n "$RUN_SHELL_AFTER_FAIL" && podman_opts="$podman_opts -it" + local podman_image="build-scratch:latest" + if test -n "$VM_CONTAINER_IMAGE" ; then + mkdir -p $BUILD_ROOT/home/abuild + # create .build.packages link + rm -rf $BUILD_ROOT/.build.packages + ln -s home/abuild/rpmbuild $BUILD_ROOT/.build.packages + podman_image="$VM_CONTAINER_IMAGE" + container_root="/mnt" + mounts+=("--mount" "type=bind,source=$BUILD_ROOT/home/abuild,destination=/home/abuild") + local build_command=(bash -c " + # Create symlinks for /mnt hidden directories that include build script and data + for item in /mnt/.[^.]*; do + [ -d \"\$item\" ] && ln -sf \"\$item\" \"/\$(basename \"\$item\")\" + done + $vm_init_script + ") + fi + mounts+=("--mount" "type=bind,source=$BUILD_ROOT,destination=$container_root") podman run \ --runtime=runc \ --rm --name "$name" --net=none $podman_opts \ - --mount "type=bind,source=$BUILD_ROOT,destination=/" \ - "$@" build-scratch:latest "$vm_init_script" + "${mounts[@]}" "$@" $podman_image "${build_command[@]}" BUILDSTATUS="$?" test "$BUILDSTATUS" != 255 || BUILDSTATUS=3 cleanup_and_exit "$BUILDSTATUS" @@ -49,9 +70,15 @@ vm_kill_podman() { } vm_fixup_podman() { - # create a scratch image for our build environment - if ! podman image exists build-scratch:latest ; then - echo "FROM scratch" | podman build -t build-scratch:latest - + if test -n "$VM_CONTAINER_IMAGE"; then + if ! podman image exists "$VM_CONTAINER_IMAGE" ; then + cleanup_and_exit 1 "ERROR: Unable to find container preinstallimage ${VM_CONTAINER_IMAGE}" + fi + else + # create a scratch image for our build environment + if ! podman image exists build-scratch:latest ; then + echo "FROM scratch" | podman build -t build-scratch:latest - + fi fi } @@ -80,7 +107,8 @@ vm_sysrq_podman() { } vm_wipe_podman() { - local name="build_${RECIPEFILE//:/-}" + source $BUILD_ROOT/.build/build.data + local name=$CONTAINER_NAME podman rm "$name" >/dev/null 2>&1 || true echo "Wiping build root: '$BUILD_ROOT'" diff --git a/init_buildsystem b/init_buildsystem index 4baaaff0..c1fb47d6 100755 --- a/init_buildsystem +++ b/init_buildsystem @@ -63,6 +63,7 @@ DLNOSIGNATURE= CACHE_DIR= OBSURL= BUILD_SYSROOT= +CONTAINER= test -z "$CONFIG_DIR" && CONFIG_DIR="$BUILD_DIR/configs" # default repository type @@ -132,6 +133,10 @@ while test -n "$1" ; do OBSURL="$2" shift 2 ;; + --use-container-preinstallimage) + CONTAINER="$2" + shift 2 + ;; *) break ;; @@ -226,10 +231,21 @@ unsafe_preinstall_check() { } preinstall_image_filter() { - for PKG in "$@" ; do - test -e "$BUILD_ROOT/.preinstall_image/$PKG" && continue - echo $PKG - done + # If we have preinstall image info file, use it to filter packages + if test -n "$PREINSTALL_IMAGE_INFO" -a -f "$PREINSTALL_IMAGE_INFO" ; then + for PKG in "$@" ; do + if grep -q "$PKG$" "$PREINSTALL_IMAGE_INFO" 2>/dev/null ; then + continue + fi + echo $PKG + done + else + # Check unpacked preinstallimage if no info file available + for PKG in "$@" ; do + test -e "$BUILD_ROOT/.preinstall_image/$PKG" && continue + echo $PKG + done + fi } # call with cmd [OPTIONS] -- ARGS... @@ -310,6 +326,27 @@ preinstall_image() { preinstall_integrate } +preinstall_container_image() { + check_exit + IMAGE_NAME="${2##*/}" + IMAGE_NAME="${IMAGE_NAME%% \[*}" + IMAGE_TAG=$(echo "$2" | awk -F'[][]' '{print $2}') + echo "Using preinstall image${2:+ $2} as container image" + preinstall_setup + # Check if the image exists + if $CONTAINER images --format "{{.Repository}}:{{.Tag}}" | grep -q "${IMAGE_NAME}:${IMAGE_TAG}"; then + echo "Preinstall container image ${IMAGE_NAME}:${IMAGE_TAG} exists for ${CONTAINER}." + else + echo "Importing preinstall container image ${IMAGE_NAME}:${IMAGE_TAG}" + if ! $CONTAINER import "$BUILD_INIT_CACHE/rpms/$1" "${IMAGE_NAME}:${IMAGE_TAG}"; then + echo "ERROR: failed to import preinstall container image." + cleanup_and_exit 1 + fi + fi + export VM_CONTAINER_IMAGE="${IMAGE_NAME}:${IMAGE_TAG}" + echo "VM_CONTAINER_IMAGE='$VM_CONTAINER_IMAGE'" >> $BUILD_ROOT/.build/build.data +} + preinstall() { local PKG="$1" check_exit @@ -758,6 +795,10 @@ else PREINSTALL_IMAGE_SOURCE="$SRC" continue fi + if test "$PKG" = "preinstallimageinfo:" ; then + PREINSTALL_IMAGE_INFO="$SRC" + continue + fi if test "$PKG" = "runscripts:" ; then RUNSCRIPTS_SEEN=true PACKAGES_TO_RUNSCRIPTS=$SRC @@ -919,7 +960,11 @@ if test ! -e $BUILD_ROOT/installed-pkg -a ! -e $BUILD_ROOT/.build/init_buildsyst PACKAGES_TO_VMINSTALL_FILTERED="$PACKAGES_TO_VMINSTALL" rm -f "$BUILD_ROOT/.preinstall_image"/* if test -n "$PREINSTALL_IMAGE" ; then - preinstall_image "$PREINSTALL_IMAGE" "$PREINSTALL_IMAGE_SOURCE" + if test -n "$CONTAINER"; then + preinstall_container_image "$PREINSTALL_IMAGE" "$PREINSTALL_IMAGE_SOURCE" + else + preinstall_image "$PREINSTALL_IMAGE" "$PREINSTALL_IMAGE_SOURCE" + fi PACKAGES_TO_PREINSTALL_FILTERED=`preinstall_image_filter $PACKAGES_TO_PREINSTALL_FILTERED` PACKAGES_TO_VMINSTALL_FILTERED=`preinstall_image_filter $PACKAGES_TO_VMINSTALL_FILTERED` fi @@ -927,21 +972,25 @@ if test ! -e $BUILD_ROOT/installed-pkg -a ! -e $BUILD_ROOT/.build/init_buildsyst if test -n "$PREPARE_VM" ; then PACKAGES_TO_VMINSTALL_FILTERED=`reorder $PACKAGES_TO_VMINSTALL_FILTERED` fi - progress_setup PACKAGES_TO_PREINSTALL_FILTERED - for PKG in $PACKAGES_TO_PREINSTALL_FILTERED ; do - progress_step PACKAGES_TO_PREINSTALL_FILTERED - preinstall ${PKG##*/} - done - if test -n "$PREPARE_VM" ; then - echo - progress_setup PACKAGES_TO_VMINSTALL_FILTERED - for PKG in $PACKAGES_TO_VMINSTALL_FILTERED ; do - progress_step PACKAGES_TO_VMINSTALL_FILTERED - preinstall ${PKG##*/} - done + # no need to preinstall packages if we are importing them in preinstall container image + if test -z "$VM_CONTAINER_IMAGE"; then + progress_setup PACKAGES_TO_PREINSTALL_FILTERED + for PKG in $PACKAGES_TO_PREINSTALL_FILTERED ; do + progress_step PACKAGES_TO_PREINSTALL_FILTERED + preinstall ${PKG##*/} + done + if test -n "$PREPARE_VM" ; then + echo + progress_setup PACKAGES_TO_VMINSTALL_FILTERED + for PKG in $PACKAGES_TO_VMINSTALL_FILTERED ; do + progress_step PACKAGES_TO_VMINSTALL_FILTERED + preinstall ${PKG##*/} + done + fi fi # for reorder check_exit + mkdir -p "$BUILD_ROOT/etc" if test -w /root ; then test -c $BUILD_ROOT/dev/null || create_devs fi @@ -962,6 +1011,10 @@ if test ! -e $BUILD_ROOT/installed-pkg -a ! -e $BUILD_ROOT/.build/init_buildsyst rm -rf "$BUILD_ROOT/installed-pkg" mkdir -p "$BUILD_ROOT/installed-pkg" else + # we still need preinstall image name even if it was already done once + if test -n "$CONTAINER"; then + preinstall_container_image "$PREINSTALL_IMAGE" "$PREINSTALL_IMAGE_SOURCE" + fi # preinstall was already done if test -z "$PREPARE_VM" ; then mount_stuff @@ -1115,7 +1168,19 @@ for PKG in $MAIN_LIST ; do if ! test -e $BUILD_ROOT/.preinstall_image/$PKG ; then cleanup_and_exit 1 "Package $PKG is missing from the preinstall image" fi - read PKG_HDRMD5 PKGID < $BUILD_ROOT/.preinstall_image/$PKG + + # Try to read PKG_HDRMD5 and PKGID from preinstallimage info file first + if test -n "$PREINSTALL_IMAGE_INFO" -a -f "$PREINSTALL_IMAGE_INFO" ; then + PKG_INFO=$(grep "$PKG$" "$PREINSTALL_IMAGE_INFO" 2>/dev/null | head -n1) + if test -n "$PKG_INFO" ; then + PKG_HDRMD5=$(echo "$PKG_INFO" | awk '{print $1}') + PKGID=$(echo "$PKG_INFO" | awk '{print $2}') + fi + else + # Fallback to reading from unpacked preinstall image + read PKG_HDRMD5 PKGID < $BUILD_ROOT/.preinstall_image/$PKG + fi + echo "preinstalled ${PKGID%% *}" echo "$PKGID" > $BUILD_ROOT/installed-pkg/$PKG continue