Make it possible to build simple VM images as part of the CheriBSD release process

Jenkinsfile

@@ -184,7 +184,7 @@ def buildRelease(params, String suffix) {
                // params.extraArgs includes --install-prefix=/rootfs but we
                // don't want the release media to end up there, so override it
                // to the top-level output directory.
-               script: "./cheribuild/jenkins-cheri-build.py --build cheribsd-release-${suffix} --cheribsd-release/install-dir=\${WORKSPACE}/tarball ${params.extraArgs}"
+               script: "./cheribuild/jenkins-cheri-build.py --build cheribsd-release-${suffix} --cheribsd-release/install-dir=\${WORKSPACE}/tarball --cheribsd-release/build-vm-images ${params.extraArgs}"
         }
     }
 }

release/Makefile

@@ -7,14 +7,16 @@
 #  memstick: Builds memory stick image (memstick.img)
 #  mini-memstick: Builds minimal memory stick image (mini-memstick.img)
 #  ftp: Sets up FTP distribution area (ftp)
-#  release: Invokes real-release, vm-release, and cloudware-release targets
+#  release: Invokes real-release, vm-release, cloudware-release and oci-release targets
 #  real-release: Build all media and FTP distribution area
 #  vm-release: Build all virtual machine image targets
 #  cloudware-release: Build all cloud hosting provider targets
-#  install: Invokes the release-install and vm-install targets
+#  oci-release: Build all OCI container images
+#  install: Invokes the release-install, vm-install and oci-install targets
 #  release-install: Copies all release installation media into ${DESTDIR}
 #  vm-install: Copies all virtual machine images into ${DESTDIR}
 #  cloud-install: Copies non-uploaded cloud images into ${DESTDIR}
+#  oci-install: Copies all OCI container images into ${DESTDIR}
 #
 # Variables affecting the build process:
 #  WORLDDIR: location of src tree -- must have built world and default kernel
@@ -118,26 +120,25 @@ clean: beforeclean
 
 base.txz:
 	mkdir -p ${DISTDIR}
-	( cd ${WORLDDIR} && ${IMAKE} distributeworld DISTDIR=${.OBJDIR}/${DISTDIR} )
+	( cd ${WORLDDIR} && ${IMAKE} -DNO_ROOT distributeworld DISTDIR=${.OBJDIR}/${DISTDIR} )
 	# Bootstrap etcupdate(8) database.
 	sh ${WORLDDIR}/usr.sbin/etcupdate/etcupdate.sh extract -B \
 	    -m "${MAKE}" -M "TARGET_ARCH=${TARGET_ARCH} TARGET=${TARGET}" \
 	    -s ${WORLDDIR} -d "${.OBJDIR}/${DISTDIR}/base/var/db/etcupdate" \
-	    -L /dev/null ${NO_ROOT:D-N}
-.if defined(NO_ROOT)
+	    -L /dev/stdout -N
 	echo "./var/db/etcupdate type=dir uname=root gname=wheel mode=0755" >> ${.OBJDIR}/${DISTDIR}/base.meta
-	sed -n 's,^\.,./var/db/etcupdate/current,p' ${.OBJDIR}/${DISTDIR}/base/var/db/etcupdate/current/METALOG \
-	    >> ${.OBJDIR}/${DISTDIR}/base.meta
+	sed -n 's,^\.,./var/db/etcupdate/current,p' \
+	    ${.OBJDIR}/${DISTDIR}/base/var/db/etcupdate/current/METALOG | \
+	    env -i LC_COLLATE=C sort >> ${.OBJDIR}/${DISTDIR}/base.meta
 	rm ${.OBJDIR}/${DISTDIR}/base/var/db/etcupdate/current/METALOG
-.endif
 # Package all components
-	( cd ${WORLDDIR} && ${IMAKE} packageworld DISTDIR=${.OBJDIR}/${DISTDIR} )
+	( cd ${WORLDDIR} && ${IMAKE} -DNO_ROOT packageworld DISTDIR=${.OBJDIR}/${DISTDIR} )
 	mv ${DISTDIR}/*.txz .
 
 kernel.txz:
 	mkdir -p ${DISTDIR}
-	( cd ${WORLDDIR} && ${IMAKE} distributekernel DISTDIR=${.OBJDIR}/${DISTDIR} )
-	( cd ${WORLDDIR} && ${IMAKE} packagekernel DISTDIR=${.OBJDIR}/${DISTDIR} )
+	( cd ${WORLDDIR} && ${IMAKE} -DNO_ROOT distributekernel DISTDIR=${.OBJDIR}/${DISTDIR} )
+	( cd ${WORLDDIR} && ${IMAKE} -DNO_ROOT packagekernel DISTDIR=${.OBJDIR}/${DISTDIR} )
 	mv ${DISTDIR}/kernel*.txz .
 
 src.txz:
@@ -169,12 +170,12 @@ disc1: packagesystem
 	    -DDB_FROM_SRC )
 # Copy distfiles
 	mkdir -p ${.TARGET}/usr/freebsd-dist
-	for dist in MANIFEST *.txz; \
+	for dist in MANIFEST $$(ls *.txz | grep -v container | grep -vE -- '(${base ${_ALL_libcompats}:L:ts|})-dbg'); \
 	    do cp $${dist} ${.TARGET}/usr/freebsd-dist; \
 	done
 .if defined(NO_ROOT)
 	echo "./usr/freebsd-dist type=dir uname=root gname=wheel mode=0755" >> ${.TARGET}/METALOG
-	for dist in MANIFEST *.txz; \
+	for dist in MANIFEST $$(ls *.txz | grep -v container | grep -vE -- '(${base ${_ALL_libcompats}:L:ts|})-dbg'); \
 	    do echo "./usr/freebsd-dist/$${dist} type=file uname=root gname=wheel mode=0644" >> ${.TARGET}/METALOG; \
 	done
 .endif
@@ -207,14 +208,12 @@ bootonly: packagesystem
 	    MK_TOOLCHAIN=no MK_PROFILE=no \
 	    MK_RESCUE=no MK_DICT=no \
 	    MK_KERNEL_SYMBOLS=no MK_TESTS=no MK_DEBUG_FILES=no \
-	    -DDB_FROM_SRC )
+	    -DDB_FROM_SRC -DNO_ROOT)
 # Copy manifest only (no distfiles) to get checksums
 	mkdir -p ${.TARGET}/usr/freebsd-dist
 	cp MANIFEST ${.TARGET}/usr/freebsd-dist
-.if defined(NO_ROOT)
 	echo "./usr/freebsd-dist type=dir uname=root gname=wheel mode=0755" >> ${.TARGET}/METALOG
 	echo "./usr/freebsd-dist/MANIFEST type=file uname=root gname=wheel mode=0644" >> ${.TARGET}/METALOG
-.endif
 # Set up installation environment
 	ln -fs /tmp/bsdinstall_etc/resolv.conf ${.TARGET}/etc/resolv.conf
 	echo sendmail_enable=\"NONE\" > ${.TARGET}/etc/rc.conf
@@ -224,13 +223,11 @@ bootonly: packagesystem
 	echo kernels_autodetect=\"NO\" >> ${.TARGET}/boot/loader.conf
 	echo loader_menu_multi_user_prompt=\"Installer\" >> ${.TARGET}/boot/loader.conf
 	cp ${.CURDIR}/rc.local ${.TARGET}/etc
-.if defined(NO_ROOT)
 	echo "./etc/resolv.conf type=link uname=root gname=wheel mode=0644 link=/tmp/bsdinstall_etc/resolv.conf" >> ${.TARGET}/METALOG
 	echo "./etc/rc.conf type=file uname=root gname=wheel mode=0644" >> ${.TARGET}/METALOG
 	echo "./etc/sysctl.conf type=file uname=root gname=wheel mode=0644" >> ${.TARGET}/METALOG
 	echo "./boot/loader.conf type=file uname=root gname=wheel mode=0644" >> ${.TARGET}/METALOG
 	echo "./etc/rc.local type=file uname=root gname=wheel mode=0644" >> ${.TARGET}/METALOG
-.endif
 
 dvd: packagesystem
 # Install system
@@ -241,12 +238,12 @@ dvd: packagesystem
 		-DDB_FROM_SRC )
 # Copy distfiles
 	mkdir -p ${.TARGET}/usr/freebsd-dist
-	for dist in MANIFEST *.txz; \
+	for dist in MANIFEST $$(ls *.txz | grep -v container); \
 	    do cp $${dist} ${.TARGET}/usr/freebsd-dist; \
 	done
 .if defined(NO_ROOT)
 	echo "./usr/freebsd-dist type=dir uname=root gname=wheel mode=0755" >> ${.TARGET}/METALOG
-	for dist in MANIFEST *.txz; \
+	for dist in MANIFEST $$(ls *.txz | grep -v container); \
 	    do echo "./usr/freebsd-dist/$${dist} type=file uname=root gname=wheel mode=0644" >> ${.TARGET}/METALOG; \
 	done
 .endif
@@ -284,18 +281,18 @@ dvd1.iso: dvd pkg-stage
 	cd dvd && sh ${.CURDIR}/${TARGET}/mkisoimages.sh -b ${VOLUME_LABEL}_DVD ../${.TARGET} .${NO_ROOT:D/METALOG} ${XTRADIR}
 
 bootonly.iso: bootonly-bootonly
-	cd bootonly-bootonly && sh ${.CURDIR}/${TARGET}/mkisoimages.sh -b ${VOLUME_LABEL}_BO ../${.TARGET} .${NO_ROOT:D/METALOG} ${XTRADIR}
+	cd bootonly-bootonly && sh ${.CURDIR}/${TARGET}/mkisoimages.sh -b ${VOLUME_LABEL}_BO ../${.TARGET} ./METALOG ${XTRADIR}
 
 memstick: memstick.img
 memstick.img: disc1-memstick
 	cd disc1-memstick && sh ${.CURDIR}/${TARGET}/make-memstick.sh .${NO_ROOT:D/METALOG} ../${.TARGET}
 
 mini-memstick: mini-memstick.img
 mini-memstick.img: bootonly-memstick
-	cd bootonly-memstick && sh ${.CURDIR}/${TARGET}/make-memstick.sh .${NO_ROOT:D/METALOG} ../${.TARGET}
+	cd bootonly-memstick && sh ${.CURDIR}/${TARGET}/make-memstick.sh ./METALOG ../${.TARGET}
 
 packagesystem: ${DISTRIBUTIONS}
-	sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST
+	sh ${.CURDIR}/scripts/make-manifest.sh $$(ls *.txz | grep -v container) > MANIFEST
 	touch ${.TARGET}
 
 pkg-stage: dvd
@@ -315,8 +312,9 @@ ftp: packagesystem
 	rm -rf ftp
 	mkdir -p ftp
 	cp *.txz MANIFEST ftp
+	rm -f ftp/container-*.txz
 
-release:	real-release vm-release cloudware-release
+release:	real-release vm-release cloudware-release oci-release
 	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} release-done
 	true
 
@@ -327,7 +325,7 @@ real-release:
 	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} obj
 	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} ${RELEASE_TARGETS}
 
-install:	release-install vm-install .WAIT cloud-install
+install:	release-install vm-install oci-install .WAIT cloud-install
 
 release-install:
 .if defined(DESTDIR) && !empty(DESTDIR)
@@ -347,3 +345,4 @@ release-install:
 
 .include "${.CURDIR}/Makefile.inc1"
 .include "${.CURDIR}/Makefile.vm"
+.include "${.CURDIR}/Makefile.oci"

release/Makefile.azure

@@ -1,6 +1,4 @@
 #
-#
-#
 # Makefile for uploading Microsoft Azure disk images.
 #
 
@@ -18,9 +16,6 @@ AZURE${VAR}!=	grep -E '^AZURE${VAR}' ${AZURE_UPLOAD_CONF} | awk -F '	' '{print $
 SNAPSHOT_DATE!=		date +-${BUILDDATE}
 .endif
 
-START_DATE!=	date -v-1d -I -u
-EXPIRY_DATE!=	date -v+1m -I -u
-
 azure-upload:		${AZURE_UPLOAD_TGTS}
 
 azure-check-depends:
@@ -52,19 +47,16 @@ azure-do-upload-${_FS}:
 		--account-name ${AZURE_ACCOUNT} --account-key ${AZURE_KEY} \
 		--container-name ${AZURE_STORAGE} --type page \
 		--file ${AZURE_IMG_${_FS}} --name ${AZURE_TARGET_${_FS}}.vhd
-	@echo "The disk access URL with shared access signature is:"
+	@echo "The disk has been uploaded to Storage Account, use following information"
+	@echo "to create an image in Compute Gallery:"
 	@echo
-	@echo -n https://${AZURE_ACCOUNT}.blob.core.windows.net/${AZURE_STORAGE}/${AZURE_TARGET_${_FS}}.vhd?
-	@/usr/local/bin/az storage container generate-sas \
-		--account-name ${AZURE_ACCOUNT} --account-key ${AZURE_KEY} \
-		--name ${AZURE_STORAGE} \
-		--permissions lr \
-		--start ${START_DATE} \
-		--expiry ${EXPIRY_DATE} | cut -d '"' -f 2
+	@echo "Storage Account name: ${AZURE_ACCOUNT}"
+	@echo "Container name:       ${AZURE_STORAGE}"
+	@echo "Blob name:            ${AZURE_TARGET_${_FS}}.vhd"
 	@echo
 	@echo "Please go to Microsoft Partner Center to create a new offer and publish it:"
 	@echo
-	@echo "https://partner.microsoft.com/dashboard/commercial-marketplace/overview"
+	@echo "https://partner.microsoft.com/dashboard/marketplace-offers/overview"
 	@echo
 	@echo "After the new offer status is live, you can delete the disk file with:"
 	@echo

release/Makefile.mirrors

@@ -207,13 +207,13 @@ vm-images-stage:
 .    for VMFORMAT in ${${CW}_FORMAT}
 .     for FS in ${${CW}_FSLIST}
 	cd ${RELEASEDIR}/vmimages && \
-		mv ${OSRELEASE}-${CW}.${FS}.${VMFORMAT}.xz \
-		${OSRELEASE}-${CW}-${SNAP_SUFFIX}.${FS}.${VMFORMAT}.xz
-	cp -p ${RELEASEDIR}/vmimages/${OSRELEASE}-${CW}-${SNAP_SUFFIX}.${FS}.${VMFORMAT}.xz \
-		${VM_DIR}/${BUILDDATE}/${OSRELEASE}-${CW}-${SNAP_SUFFIX}.${FS}.${VMFORMAT}.xz
+		mv ${OSRELEASE}-${CW}-${FS}.${VMFORMAT}.xz \
+		${OSRELEASE}-${CW}-${SNAP_SUFFIX}-${FS}.${VMFORMAT}.xz
+	cp -p ${RELEASEDIR}/vmimages/${OSRELEASE}-${CW}-${SNAP_SUFFIX}-${FS}.${VMFORMAT}.xz \
+		${VM_DIR}/${BUILDDATE}/${OSRELEASE}-${CW}-${SNAP_SUFFIX}-${FS}.${VMFORMAT}.xz
 	cd ${VM_DIR}/Latest && \
-		ln -s ../${BUILDDATE}/${OSRELEASE}-${CW}-${SNAP_SUFFIX}.${FS}.${VMFORMAT}.xz \
-		${OSRELEASE}-${CW}.${FS}.${VMFORMAT}.xz
+		ln -s ../${BUILDDATE}/${OSRELEASE}-${CW}-${SNAP_SUFFIX}-${FS}.${VMFORMAT}.xz \
+		${OSRELEASE}-${CW}-${FS}.${VMFORMAT}.xz
 .     endfor
 .    endfor
 .   endif
@@ -245,8 +245,8 @@ vm-images-stage:
 .   if ${CW} == BASIC-CLOUDINIT
 .    for VMFORMAT in ${${CW}_FORMAT}
 .     for FS in ${${CW}_FSLIST}
-	cp -p ${RELEASEDIR}/vmimages/${OSRELEASE}-${CW}.${FS}.${VMFORMAT}.xz \
-		${VM_DIR}/Latest/${OSRELEASE}-${CW}.${FS}.${VMFORMAT}.xz
+	cp -p ${RELEASEDIR}/vmimages/${OSRELEASE}-${CW}-${FS}.${VMFORMAT}.xz \
+		${VM_DIR}/Latest/${OSRELEASE}-${CW}-${FS}.${VMFORMAT}.xz
 .     endfor
 .    endfor
 .   endif
@@ -272,12 +272,12 @@ ci-images-stage:
 .    for FS in ${${CW}_FSLIST}
 	cd ${RELEASEDIR}/ciimages && \
 		mv ${OSRELEASE}-${CW}-${FS}.${VMFORMAT}.xz \
-		${OSRELEASE}-${CW}-${SNAP_SUFFIX}.${FS}.${VMFORMAT}.xz
-	cp -p ${RELEASEDIR}/ciimages/${OSRELEASE}-${CW}-${SNAP_SUFFIX}.${FS}.${VMFORMAT}.xz \
-		${CI_DIR}/${BUILDDATE}/${OSRELEASE}-${CW}-${SNAP_SUFFIX}.${FS}.${VMFORMAT}.xz
+		${OSRELEASE}-${CW}-${SNAP_SUFFIX}-${FS}.${VMFORMAT}.xz
+	cp -p ${RELEASEDIR}/ciimages/${OSRELEASE}-${CW}-${SNAP_SUFFIX}-${FS}.${VMFORMAT}.xz \
+		${CI_DIR}/${BUILDDATE}/${OSRELEASE}-${CW}-${SNAP_SUFFIX}-${FS}.${VMFORMAT}.xz
 	cd ${CI_DIR}/Latest && \
-		ln -s ../${BUILDDATE}/${OSRELEASE}-${CW}-${SNAP_SUFFIX}.${FS}.${VMFORMAT}.xz \
-		${OSRELEASE}-${CW}.${FS}.${VMFORMAT}.xz
+		ln -s ../${BUILDDATE}/${OSRELEASE}-${CW}-${SNAP_SUFFIX}-${FS}.${VMFORMAT}.xz \
+		${OSRELEASE}-${CW}-${FS}.${VMFORMAT}.xz
 .    endfor
 .   endfor
 .  endif
@@ -302,7 +302,7 @@ ci-images-stage:
 .   for VMFORMAT in ${${CW}_FORMAT}
 .    for FS in ${${CW}_FSLIST}
 	cp -p ${RELEASEDIR}/ciimages/${OSRELEASE}-${CW}-${FS}.${VMFORMAT}.xz \
-		${CI_DIR}/Latest/${OSRELEASE}-${CW}.${FS}.${VMFORMAT}.xz
+		${CI_DIR}/Latest/${OSRELEASE}-${CW}-${FS}.${VMFORMAT}.xz
 .    endfor
 .   endfor
 .  endif

release/Makefile.oci

@@ -0,0 +1,34 @@
+#
+#
+#
+# Makefile for building OCI container images.
+#
+
+.if defined(WITH_OCIIMAGES) && !empty(WITH_OCIIMAGES)
+OCI_IMAGES= static dynamic minimal
+.endif
+
+oci-install:
+.if defined(WITH_OCIIMAGES) && !empty(WITH_OCIIMAGES)
+	mkdir -p ${DESTDIR}/ociimages
+. for _IMG in ${OCI_IMAGES}
+	cp -p ${.OBJDIR}/container-image-${_IMG}.txz ${DESTDIR}/ociimages
+. endfor
+.endif
+
+OCI_TARGETS=
+OCI_DEPS_static=
+OCI_DEPS_dynamic= container-image-static.txz
+OCI_DEPS_minimal= container-image-dynamic.txz
+
+.for _IMG in ${OCI_IMAGES}
+OCI_TARGETS+= container-image-${_IMG}.txz
+container-image-${_IMG}.txz: ${OCI_DEPS_${_IMG}}
+	sh ${.CURDIR}/scripts/make-oci-image.sh ${.CURDIR} ${REVISION} ${BRANCH} ${TARGET_ARCH} ${_IMG}
+	skopeo copy \
+		containers-storage:localhost/freebsd${REVISION:R}-${_IMG}:latest \
+		oci-archive:${.OBJDIR}/container-image-${_IMG}.tar:freebsd${REVISION:R}-${_IMG}:${REVISION}-${BRANCH}-${TARGET_ARCH}
+	${XZ_CMD} < ${.OBJDIR}/container-image-${_IMG}.tar > ${.OBJDIR}/container-image-${_IMG}.txz
+.endfor
+
+oci-release: ${OCI_TARGETS}