overhaul

Author: gquintard

.github/workflows/packages.yml

@@ -15,7 +15,7 @@ jobs:
       matrix:
         arch:
           - ""
-          - "-arm"
+#          - "-arm"
         target:
           - debian-bullseye-deb
           - debian-bookworm-deb
@@ -30,39 +30,25 @@ jobs:
     steps:
       - name: Checkout Code
         uses: actions/checkout@v5
-        with:
-          submodules: true
-
-      - name: Download Varnish
-        run: |
-          source pkg.env
-          curl -LO https://varnish-cache.org/downloads/varnish-${varnish_version}.tgz
 
       - name: Build Varnish packages
         run: |
-          ( cd varnish-cache/; tar cvzf ../debian.tar.gz debian --dereference ; )
-          ( cd varnish-cache/; tar cvzf ../redhat.tar.gz redhat --dereference ; )
-
-          mkdir -p packages
-
-          if [ -z "${{ matrix.arch }}" ]; then
-            PARAM_ARCH=x86_64
-          else
-            PARAM_ARCH=aarch64
-          fi
           PARAM_DIST=`echo ${{ matrix.target}} | cut -d- -f 1`
           PARAM_RELEASE=`echo ${{ matrix.target }} | cut -d- -f 2`
           EXT=`echo ${{ matrix.target }} | cut -d- -f 3`
 
+          cp pkg.env varnish/
+
           docker run \
             --rm \
             --security-opt seccomp=unconfined \
-            -e PDIR=packages/$PARAM_DIST/$PARAM_RELEASE/$PARAM_ARCH \
-            -v $(pwd):/varnish-cache \
+            --workdir /workdir/varnish \
+            -v$(pwd):/workdir \
+            -e PDIR=/workdir/packages/ \
             $PARAM_DIST:$PARAM_RELEASE \
-            /varnish-cache/build_scripts/make-$EXT-packages.sh
+            /workdir/build_scripts/make-$EXT-packages.sh
 
-          ls packages/*/*/
+          tree packages
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
@@ -79,7 +65,7 @@ jobs:
       matrix:
         arch:
           - ""
-          - "-arm"
+#          - "-arm"
         target:
           - debian-bullseye-deb
           - debian-bookworm-deb
@@ -125,29 +111,22 @@ jobs:
         run: |
           tree $(pwd)/packages
           PACKAGE=${{ matrix.package }}
-          if [ -z "${{ matrix.arch }}" ]; then
-            PARAM_ARCH=x86_64
-          else
-            PARAM_ARCH=aarch64
-          fi
           PARAM_DIST=`echo ${{ matrix.target}} | cut -d- -f 1`
           PARAM_RELEASE=`echo ${{ matrix.target }} | cut -d- -f 2`
           EXT=`echo ${{ matrix.target }} | cut -d- -f 3`
 
           cp pkg.env $PACKAGE/
 
-          PDIR=new_packages/$PARAM_DIST/$PARAM_RELEASE/$PARAM_ARCH
-          mkdir -p $PDIR
-
           docker run \
             --rm \
-            -e PDIR=/$PDIR \
+            -e PDIR=/new_packages/ \
             --workdir /workdir/$PACKAGE \
             -v$(pwd):/workdir \
             -v$(pwd)/packages:/deps \
             -v$(pwd)/new_packages:/new_packages \
             $PARAM_DIST:$PARAM_RELEASE \
-            /workdir/build_scripts/make-$EXT-packages-vmod.sh
+            /workdir/build_scripts/make-$EXT-packages.sh
+        
           tree new_packages
 
       - name: Upload artifacts
@@ -212,3 +191,58 @@ jobs:
         with:
           name: signed-packages
           path: all-packages/
+
+  test_all_packages:
+    name: Load all packages
+    needs: collect_all_packages
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        arch:
+          - ""
+#          - "-arm"
+        target:
+          - debian-bullseye-deb
+          - debian-bookworm-deb
+          - debian-trixie-deb
+          - ubuntu-jammy-deb
+          - ubuntu-noble-deb
+          - ubuntu-plucky-deb
+          - almalinux-8-rpm
+          - almalinux-9-rpm
+          - almalinux-10-rpm
+          - amazonlinux-2023-rpm
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v5
+
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: signed-packages
+
+      - name: Run varnishtest
+        run: |
+          set -x
+
+          pwd
+          tree
+          PARAM_DIST=`echo ${{ matrix.target}} | cut -d- -f 1`
+          PARAM_RELEASE=`echo ${{ matrix.target }} | cut -d- -f 2`
+
+          if [ -z "${{ matrix.arch }}" ]; then
+              PKG_DIR=$(pwd)/$PARAM_DIST/$PARAM_RELEASE/amd64
+          else
+              PKG_DIR=$(pwd)/$PARAM_DIST/$PARAM_RELEASE/arm64
+          fi
+
+          ls $(pwd)/build_scripts/test-packages.sh
+          ls $PKG_DIR
+          docker run \
+            --rm \
+            -w /workdir \
+            -v $PKG_DIR:/workdir \
+            -v $(pwd)/build_scripts:/workdir/build_scripts/ \
+            $PARAM_DIST:$PARAM_RELEASE \
+            //workdir/build_scripts/test-packages.sh

build_scripts/digest.vtc

@@ -0,0 +1,8 @@
+varnishtest "Make sure digest"
+
+varnish v1 -vcl {
+	vcl 4.1;
+
+	import digest;
+	backend default none;
+} -start

build_scripts/main.vtc

@@ -0,0 +1,32 @@
+varnishtest "Make sure most vmods load"
+
+varnish v1 -vcl {
+	vcl 4.1;
+
+	import digest;
+	import fileserver;
+	import jq;
+	import querystring;
+	import redis;
+	import reqwest;
+	import rers;
+	import uuid;
+	import geoip2;
+
+	# varnish-modules
+	import accept;
+	import bodyaccess;
+	import header;
+	import saintmode;
+	import str;
+	import tcp;
+	import var;
+	import vsthrottle;
+	import xkey;
+
+	backend default none;
+
+	sub vcl_init {
+		new re_cache = rers.init(100);
+	}
+} -start

build_scripts/make-deb-packages-vmod.sh

@@ -2,50 +2,77 @@
 
 set -eux
 
+source /etc/os-release
+source ./pkg.env
+
 export DEBIAN_FRONTEND=noninteractive
 export DEBCONF_NONINTERACTIVE_SEEN=true
-apt-get update
-apt-get install -y dpkg-dev debhelper devscripts equivs pkg-config apt-utils fakeroot
-
-# Ubuntu 20.04 aarch64 fails when using fakeroot-sysv with:
-#    semop(1): encountered an error: Function not implemented
-update-alternatives --set fakeroot /usr/bin/fakeroot-tcp
 
-cd /varnish-cache
-ls -la
+PKG_NAME=$(basename $(pwd))
+DEB_ORIG=${VARS[${PKG_NAME}_version]}.orig.tar.gz
+if [ "`uname -m`" = "x86_64" ]; then
+	ARCH="amd64"
+else
+	ARCH="arm64"
+fi
+PDIR="$PDIR/$ID/${VERSION_CODENAME}/${ARCH}"
 
-echo "Untar debian..."
-tar xavf debian.tar.gz
+apt-get update
+apt-get install -y \
+	apt-utils \
+	debhelper \
+	devscripts \
+	dpkg-dev \
+	equivs \
+	fakeroot \
+	pkg-config \
+	python*-docutils \
+	$(test -d /deps && find /deps/ -name '*.deb')
+# Create build folder and copy debian folder there
+mkdir -p ./pkgbuild
+cd pkgbuild
 
-echo "Untar orig..."
-tar xavf varnish-*.tgz --strip 1
+# Save the tarball source file one level up 
+curl -L "${VARS[${PKG_NAME}_source]}" -o ../$DEB_ORIG
+tar xvfz ../$DEB_ORIG --strip 1
 
-echo "Update changelog version..."
-if [ -e .is_weekly ]; then
-    WEEKLY='-weekly'
+# Update changelog version
+if [ "$PKG_NAME" == "varnish" ]; then
+	if [ -e .is_weekly ]; then
+		WEEKLY='-weekly'
+	else
+		WEEKLY=
+	fi
+	VERSION=$(./configure --version | awk 'NR == 1 {print $NF}')$WEEKLY-${package_release}
+	./configure --version
 else
-    WEEKLY=
+	VERSION="$(dpkg -l varnish | awk '$2 == "varnish" {print $3}')"
+	dpkg -l varnish
 fi
-source /etc/os-release
-source ./pkg.env
-VERSION=$(./configure --version | awk 'NR == 1 {print $NF}')$WEEKLY-${package_release}~$VERSION_CODENAME
-sed -i -e "s|@VERSION@|$VERSION|"  "debian/changelog"
 
-echo "Install Build-Depends packages..."
+# remove potential debian/ package included in the tarball
+rm -rf debian
+cat << EOF > ../debian/changelog
+${PKG_NAME} (${VERSION}) unstable; urgency=low
+
+  * Changelog not maintained, please see
+    https://github.com/varnish/all-packager/releases/tag/v${VERSION%~*}
+
+ -- Varnish Software <opensource@varnish-software.com>
+EOF
+ls -halt ../debian
+cp -Lrf ../debian .
+
+cat debian/changelog
+# Install Build-Depends packages
 yes | mk-build-deps --install debian/control || true
 
-echo "Build the packages..."
+# Build the packages
 dpkg-buildpackage -us -uc -j16
 
-echo "Prepare the packages for storage..."
-mkdir -p $PDIR
-mv ../*.deb $PDIR
-
-if [ "`uname -m`" = "x86_64" ]; then
-  ARCH="amd64"
-else
-  ARCH="arm64"
-fi
+# Prepare the packages for storage
+mkdir -p "$PDIR"
+mv ../*.deb "$PDIR"
 
 DSC_FILE=$(ls ../*.dsc)
 DSC_FILE_WO_EXT=$(basename ${DSC_FILE%.*})