NO-JIRA: chore(workflows): refresh cache when running on-schedule builds and deduplicate make targets in build workflows

Author: jiridanek

.github/workflows/build-notebooks-TEMPLATE.yaml

@@ -135,6 +135,7 @@ jobs:
       # region Free up disk space
 
       - name: Free up additional disk space
+        uses: ./.github/actions/free-up-disk-space
         uses: ./.github/actions/free-up-disk-space
         # https://docs.github.com/en/actions/learn-github-actions/expressions
         # NOTE: the arm64 GitHub hosted runner does not have the /mnt-mounted scratch disk
@@ -192,50 +193,59 @@ jobs:
           fi
           echo "${{ secrets.AIPCC_QUAY_BOT_PASSWORD }}" | podman login quay.io/aipcc -u "${{ secrets.AIPCC_QUAY_BOT_USERNAME }}" --password-stdin
 
-      - name: Compute extra podman build args
-        id: extra-podman-build-args
-        run: |
-          set -Eeuxo pipefail
-
-          EXTRA_PODMAN_BUILD_ARGS=""
-          if [[ "${{ inputs.platform }}" == "linux/s390x" ]]; then
-            # workaround for known issue https://github.com/zeromq/libzmq/pull/4486
-            # In qemu-user, CACHELINE_SIZE probe is undefined
-            # Compiling pyzmq through UV needs this in CFLAGS
-            EXTRA_PODMAN_BUILD_ARGS+='--env=CFLAGS=-Dundefined=64 --env=CXXFLAGS=-Dundefined=64 --unsetenv=CFLAGS --unsetenv=CXXFLAGS'
-          fi
-          echo "EXTRA_PODMAN_BUILD_ARGS=$EXTRA_PODMAN_BUILD_ARGS" >> $GITHUB_OUTPUT
-
       # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push
       # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
 
-      - name: "push|schedule|workflow_dispatch: make ${{ inputs.target }}"
-        id: push-make-target
+      # We use `CONTAINER_BUILD_CACHE_ARGS` to smuggle in various misc `podman build` args, not just cache-related ones
+      - name: Compute CONTAINER_BUILD_CACHE_ARGS
+        id: extra-container-build-args
+        shell: python
+        # language=python
         run: |
-          # print running stats on disk occupancy
-          (while true; do df -h | grep "${HOME}/.local/share/containers"; sleep 30; done) &
-
-          make ${{ inputs.target }}
-        if: ${{ fromJson(inputs.github).event_name == 'push' ||
-          fromJson(inputs.github).event_name == 'schedule' ||
-          fromJson(inputs.github).event_name == 'workflow_dispatch' }}
-        env:
-          IMAGE_TAG: "${{ steps.calculated_vars.outputs.IMAGE_TAG }}"
-          CONTAINER_BUILD_CACHE_ARGS: "${{ steps.extra-podman-build-args.outputs.EXTRA_PODMAN_BUILD_ARGS }} --cache-from ${{ env.CACHE }} --cache-to ${{ env.CACHE }}"
-      - name: "pull_request: make ${{ inputs.target }}"
-        id: pr-make-target
+          import os
+          import sys
+          from typing import Literal
+
+          extra_podman_build_args = ""
+          if "${{ inputs.platform }}" == "linux/s390x":
+             # workaround for known issue https://github.com/zeromq/libzmq/pull/4486
+             # In qemu-user, CACHELINE_SIZE probe is undefined
+             # Compiling pyzmq through UV needs this in CFLAGS
+             extra_podman_build_args += '--env=CFLAGS=-Dundefined=64 --env=CXXFLAGS=-Dundefined=64 --unsetenv=CFLAGS --unsetenv=CXXFLAGS'
+
+          event_name: Literal['push', 'pull_request', 'schedule', 'workflow_dispatch'] = "${{ fromJson(inputs.github).event_name }}"
+          cache = "${{ env.CACHE }}"
+
+          if event_name == "schedule":
+              # For schedule events, skip --cache-from to regenerate cache
+              cache_args = f"--cache-to {cache}"
+              print("Regenerating cache (schedule event)", file=sys.stderr)
+          elif event_name == "push" or event_name == "workflow_dispatch":
+              # push or workflow_dispatch: use and update cache
+              cache_args = f"--cache-from {cache} --cache-to {cache}"
+              print("Using and updating cache", file=sys.stderr)
+          elif event_name == "pull_request":
+              # pull_request: use cache, but we don't have write access, so we can't update even if we wanted to
+              cache_args = f"--cache-from {cache}"
+              print("Using cache", file=sys.stderr)
+          else:
+              raise ValueError(f"Unexpected event name: {event_name}")
+
+          with open(os.environ["GITHUB_OUTPUT"], "a") as f:
+             f.write(f"CONTAINER_BUILD_CACHE_ARGS={extra_podman_build_args} {cache_args}\n")
+
+      - name: "Build: make ${{ inputs.target }}"
+        id: make-target
         run: |
           # print running stats on disk occupancy
           (while true; do df -h | grep "${HOME}/.local/share/containers"; sleep 30; done) &
 
           make ${{ inputs.target }}
-        if: "${{ fromJson(inputs.github).event_name == 'pull_request' ||
-          fromJson(inputs.github).event_name == 'pull_request_target' }}"
         env:
           IMAGE_TAG: "${{ steps.calculated_vars.outputs.IMAGE_TAG }}"
-          CONTAINER_BUILD_CACHE_ARGS: "${{ steps.extra-podman-build-args.outputs.EXTRA_PODMAN_BUILD_ARGS }} --cache-from ${{ env.CACHE }}"
-          # We don't have access to image registry, so disable pushing
-          PUSH_IMAGES: "no"
+          CONTAINER_BUILD_CACHE_ARGS: "${{ steps.extra-container-build-args.outputs.CONTAINER_BUILD_CACHE_ARGS }}"
+          # We don't have access to image registry for PRs, so disable pushing
+          PUSH_IMAGES: "${{ (fromJson(inputs.github).event_name == 'pull_request' || fromJson(inputs.github).event_name == 'pull_request_target') && 'no' || 'yes' }}"
 
       - name: "Show podman images information"
         run: podman images --digests
@@ -429,7 +439,7 @@ jobs:
       # we leave little free disk space after we mount LVM for podman storage
       # not enough to install playwright; running playwright in podman uses the space we have
       - name: Run Playwright tests
-        if: ${{ !cancelled() && contains(inputs.target, 'codeserver') && (steps.push-make-target.outcome == 'success' || steps.pr-make-target.outcome == 'success') }}
+        if: ${{ !cancelled() && contains(inputs.target, 'codeserver') && steps.make-target.outcome == 'success' }}
         # --ipc=host because Microsoft says so in Playwright docs
         # --net=host because testcontainers connects to the Reaper container's exposed port
         # we need to pass through the relevant environment variables