CI-rpuu.yml

name: Update raw.pixls.us masterset cache management

on:
  workflow_call:
    outputs:
      rpuu-cache-key:
        value: ${{ jobs.linux.outputs.rpuu-cache-key }}
  schedule:
    - cron: "30 0 * * SUN"
  workflow_dispatch:

concurrency:
  group: rpuu-${{ github.head_ref || github.run_id }}
  cancel-in-progress: false

env:
  RPUU_DST: ${{ github.workspace }}/raw-camera-samples/raw.pixls.us-unique
  CACHE_WARMUP_ONLY: ${{ github.workflow != 'Update raw.pixls.us masterset cache management' }}

jobs:
  linux:
    runs-on: ubuntu-latest
    name: Cache RPU masterset
    timeout-minutes: 3
    container:
      image: "debian:bookworm-slim"
      options: --tmpfs /tmp:exec
    steps:
      - name: Configure APT
        timeout-minutes: 1
        run: |
          set -xe
          tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io > /dev/null <<EOT
          force-unsafe-io
          EOT
          tee /etc/apt/apt.conf.d/tmpfs > /dev/null <<EOT
          Dir::Cache::Archives "/tmp/apt/archives";
          APT::ExtractTemplates::TempDir "/tmp/apt/temp";
          EOT
          mkdir -p /tmp/apt/archives
          tee /etc/apt/apt.conf.d/80retry > /dev/null <<EOT
          Acquire::Retries "10";
          EOT
          tee /etc/apt/apt.conf.d/80recommends > /dev/null <<EOT
          APT::Install-Recommends "false";
          EOT
          tee /etc/apt/apt.conf.d/80suggests > /dev/null <<EOT
          APT::Install-Suggests "false";
          EOT
          tee /etc/apt/apt.conf.d/80forceyes > /dev/null <<EOT
          APT::Get::Assume-Yes "true";
          EOT
          tee /etc/apt/apt.conf.d/80fixmissing > /dev/null <<EOT
          APT::Get::Fix-Missing "true";
          EOT
          rm -rf /etc/apt/sources.list*
          tee /etc/apt/sources.list > /dev/null <<EOT
          deb http://debian-archive.trafficmanager.net/debian bookworm main
          deb http://debian-archive.trafficmanager.net/debian bookworm-updates main
          deb http://debian-archive.trafficmanager.net/debian-security bookworm-security main
          deb http://debian-archive.trafficmanager.net/debian bookworm-backports main
          EOT
      - name: Update base packages
        timeout-minutes: 1
        env:
          DEBIAN_FRONTEND: noninteractive
        run: |
          set -xe
          rm -rf /var/lib/apt/lists/*
          apt update
          apt install eatmydata
          eatmydata apt upgrade
      - name: Install necessary packages
        timeout-minutes: 1
        env:
          DEBIAN_FRONTEND: noninteractive
        run: |
          set -xe
          eatmydata apt install ca-certificates wget git git-annex zstd
          git config --global user.email "bot@local"
      - name: Check raw.pixls.us masterset cache availability
        timeout-minutes: 1
        id: cache-restore
        uses: actions/cache/restore@v5
        with:
          path: ${{ env.RPUU_DST }}
          key: "raw.pixls.us-masterset"
          fail-on-cache-miss: false
          lookup-only: ${{ env.CACHE_WARMUP_ONLY == 'true' }}
      - name: Fetch raw.pixls.us masterset
        timeout-minutes: 3
        id: cache-get
        if: steps.cache-restore.outputs.cache-matched-key == ''
        env:
          RPUU_DST: ${{ env.RPUU_DST }}
        run: |
          set -xe
          git clone https://raw.pixls.us/data-unique.annex.git ${RPUU_DST}
          cd ${RPUU_DST}
          git annex get --jobs cpus .
          sha256sum -c --strict filelist.sha256
      - name: Update raw.pixls.us masterset
        timeout-minutes: 3
        id: cache-update
        if: env.CACHE_WARMUP_ONLY != 'true' && steps.cache-restore.outputs.cache-matched-key != ''
        env:
          RPUU_DST: ${{ env.RPUU_DST }}
        run: |
          set -xe
          cd ${RPUU_DST}
          git fetch --all --prune
          git branch -f git-annex origin/git-annex
          git reset --hard origin/master
          git reflog expire --all --expire all
          git annex get --jobs cpus .
          git annex unused
          git-annex dropunused all --force
          git repack -a -d -f -F --depth=4095 --window=1000000
          sha256sum -c --strict filelist.sha256
      - name: Get the raw.pixls.us masterset cache key
        timeout-minutes: 1
        env:
          RPUU_DST: ${{ env.RPUU_DST }}
        id: memoize-key
        run: |
          set -xe
          if [ -f "${RPUU_DST}/timestamp.txt" ]; then
          echo "rpuu-cache-key=raw.pixls.us-masterset-$(cat ${RPUU_DST}/timestamp.txt)" >> $GITHUB_OUTPUT
          else
          echo "rpuu-cache-key=${{ steps.cache-restore.outputs.cache-matched-key }}" >> $GITHUB_OUTPUT
          fi
      - name: Save updated raw.pixls.us masterset cache
        if: ((steps.cache-get.outcome == 'success') || (steps.cache-update.outcome == 'success')) && (steps.memoize-key.outputs.rpuu-cache-key != steps.cache-restore.outputs.cache-matched-key)
        timeout-minutes: 1
        id: cache-save
        uses: actions/cache/save@v5
        with:
          path: ${{ env.RPUU_DST }}
          key: ${{ steps.memoize-key.outputs.rpuu-cache-key }}
    outputs:
      rpuu-cache-key: ${{ steps.memoize-key.outputs.rpuu-cache-key }}