Step 2, Generate the project from BaseVM #151
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Step 2, Generate the project from BaseVM" | |
| on: | |
| schedule: | |
| - cron: '20 22 * * *' | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - 'main' | |
| paths: | |
| - 'conf/*' | |
| - '.github/data/*' | |
| - '.github/workflows/generate.yml' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| generate: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| token: ${{ secrets.VM_TOKEN }} | |
| fetch-depth: '2' | |
| # checkout fetches NO tags at any fetch-depth unless asked. Without | |
| # this, `git tag` below returns nothing, LATEST_TAG is empty and | |
| # ${LATEST_TAG:1} stamps package.json's version as "" -- which is | |
| # what every sibling shipped until 2026-08-19. | |
| fetch-tags: true | |
| - name: Preparing env variables | |
| run: | | |
| git switch main || (git fetch --all && git checkout -b main origin/main) | |
| . conf/default.release.conf | |
| echo "DEFAULT_RELEASE=$DEFAULT_RELEASE" >> $GITHUB_ENV | |
| echo "ALL_RELEASES=\"$(ls conf/ | grep -v default | sed 's/.conf//g' | tr '\n' ',' | sed "s/,\$//" | sed 's/,/", "/g')\"" >> $GITHUB_ENV | |
| LATEST_TAG="$(git tag --sort=-v:refname | head -n 1)" | |
| echo "LATEST_TAG=${LATEST_TAG}" >> $GITHUB_ENV | |
| echo "LATEST_VERSION_NUMBER=${LATEST_TAG:1}" >> $GITHUB_ENV | |
| oldHash="$(cat .github/workflows/test.yml | grep -i "uses: ${{ github.repository }}" | head -1 | cut -d @ -f 2)" | |
| echo "OLD_HASH=${oldHash}" >> $GITHUB_ENV | |
| git clone https://github.com/vmactions/base-vm.git | |
| # Collect base-vm commit messages since last generate | |
| BASEVM_HASH_FILE=".github/data/base-vm-hash.txt" | |
| BASEVM_NEW_HASH="$(git -C base-vm rev-parse HEAD)" | |
| if [ -f "$BASEVM_HASH_FILE" ]; then | |
| BASEVM_OLD_HASH="$(cat "$BASEVM_HASH_FILE")" | |
| BASEVM_LOG="$(git -C base-vm log --format="https://github.com/vmactions/base-vm/commit/%H %s" "${BASEVM_OLD_HASH}..${BASEVM_NEW_HASH}" 2>/dev/null || git -C base-vm log --format="https://github.com/vmactions/base-vm/commit/%H %s" -20)" | |
| else | |
| BASEVM_LOG="$(git -C base-vm log --format="https://github.com/vmactions/base-vm/commit/%H %s" -1)" | |
| fi | |
| echo "$BASEVM_NEW_HASH" > "$BASEVM_HASH_FILE" | |
| # Save log for later commit message | |
| echo "$BASEVM_LOG" > /tmp/basevm-log.txt | |
| mkdir -p .github/tpl | |
| cat base-vm/.github/tpl/README.tpl.md >.github/tpl/README.tpl.md | |
| - name: Bake sync methods from the builder release index | |
| run: | | |
| # Each conf/<release>[-<arch>].conf pins a BUILDER_VERSION. That | |
| # builder release publishes releases.json (gendata's index) as a | |
| # release asset, carrying per-release "sync" (VM_SYNC_METHODS, | |
| # comma separated, first = default) and "shutdown" | |
| # (VM_SHUTDOWN_CMD, the in-guest poweroff command used by | |
| # cache-after-prepare). Copy those into the local conf so the | |
| # action and test workflow read them with no run-time network call. | |
| # | |
| # The index is fetched as a RELEASE ASSET at the pinned version -- | |
| # never a branch, never releases/latest -- so the values and the | |
| # image that ships in that release are a matched pair. One fetch | |
| # per distinct BUILDER_VERSION (cached below), not one per release. | |
| os="ghostbsd" | |
| mkdir -p /tmp/relindex | |
| # jq -r hands back the UNESCAPED string, which then lands in a sed | |
| # replacement (where the delimiter, & and backslash are special) and | |
| # inside double quotes in the conf. Today every value is a plain | |
| # command, but escape/reject rather than trust that: | |
| # sed_escape - backslash-escape the sed-special chars | |
| # unsafe - a value with a double quote or a control char would | |
| # emit a conf that breaks when sourced; skip it loudly | |
| sed_escape() { printf '%s' "$1" | sed -e 's/[\\&|]/\\&/g'; } | |
| unsafe() { case "$1" in *'"'*) return 0 ;; esac; return 1; } | |
| for f in conf/*.conf; do | |
| base="$(basename "$f" .conf)" | |
| case "$base" in | |
| default.release) continue ;; | |
| esac | |
| bv="$( . "$f"; printf '%s' "$BUILDER_VERSION" )" | |
| if [ -z "$bv" ]; then | |
| echo "${base}: no BUILDER_VERSION, skip" | |
| continue | |
| fi | |
| url="https://github.com/anyvm-org/${os}-builder/releases/download/v${bv}/releases.json" | |
| idx="/tmp/relindex/${bv}.json" | |
| # Releases cut before the index asset existed (and old pins with | |
| # no tag at all, e.g. openbsd 7.2-7.6) 404 here. The step runs | |
| # under bash -e, so a bare command substitution would abort the | |
| # whole job: tolerate the failure and degrade to an empty index | |
| # -> both fields fall into their "leaving conf as-is" branches, | |
| # exactly the legacy behavior. | |
| if [ ! -f "$idx" ]; then | |
| curl -fsSL "$url" -o "$idx" 2>/dev/null || : > "$idx" | |
| fi | |
| # jq -e fails (non-zero) on null/absent, which `|| true` absorbs. | |
| methods="$(jq -re --arg t "$base" '.releases[] | select(.tag == $t) | .sync' "$idx" 2>/dev/null | tail -n 1 || true)" | |
| if [ -z "$methods" ]; then | |
| echo "${base}: no sync for tag ${base} in ${url}, leaving conf as-is" | |
| elif unsafe "$methods"; then | |
| echo "${base}: sync value contains a quote, refusing to write: ${methods}" | |
| else | |
| if grep -q '^VM_SYNC_METHODS=' "$f"; then | |
| sed -i "s|^VM_SYNC_METHODS=.*|VM_SYNC_METHODS=\"$(sed_escape "$methods")\"|" "$f" | |
| else | |
| printf '\nVM_SYNC_METHODS="%s"\n' "${methods}" >> "$f" | |
| fi | |
| echo "${base}: VM_SYNC_METHODS=\"${methods}\"" | |
| fi | |
| shutcmd="$(jq -re --arg t "$base" '.releases[] | select(.tag == $t) | .shutdown' "$idx" 2>/dev/null | tail -n 1 || true)" | |
| if [ -z "$shutcmd" ]; then | |
| echo "${base}: no shutdown for tag ${base} in ${url}, leaving conf as-is" | |
| elif unsafe "$shutcmd"; then | |
| echo "${base}: shutdown value contains a quote, refusing to write: ${shutcmd}" | |
| else | |
| if grep -q '^VM_SHUTDOWN_CMD=' "$f"; then | |
| sed -i "s|^VM_SHUTDOWN_CMD=.*|VM_SHUTDOWN_CMD=\"$(sed_escape "$shutcmd")\"|" "$f" | |
| else | |
| printf '\nVM_SHUTDOWN_CMD="%s"\n' "${shutcmd}" >> "$f" | |
| fi | |
| echo "${base}: VM_SHUTDOWN_CMD=\"${shutcmd}\"" | |
| fi | |
| done | |
| - name: Compute sync method map and missing-arch excludes | |
| run: | | |
| # Turn the per-conf VM_SYNC_METHODS into three kinds of value for the | |
| # renderer: | |
| # VM_SYNC_METHODS_MAP - {"<arch>":["method",...]} keyed by matrix.arch, | |
| # so test.yml gates each sync job with `if:`. | |
| # VM_TEST_EXCLUDE_MISSING - matrix excludes for release/arch combos | |
| # that have no conf (so they never spawn). | |
| # VM_TEST_EXCLUDE_<METHOD> - matrix excludes for combos whose arch | |
| # does not support that job's sync method | |
| # (so they never spawn either). | |
| . conf/default.release.conf | |
| def="$DEFAULT_RELEASE" | |
| # non-x86_64 arches the test matrix covers (keep in sync with test.tpl.yml) | |
| ARCHES="aarch64 riscv64 powerpc64 sparc64 ppc64le s390x i386 loongarch64 armv7" | |
| # methods (comma list) for an arch suffix, from the default release | |
| # conf; fall back to any release that has that arch. | |
| methods_for() { | |
| a="$1" | |
| if [ -n "$a" ]; then cf="conf/${def}-${a}.conf"; else cf="conf/${def}.conf"; fi | |
| if [ -n "$a" ] && [ ! -e "$cf" ]; then | |
| cf="$(ls conf/*-"$a".conf 2>/dev/null | head -n 1)" | |
| fi | |
| [ -n "$cf" ] && [ -e "$cf" ] && ( . "$cf"; printf '%s' "$VM_SYNC_METHODS" ) | |
| } | |
| # comma list -> JSON array: rsync,scp -> ["rsync","scp"] (empty -> []) | |
| to_json() { | |
| printf '[' | |
| _first=1 | |
| _old_ifs="$IFS"; IFS=',' | |
| for _m in $1; do | |
| _m="$(printf '%s' "$_m" | tr -d ' ')" | |
| [ -z "$_m" ] && continue | |
| [ "$_first" = 1 ] || printf ',' | |
| printf '"%s"' "$_m" | |
| _first=0 | |
| done | |
| IFS="$_old_ifs" | |
| printf ']' | |
| } | |
| map="{\"x86_64\":$(to_json "$(methods_for '')")" | |
| for a in $ARCHES; do | |
| map="${map},\"${a}\":$(to_json "$(methods_for "$a")")" | |
| done | |
| printf '%s}' "$map" > .github/data/sync-map.json | |
| echo "sync-map: $(cat .github/data/sync-map.json)" | |
| : > .github/data/exclude-missing.txt | |
| for r in $(sed 's/[",]/ /g' conf/test.releases) ""; do | |
| rc="$r"; [ -z "$rc" ] && rc="$def" | |
| for a in $ARCHES; do | |
| if [ ! -e "conf/${rc}-${a}.conf" ]; then | |
| printf ' - release: "%s"\n arch: %s\n' "$r" "$a" >> .github/data/exclude-missing.txt | |
| fi | |
| done | |
| done | |
| echo "exclude-missing:"; cat .github/data/exclude-missing.txt | |
| # Per-method excludes. A sync job's leg is pointless when that arch's | |
| # VM_SYNC_METHODS does not list the job's method: every step in it is | |
| # gated off by the same map, so the leg burns a runner slot and a | |
| # checkout to produce a green job that did nothing. Read from the very | |
| # same methods_for() as sync-map.json above, so an exclude and a | |
| # step-level `if:` can never disagree. | |
| # | |
| # Combos with no conf are skipped -- exclude-missing.txt already | |
| # covers them. x86_64 is the exception: it has no entry there (the | |
| # loop above walks $ARCHES, which is suffix arches only), so it is | |
| # checked unconditionally here, which also drops the dead x86_64 leg | |
| # on guests that ship no conf/<release>.conf at all (methods_for | |
| # returns empty for a missing conf -> supports nothing -> excluded). | |
| SYNC_JOB_METHODS="sshfs nfs scp rsync" | |
| for meth in $SYNC_JOB_METHODS; do | |
| : > ".github/data/exclude-${meth}.txt" | |
| done | |
| for r in $(sed 's/[",]/ /g' conf/test.releases) ""; do | |
| rc="$r"; [ -z "$rc" ] && rc="$def" | |
| for a in "" $ARCHES; do | |
| if [ -n "$a" ] && [ ! -e "conf/${rc}-${a}.conf" ]; then continue; fi | |
| have=",$(methods_for "$a" | tr -d ' ')," | |
| for meth in $SYNC_JOB_METHODS; do | |
| case "$have" in | |
| *",${meth},"*) ;; | |
| *) printf ' - release: "%s"\n arch: "%s"\n' \ | |
| "$r" "$a" >> ".github/data/exclude-${meth}.txt" ;; | |
| esac | |
| done | |
| done | |
| done | |
| for meth in $SYNC_JOB_METHODS; do | |
| echo "exclude-${meth}:"; cat ".github/data/exclude-${meth}.txt" | |
| done | |
| # expose them all to template-render via the datafile (idempotent) | |
| df=".github/data/datafile.ini" | |
| grep -q '^VM_SYNC_METHODS_MAP=' "$df" || printf '\nVM_SYNC_METHODS_MAP=@.github/data/sync-map.json\n' >> "$df" | |
| grep -q '^VM_TEST_EXCLUDE_MISSING=' "$df" || printf '\nVM_TEST_EXCLUDE_MISSING=@.github/data/exclude-missing.txt\n' >> "$df" | |
| for meth in $SYNC_JOB_METHODS; do | |
| v="VM_TEST_EXCLUDE_$(printf '%s' "$meth" | tr 'a-z' 'A-Z')" | |
| grep -q "^${v}=" "$df" || printf '\n%s=@.github/data/exclude-%s.txt\n' "$v" "$meth" >> "$df" | |
| done | |
| - name: Generate files | |
| uses: anyvm-org/template-render@v0.0.4 | |
| with: | |
| datafile: .github/data/datafile.ini | |
| files: | | |
| base-vm/.github/tpl/test.tpl.yml : .github/workflows/test.yml | |
| base-vm/.github/tpl/manual.tpl.yml : .github/workflows/manual.yml | |
| base-vm/.github/tpl/bump.tpl.yml : .github/workflows/bump.yml | |
| base-vm/.github/FUNDING.yml : .github/FUNDING.yml | |
| base-vm/.github/workflows/readme.yml : .github/workflows/readme.yml | |
| base-vm/.github/workflows/major.yml : .github/workflows/major.yml | |
| base-vm/package.json : package.json | |
| base-vm/index.js : index.js | |
| base-vm/action.yml : action.yml | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v7 | |
| with: | |
| node-version: '24' | |
| - name: Update node_modules | |
| run: | | |
| node -v | |
| [ -e "node_modules" ] && git rm -r node_modules | |
| npm install --save | |
| git add node_modules | |
| - name: Check modifications | |
| run: | | |
| #if only hash id in test.yml changes, skip | |
| currentHash="$(cat .github/workflows/test.yml | grep -i "uses: ${{ github.repository }}" | head -1 | cut -d @ -f 2)" | |
| echo "Current hash: $currentHash" | |
| # On a brand-new repo with a single commit, HEAD~1 does not exist. | |
| # Under `bash -e` that aborts the step (exit 128), so fall back to | |
| # HEAD. --verify --quiet matters: a plain `git rev-parse HEAD~1` | |
| # echoes the literal string "HEAD~1" to stdout even when the rev is | |
| # missing, so the fallback would CONCATENATE ("HEAD~1\n<sha>") and | |
| # the newline then breaks the sed below with "unterminated s command" | |
| # (hit on nextbsd-vm's very first push). | |
| lastHash="$(git rev-parse --verify --quiet 'HEAD~1' 2>/dev/null || git rev-parse HEAD)" | |
| echo "Last hash: $lastHash" | |
| oldHash="$lastHash" | |
| echo "OLD_HASH: $OLD_HASH" | |
| echo "Old hash using last hash: $oldHash" | |
| git diff | |
| cp .github/workflows/test.yml test.yml.back | |
| sed -i "s/$currentHash/$oldHash/g" .github/workflows/test.yml | |
| if git diff --quiet; then | |
| echo "no changes" | |
| else | |
| #has changes: | |
| rm -f test.yml.back | |
| rm -rf base-vm | |
| #commit all changes and get new hash | |
| git config user.email "ghactions@vmactions.org" | |
| git config user.name "Generate.yml" | |
| git add . | |
| git commit -m "Generated from base-vm" | |
| newHash="$(git rev-parse main)" | |
| echo "New hash: $newHash" | |
| #modify test.yml | |
| sed -i "s/$oldHash/$newHash/g" .github/workflows/test.yml | |
| fi | |
| - name: Commit and push changes | |
| run: | | |
| git config user.email "ghactions@vmactions.org" | |
| git config user.name "Generate.yml" | |
| git add \ | |
| .github/workflows/readme.yml \ | |
| .github/workflows/major.yml \ | |
| .github/workflows/manual.yml \ | |
| .github/workflows/test.yml \ | |
| .github/workflows/bump.yml \ | |
| .github/tpl/README.tpl.md \ | |
| .github/FUNDING.yml \ | |
| .github/data/base-vm-hash.txt \ | |
| .github/data/datafile.ini \ | |
| .github/data/sync-map.json \ | |
| .github/data/exclude-missing.txt \ | |
| .github/data/exclude-sshfs.txt \ | |
| .github/data/exclude-nfs.txt \ | |
| .github/data/exclude-scp.txt \ | |
| .github/data/exclude-rsync.txt \ | |
| conf \ | |
| package.json \ | |
| package-lock.json \ | |
| index.js \ | |
| action.yml \ | |
| node_modules | |
| if git diff --cached --quiet; then | |
| echo "Nothing to commit, skipping push" | |
| else | |
| printf "Update ${{ github.repository }} from base-vm\n\n%s\n" "$(cat /tmp/basevm-log.txt)" > /tmp/commit-msg.txt | |
| git commit -F /tmp/commit-msg.txt | |
| # test.yml pins the action to the "Generated from base-vm" commit (HEAD~1). | |
| # The git pull --rebase below can rewrite that commit when Step 1 / Update | |
| # Readme push to main at the same time, which would leave test.yml pointing | |
| # at a SHA that no longer exists ("unable to resolve action"). So pin the | |
| # hash AFTER each rebase and retry until the push lands cleanly. | |
| REPO="${{ github.repository }}" | |
| for attempt in 1 2 3 4 5; do | |
| if ! git pull --rebase --autostash; then | |
| # A concurrent generate run already pushed a regenerated tree; | |
| # the only conflict is the self-referencing pin in test.yml and | |
| # that run's result is equivalent. Defer to it instead of | |
| # letting the rebase conflict fail this step under `bash -e`. | |
| git rebase --abort || true | |
| git reset --hard origin/main | |
| echo "Concurrent generate detected; deferring to origin/main" | |
| break | |
| fi | |
| parent="$(git rev-parse HEAD~1)" | |
| if [ -n "$parent" ]; then | |
| sed -i "s#\(uses: ${REPO}@\)[0-9a-fA-F]\{7,40\}#\1${parent}#g" .github/workflows/test.yml | |
| fi | |
| if ! git diff --quiet -- .github/workflows/test.yml; then | |
| git add .github/workflows/test.yml | |
| git commit --amend --no-edit | |
| fi | |
| if git push; then | |
| echo "Pushed on attempt ${attempt}; test.yml pinned to ${parent}" | |
| break | |
| fi | |
| if [ "${attempt}" -eq 5 ]; then | |
| echo "Failed to push after ${attempt} attempts" | |
| exit 1 | |
| fi | |
| echo "Push was contended, retrying (${attempt})..." | |
| done | |
| fi | |