percona · nogueiraanderson · Jul 16, 2026 · Jul 16, 2026
@@ -0,0 +1,132 @@
+name: jenkins-agent image factory
+
+# Bake -> smoke -> promote the Tier-A agent AMIs (strategy layer 1). PR runs
+# validate templates with no AWS access; the bake job is manual dispatch first
+# (weekly cron follows once the pxb canary proves the images). Promotion is a
+# role-tag flip, so consumers never resolve an unsmoked image.
+
+on:
+  pull_request:
+    paths:
+      - 'images/jenkins-agent/**'
+      - '.github/workflows/jenkins-agent-image-factory.yml'
+  workflow_dispatch:
+    inputs:
+      arches:
+        description: JSON list of arches to bake
+        default: '["arm64","x86_64"]'
+
+env:
+  AGENT_FACTORY_REGION: eu-central-1
+  PACKER_VERSION: 1.15.4
+
+jobs:
+  # No-AWS gate: malformed templates or scripts fail here before any EC2 spins
+  # up. No id-token, so this job can never assume the factory role.
+  validate:
+    name: Validate templates (no AWS)
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+      - uses: hashicorp/setup-packer@1aa358be5cf73883762b302a3a03abd66e75b232 # v3.1.0
+        with:
+          version: ${{ env.PACKER_VERSION }}
+      - name: fmt + validate (both arches)
+        working-directory: images/jenkins-agent
+        run: |
+          set -euo pipefail
+          packer fmt -check -recursive .
+          for arch in arm64 x86_64; do
+            packer init .
+            packer validate -var arch="$arch" .
+            packer init smoke
+            packer validate -var candidate_ami=ami-000000000000000ff -var arch="$arch" smoke
+          done
+      - name: shellcheck provisioners + smoke
+        run: shellcheck images/jenkins-agent/provisioners/*.sh images/jenkins-agent/smoke/*.sh
+
+  bake:
+    name: Bake + smoke + promote (${{ matrix.arch }})
+    if: github.event_name == 'workflow_dispatch'
+    needs: validate
+    runs-on: ubuntu-latest
+    # Stays under the OIDC role's one-hour session: credentials do not refresh
+    # mid-job, so a run that would exceed the session must fail early instead
+    # of losing AWS access halfway through cleanup or promotion.
+    timeout-minutes: 55
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: ${{ fromJSON(github.event.inputs.arches || '["arm64","x86_64"]') }}
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+      - uses: hashicorp/setup-packer@1aa358be5cf73883762b302a3a03abd66e75b232 # v3.1.0
+        with:
+          version: ${{ env.PACKER_VERSION }}
+      - uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
+        with:
+          role-to-assume: ${{ secrets.AGENT_FACTORY_ROLE_ARN }}
+          aws-region: ${{ env.AGENT_FACTORY_REGION }}
+
+      - name: Bake candidate
+        id: bake
+        working-directory: images/jenkins-agent
+        run: |
+          set -euo pipefail
+          packer init .
+          packer build -color=false -var arch="${{ matrix.arch }}" -var region="$AGENT_FACTORY_REGION" .
+          ami=$(jq -r '.builds[-1].artifact_id | split(":")[1]' manifest.json)
+          if [ -z "$ami" ] || [ "$ami" = "null" ]; then
+            echo "FATAL: no AMI id in packer manifest"
+            exit 1
+          fi
+          arch_tag=$(aws ec2 describe-images --region "$AGENT_FACTORY_REGION" --image-ids "$ami" \
+            --query "Images[0].Tags[?Key=='arch']|[0].Value" --output text)
+          if [ "$arch_tag" != "${{ matrix.arch }}" ]; then
+            echo "FATAL: manifest AMI $ami carries arch=$arch_tag, expected ${{ matrix.arch }}"
+            exit 1
+          fi
+          echo "ami=$ami" >> "$GITHUB_OUTPUT"
+          echo "- bake (${{ matrix.arch }}): $ami" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Smoke (fresh boot + assertions, no by-product AMI)
+        working-directory: images/jenkins-agent
+        env:
+          AMI: ${{ steps.bake.outputs.ami }}
+        run: |
+          set -euo pipefail
+          packer init smoke
+          packer build -color=false -var candidate_ami="$AMI" -var arch="${{ matrix.arch }}" -var region="$AGENT_FACTORY_REGION" smoke
+          echo "- smoke (${{ matrix.arch }}): pass" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Promote (role tag flip)
+        id: promote
+        env:
+          AMI: ${{ steps.bake.outputs.ami }}
+        run: |
+          set -euo pipefail
+          for attempt in 1 2 3 4 5; do
+            aws ec2 create-tags --region "$AGENT_FACTORY_REGION" --resources "$AMI" \
+              --tags Key=role,Value=jenkins-agent Key=smoke,Value=passed && { echo "promoted $AMI"; break; }
+            [ "$attempt" = 5 ] && { echo "- promote (${{ matrix.arch }}): FAILED ($AMI left as candidate)" >> "$GITHUB_STEP_SUMMARY"; exit 1; }
+            sleep $((attempt * 5))
+          done
+          echo "- promote (${{ matrix.arch }}): $AMI is role=jenkins-agent" >> "$GITHUB_STEP_SUMMARY"
+
+      # A candidate that exists but never promoted (smoke failure, cancellation,
+      # parse failure) must not linger: deregister it with its snapshots.
+      - name: Cleanup unpromoted candidate
+        if: always() && steps.bake.outputs.ami != '' && steps.promote.outcome != 'success'
+        env:
+          AMI: ${{ steps.bake.outputs.ami }}
+        run: |
+          set -euo pipefail
+          echo "cleaning up unpromoted candidate $AMI"
+          aws ec2 deregister-image --region "$AGENT_FACTORY_REGION" --image-id "$AMI" --delete-associated-snapshots
+          echo "- cleanup (${{ matrix.arch }}): unpromoted candidate $AMI deregistered" >> "$GITHUB_STEP_SUMMARY"
@@ -0,0 +1,51 @@
+# Jenkins agent-image factory
+
+Packer factory for the worker **agent AMIs** (strategy layer 1): one baked
+image per (os, arch, capability profile) carrying the fleet-common payload
+every worker label family installs at boot today. Baking removes the measured
+multi-minute per-launch setup and the boot-time network dependencies.
+
+First delivered images: `jenkins-agent-al2023-{x86_64,arm64}`. The arm64
+profile bakes qemu user-mode emulation (persistent binfmt via systemd) so
+x86_64-only containers run on Graviton workers.
+
+## Layout (extension recipe)
+
+| Path | Role |
+|------|------|
+| `agent.pkr.hcl` | The single parameterized template. New (os, arch) combos extend variables, never copy the file |
+| `provisioners/00-common.sh` | Fleet-common payload (java, git, docker, awscli, archive tools) |
+| `provisioners/10-qemu-binfmt.sh` | arm64 capability profile: digest-pinned qemu + persistent binfmt. Self-guards on arch |
+| `smoke/` | Fresh-boot assertion build. A candidate promotes only after this passes |
+| `justfile` | Local recipes mirroring the workflow: `fmt-check`, `validate`, `bake`, `smoke`, `promote`, `list` |
+
+To add an image family: add a provisioner script (numbered, self-guarded),
+extend the smoke assertions, and if a new OS is involved add its base-AMI SSM
+parameter mapping in `agent.pkr.hcl`. The workflow matrix picks it up via the
+`arches`/vars inputs.
+
+## Lifecycle
+
+Bake (candidate, `role=jenkins-agent-candidate`) -> fresh-boot smoke ->
+promote (`role=jenkins-agent` + `smoke=passed` tag flip). Failed smokes
+deregister the candidate; consumers resolving by role never see an unsmoked
+image. Superseded images age out via native EC2 image deprecation; pruning
+follows the fail-safe rails of the ppg factory prune (keep >= 2, protected
+roles, loud no-op).
+
+## Consumption
+
+Worker fleets consume committed per-region literal AMI ids (ADR 0029), bumped
+by PR after promotion. The Graviton fleets (`modules/jenkins-arm-fleet`)
+currently resolve the latest Amazon Linux 2 arm64 dynamically; adopting a
+baked image means passing `ami_id` explicitly per fleet, starting with a pxb
+canary, and slimming the boot user data to the instance-shape-specific steps
+(ephemeral /mnt mount, docker data-root) that stay out of the bake.
+
+## Roadmap
+
+- Per-region `copy-image` + per-region smoke before multi-region adoption
+  (the factory bakes in one region; consumer fleets span five).
+- Weekly refresh cron once the pxb canary proves the images.
+- Hetzner snapshots via the `hcloud` builder in this same directory.
+- Rocky/native-OS `min-*` profiles ride existing demand.
@@ -0,0 +1,120 @@
+packer {
+  required_plugins {
+    amazon = {
+      source  = "github.com/hashicorp/amazon"
+      version = "= 1.8.2"
+    }
+  }
+}
+
+# Parameterized Jenkins agent-image factory (strategy layer 1). One template
+# serves every (os, arch, profile) combo; the first delivered images are
+# jenkins-agent-al2023-{x86_64,arm64}, the arm64 profile carrying baked qemu
+# user-mode emulation. Extend by adding an os/arch value plus a provisioner,
+# not by copying the template.
+
+variable "region" {
+  type    = string
+  default = "eu-central-1"
+}
+
+variable "os_name" {
+  type    = string
+  default = "al2023"
+}
+
+variable "arch" {
+  type = string
+  validation {
+    condition     = contains(["x86_64", "arm64"], var.arch)
+    error_message = "The arch value must be x86_64 or arm64."
+  }
+}
+
+variable "builder_instance_profile" {
+  type    = string
+  default = "jenkins-agent-builder-ssm"
+}
+
+variable "builder_sg_name" {
+  type    = string
+  default = "jenkins-agent-factory-builder"
+}
+
+variable "billing_tag" {
+  type    = string
+  default = "jenkins-agent-factory"
+}
+
+locals {
+  timestamp     = regex_replace(timestamp(), "[- TZ:]", "")
+  image_name    = "jenkins-agent-${var.os_name}-${var.arch}-${local.timestamp}"
+  instance_type = var.arch == "arm64" ? "c7g.large" : "c6i.large"
+
+  # Standard (not minimal) AL2023: the SSM agent ships preinstalled, which the
+  # session_manager communicator and the fleet's shell access both rely on.
+  base_ami_param = "/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-${var.arch}"
+
+  common_tags = {
+    Name              = local.image_name
+    role              = "jenkins-agent-candidate"
+    os                = var.os_name
+    arch              = var.arch
+    source            = "factory"
+    "iit-billing-tag" = var.billing_tag
+  }
+}
+
+data "amazon-parameterstore" "base_ami" {
+  region = var.region
+  name   = local.base_ami_param
+}
+
+source "amazon-ebs" "agent" {
+  region        = var.region
+  source_ami    = data.amazon-parameterstore.base_ami.value
+  instance_type = local.instance_type
+  ami_name      = local.image_name
+
+  # SSM-only builder: no inbound SSH, the pre-created egress-only SG disables
+  # Packer's temporary SG and the OIDC role carries no SG mutations.
+  communicator         = "ssh"
+  ssh_username         = "ec2-user"
+  ssh_interface        = "session_manager"
+  iam_instance_profile = var.builder_instance_profile
+
+  security_group_filter {
+    filters = {
+      "group-name" = var.builder_sg_name
+    }
+  }
+
+  launch_block_device_mappings {
+    device_name           = "/dev/xvda"
+    volume_size           = 20
+    volume_type           = "gp3"
+    delete_on_termination = true
+  }
+
+  run_tags      = merge(local.common_tags, { Name = "${local.image_name}-builder" })
+  tags          = local.common_tags
+  snapshot_tags = local.common_tags
+}
+
+build {
+  sources = ["source.amazon-ebs.agent"]
+
+  # 10-qemu-binfmt self-guards on uname and no-ops on x86_64 builders, so the
+  # provisioner list stays arch-agnostic.
+  provisioner "shell" {
+    scripts = [
+      "provisioners/00-common.sh",
+      "provisioners/10-qemu-binfmt.sh",
+    ]
+  }
+
+  # The workflow reads the AMI id from here, never from human-readable output.
+  post-processor "manifest" {
+    output = "manifest.json"
+  }
+}
@@ -0,0 +1,43 @@
+# Jenkins agent-image factory. Bake/smoke/promote one (os, arch) combo at a
+# time; the GHA workflow drives the same recipes. Promotion is a role-tag flip
+# (candidate -> jenkins-agent) so consumers resolving by tag never see an
+# unsmoked image. Worker clouds consume committed per-region literal AMI ids
+# (ADR 0029), bumped by PR after promotion.
+
+region  := env_var_or_default("AGENT_FACTORY_REGION", "eu-central-1")
+profile := env_var_or_default("AWS_PROFILE", "")
+
+_aws_profile_args := if profile == "" { "" } else { "--profile " + profile }
+
+default:
+    @just --list
+
+fmt-check:
+    packer fmt -check -recursive .
+
+validate arch="arm64":
+    packer init .
+    packer validate -var arch={{arch}} -var region={{region}} .
+    packer init smoke
+    packer validate -var candidate_ami=ami-000000000000000ff -var arch={{arch}} -var region={{region}} smoke
+
+# Bake one candidate AMI. Output ends with the ami id (packer manifest).
+bake arch="arm64":
+    packer init .
+    packer build -color=false -var arch={{arch}} -var region={{region}} .
+
+# Boot + assert a candidate (skip_create_ami: no by-product AMI to clean up).
+smoke candidate_ami arch="arm64":
+    packer init smoke
+    packer build -color=false -var candidate_ami={{candidate_ami}} -var arch={{arch}} -var region={{region}} smoke
+
+# Promote a smoked candidate: role tag flip + smoke marker.
+promote candidate_ami:
+    aws ec2 create-tags {{_aws_profile_args}} --region {{region}} --resources {{candidate_ami}} \
+      --tags Key=role,Value=jenkins-agent Key=smoke,Value=passed
+
+# Current agent images, newest first.
+list:
+    aws ec2 describe-images {{_aws_profile_args}} --region {{region}} --owners self --include-deprecated \
+      --filters Name=tag:role,Values=jenkins-agent \
+      --query 'reverse(sort_by(Images,&CreationDate))[].[ImageId,Name,Tags[?Key==`arch`]|[0].Value]' --output table
@@ -0,0 +1,32 @@
+#!/bin/bash
+# Fleet-common agent payload: everything every worker label family installs at
+# boot today (the 2.5-8 min per-launch setup this factory removes). Baked once
+# per (os, arch); instance-shape-specific steps (ephemeral-disk mkfs/mount) and
+# env-specific ones (/etc/hosts entries, Jenkins remoting) stay in the
+# now-minimal boot initScript.
+set -euo pipefail
+
+sudo dnf -y -q update
+
+# file(1) is load-bearing for 10-qemu-binfmt.sh's BuildID verification.
+sudo dnf -y -q install \
+  java-17-amazon-corretto-headless \
+  git \
+  docker \
+  awscli-2 \
+  file \
+  tar \
+  unzip \
+  p7zip
+
+sudo systemctl enable docker
+sudo systemctl start docker
+
+# Workers run builds as ec2-user via the Jenkins agent: docker group membership
+# replaces the per-boot usermod the initScript does today.
+sudo usermod -aG docker ec2-user
+
+java -version 2>&1 | head -1
+git --version
+docker --version
+aws --version