package-updated #990
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update APT Repository | |
| on: | |
| repository_dispatch: | |
| types: [package-updated] # This listens for the event from package repos | |
| workflow_dispatch: # Manual trigger | |
| # Daily rebuild disabled during apt.halos.fi migration. | |
| # A full rebuild now would drop HaLOS packages (repos moved to halos-org) | |
| # and break existing installations before bridge packages are deployed. | |
| # schedule: | |
| # - cron: '0 6 * * *' | |
| # Each dispatch run gets a unique group (unlimited parallel). | |
| # Scheduled/manual runs share a group (serialized, one at a time). | |
| concurrency: | |
| group: ${{ github.event_name == 'repository_dispatch' && format('apt-dispatch-{0}', github.run_id) || 'apt-repo-update' }} | |
| cancel-in-progress: false | |
| permissions: | |
| contents: write | |
| jobs: | |
| update-apt-repo: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Debug repository dispatch | |
| if: github.event_name == 'repository_dispatch' | |
| run: | | |
| echo "Event type: ${{ github.event.action }}" | |
| echo "Client payload: ${{ toJson(github.event.client_payload) }}" | |
| echo "Triggered by: ${{ github.event.client_payload.repository }}" | |
| - name: Parse payload and determine target distribution | |
| id: payload | |
| run: | | |
| # Define supported distributions (single source of truth) | |
| ALL_DISTRIBUTIONS="stable unstable bookworm-stable bookworm-unstable trixie-stable trixie-unstable" | |
| echo "all_distributions=$ALL_DISTRIBUTIONS" >> $GITHUB_OUTPUT | |
| # Extract payload fields (defaults for backward compatibility and scheduled runs) | |
| DISTRO="${{ github.event.client_payload.distro }}" | |
| CHANNEL="${{ github.event.client_payload.channel }}" | |
| COMPONENT="${{ github.event.client_payload.component }}" | |
| TARGET_REPO="${{ github.event.client_payload.repository }}" | |
| # Set defaults if not provided | |
| DISTRO="${DISTRO:-any}" | |
| CHANNEL="${CHANNEL:-stable}" | |
| COMPONENT="${COMPONENT:-main}" | |
| # Normalize to lowercase for case-insensitive comparison | |
| DISTRO=$(echo "$DISTRO" | tr '[:upper:]' '[:lower:]') | |
| CHANNEL=$(echo "$CHANNEL" | tr '[:upper:]' '[:lower:]') | |
| COMPONENT=$(echo "$COMPONENT" | tr '[:upper:]' '[:lower:]') | |
| # Map to distribution name | |
| if [ "$DISTRO" = "any" ]; then | |
| DISTRIBUTION="$CHANNEL" | |
| else | |
| DISTRIBUTION="${DISTRO}-${CHANNEL}" | |
| fi | |
| echo "=== Payload Configuration ===" | |
| echo "Distro: $DISTRO" | |
| echo "Channel: $CHANNEL" | |
| echo "Component: $COMPONENT" | |
| echo "Distribution: $DISTRIBUTION" | |
| echo "Target Repository: ${TARGET_REPO:-<all repositories>}" | |
| # Export for later steps | |
| echo "distro=$DISTRO" >> $GITHUB_OUTPUT | |
| echo "channel=$CHANNEL" >> $GITHUB_OUTPUT | |
| echo "component=$COMPONENT" >> $GITHUB_OUTPUT | |
| echo "distribution=$DISTRIBUTION" >> $GITHUB_OUTPUT | |
| echo "target_repo=$TARGET_REPO" >> $GITHUB_OUTPUT | |
| # Determine if this is a targeted update or full rebuild | |
| if [ -n "$TARGET_REPO" ]; then | |
| echo "mode=targeted" >> $GITHUB_OUTPUT | |
| echo "Mode: Targeted update for $TARGET_REPO" | |
| else | |
| echo "mode=full" >> $GITHUB_OUTPUT | |
| echo "Mode: Full repository rebuild" | |
| fi | |
| - name: Discover package repositories | |
| id: discover | |
| run: | | |
| echo "Discovering repositories with 'apt-package' topic..." | |
| # Use GitHub API to find repos with apt-package topic | |
| repos=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ | |
| "https://api.github.com/search/repositories?q=org:hatlabs+topic:apt-package" | \ | |
| jq -r '.items[].full_name') | |
| echo "Found repositories:" | |
| echo "$repos" | |
| # Store for next step | |
| echo "repos<<EOF" >> $GITHUB_OUTPUT | |
| echo "$repos" >> $GITHUB_OUTPUT | |
| echo "EOF" >> $GITHUB_OUTPUT | |
| - name: Setup build environment | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y dpkg-dev apt-utils curl jq gnupg | |
| # Create packages directory | |
| mkdir -p packages | |
| MODE="${{ steps.payload.outputs.mode }}" | |
| if [ "$MODE" = "full" ]; then | |
| echo "=== Full rebuild mode: Creating fresh directory structure ===" | |
| for dist in ${{ steps.payload.outputs.all_distributions }}; do | |
| echo "Creating distribution: $dist" | |
| mkdir -p "apt-repo/pool/$dist/main" | |
| mkdir -p "apt-repo/dists/$dist/main/binary-arm64" | |
| mkdir -p "apt-repo/dists/$dist/main/binary-armhf" | |
| mkdir -p "apt-repo/dists/$dist/main/binary-all" | |
| done | |
| else | |
| echo "=== Targeted mode: apt-repo will be cloned from gh-pages in publish step ===" | |
| fi | |
| - name: Import GPG signing key | |
| run: | | |
| echo "${{ secrets.APT_SIGNING_KEY }}" | gpg --import --batch | |
| # Get the key ID for signing | |
| GPG_KEY_ID=$(gpg --list-secret-keys --keyid-format LONG | grep sec | head -1 | sed 's/.*\/\([A-Z0-9]*\) .*/\1/') | |
| echo "GPG_KEY_ID=$GPG_KEY_ID" >> $GITHUB_ENV | |
| echo "Using GPG key: $GPG_KEY_ID" | |
| - name: Download packages | |
| run: | | |
| MODE="${{ steps.payload.outputs.mode }}" | |
| TARGET_REPO="${{ steps.payload.outputs.target_repo }}" | |
| CHANNEL="${{ steps.payload.outputs.channel }}" | |
| # Source helper functions (fail fast if missing) | |
| source scripts/suffix-parsing-functions.sh || { echo "Failed to load suffix parsing functions"; exit 1; } | |
| source scripts/routing-functions.sh || { echo "Failed to load routing functions"; exit 1; } | |
| # Function to download packages from a repository | |
| download_from_repo() { | |
| local repo=$1 | |
| local release_tag=$2 | |
| echo "=== Processing $repo (release: $release_tag) ===" | |
| # Get release info | |
| if [ "$release_tag" = "latest" ]; then | |
| echo "Fetching latest release from $repo..." | |
| release_info=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ | |
| "https://api.github.com/repos/$repo/releases/latest") | |
| elif [ "$release_tag" = "prerelease" ]; then | |
| echo "Fetching latest pre-release from $repo..." | |
| release_info=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ | |
| "https://api.github.com/repos/$repo/releases" | \ | |
| jq '[.[] | select(.prerelease == true)] | sort_by(.published_at) | reverse | .[0]') | |
| else | |
| echo "Fetching release $release_tag from $repo..." | |
| release_info=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ | |
| "https://api.github.com/repos/$repo/releases/tags/$release_tag") | |
| fi | |
| # Check if release exists | |
| if [ "$(echo "$release_info" | jq -r '.message // empty')" = "Not Found" ]; then | |
| echo "No release found for $repo (tag: $release_tag)" | |
| return | |
| fi | |
| # Check for null/empty release_info (e.g., when no pre-releases exist) | |
| if [ -z "$release_info" ] || [ "$release_info" = "null" ]; then | |
| if [ "$release_tag" = "prerelease" ]; then | |
| echo "No pre-release found for $repo" | |
| else | |
| echo "No release information found for $repo" | |
| fi | |
| return | |
| fi | |
| local actual_tag=$(echo "$release_info" | jq -r '.tag_name') | |
| echo "Release tag: $actual_tag" | |
| # Download all .deb files from the release | |
| echo "Downloading .deb files..." | |
| echo "$release_info" | jq -r '.assets[] | select(.name | endswith(".deb")) | "\(.name)|\(.browser_download_url)"' | \ | |
| while IFS='|' read -r name url; do | |
| if [ -n "$name" ] && [ -n "$url" ]; then | |
| echo "Downloading $name..." | |
| temp_file="packages/temp_$name" | |
| curl -L --fail -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ | |
| -H "Accept: application/octet-stream" \ | |
| "$url" -o "$temp_file" | |
| if [ $? -eq 0 ]; then | |
| echo "Downloaded $name" | |
| # Extract package metadata | |
| actual_version=$(dpkg-deb --field "$temp_file" Version 2>/dev/null || true) | |
| package_name=$(dpkg-deb --field "$temp_file" Package 2>/dev/null || true) | |
| architecture=$(dpkg-deb --field "$temp_file" Architecture 2>/dev/null || true) | |
| if [ -n "$actual_version" ] && [ -n "$package_name" ] && [ -n "$architecture" ]; then | |
| # Parse suffix from original filename | |
| parse_package_suffix "$name" pkg_distro pkg_component || true | |
| if [ $? -eq 0 ]; then | |
| echo " Parsed suffix: distro=$pkg_distro, component=$pkg_component" | |
| if ! validate_suffix "$pkg_distro" "$pkg_component"; then | |
| echo " Using fallback: distro=any, component=main" | |
| pkg_distro="any" | |
| pkg_component="main" | |
| fi | |
| else | |
| echo " No suffix found, using defaults: distro=$pkg_distro, component=$pkg_component" | |
| fi | |
| # Create canonical filename | |
| correct_filename="${package_name}_${actual_version}_${architecture}.deb" | |
| final_path="packages/$correct_filename" | |
| mv "$temp_file" "$final_path" | |
| # Store metadata for routing | |
| { | |
| echo "package=$package_name" | |
| echo "version=$actual_version" | |
| echo "architecture=$architecture" | |
| echo "distro=$pkg_distro" | |
| echo "component=$pkg_component" | |
| echo "original_filename=$name" | |
| } > "${final_path}.meta" | |
| echo " Package: $package_name" | |
| echo " Version: $actual_version" | |
| echo " Architecture: $architecture" | |
| echo " Distro: $pkg_distro" | |
| echo " Component: $pkg_component" | |
| echo " Filename: $correct_filename" | |
| else | |
| echo " Could not extract package metadata, keeping original filename" | |
| mv "$temp_file" "packages/$name" | |
| fi | |
| else | |
| echo "Failed to download $name" | |
| rm -f "$temp_file" | |
| fi | |
| fi | |
| done | |
| echo "" | |
| } | |
| # Determine which repositories and releases to process | |
| if [ "$MODE" = "targeted" ]; then | |
| echo "=== Targeted Update Mode ===" | |
| if [ "$CHANNEL" = "unstable" ]; then | |
| RELEASE_TAG="prerelease" | |
| else | |
| RELEASE_TAG="latest" | |
| fi | |
| download_from_repo "$TARGET_REPO" "$RELEASE_TAG" | |
| else | |
| echo "=== Full Rebuild Mode ===" | |
| echo "Downloading all packages and routing based on metadata..." | |
| for channel in stable unstable; do | |
| if [ "$channel" = "unstable" ]; then | |
| RELEASE_TAG="prerelease" | |
| else | |
| RELEASE_TAG="latest" | |
| fi | |
| echo "" | |
| echo "--- Downloading $channel packages (release: $RELEASE_TAG) ---" | |
| rm -rf packages/* | |
| mkdir -p packages | |
| echo "${{ steps.discover.outputs.repos }}" | while IFS= read -r repo; do | |
| if [ -n "$repo" ]; then | |
| source scripts/suffix-parsing-functions.sh || { echo "Failed to load suffix parsing functions"; exit 1; } | |
| source scripts/routing-functions.sh || { echo "Failed to load routing functions"; exit 1; } | |
| download_from_repo "$repo" "$RELEASE_TAG" | |
| fi | |
| done | |
| # Route downloaded packages based on their metadata | |
| if ls packages/*.deb 1> /dev/null 2>&1; then | |
| pkg_count=$(ls packages/*.deb | wc -l) | |
| echo "Routing $pkg_count packages ($channel channel) to appropriate distributions..." | |
| routing_failed=0 | |
| for pkg in packages/*.deb; do | |
| pkg_name=$(basename "$pkg") | |
| meta_file="${pkg}.meta" | |
| unset distro component package version architecture original_filename | |
| if [ -f "$meta_file" ]; then | |
| source "$meta_file" 2>/dev/null || { | |
| echo "Failed to read metadata for $pkg_name" | |
| routing_failed=$((routing_failed + 1)) | |
| continue | |
| } | |
| if [ "$distro" = "any" ]; then | |
| echo " -> $pkg_name (distro=any, component=$component)" | |
| targets_list="" | |
| for d in "${SUPPORTED_DISTROS[@]}"; do | |
| targets_list="$targets_list ${d}-${channel}/$component" | |
| done | |
| echo " Routes to:$targets_list" | |
| if [ "$component" = "hatlabs" ]; then | |
| echo " + Legacy: ${channel}/main" | |
| fi | |
| else | |
| echo " -> $pkg_name (distro=$distro, component=$component)" | |
| echo " Routes to: ${distro}-${channel}/$component" | |
| fi | |
| fi | |
| if ! route_package "$pkg" "$channel"; then | |
| echo "Failed to route $pkg_name" | |
| routing_failed=$((routing_failed + 1)) | |
| fi | |
| done | |
| routed_count=$((pkg_count - routing_failed)) | |
| if [ $routing_failed -eq 0 ]; then | |
| echo "Successfully routed all $pkg_count $channel packages" | |
| else | |
| echo "Failed to route $routing_failed/$pkg_count packages" | |
| exit 1 | |
| fi | |
| else | |
| echo "No packages downloaded for $channel" | |
| fi | |
| done | |
| rm -rf packages/* | |
| fi | |
| echo "=== Download step completed ===" | |
| - name: Install test dependencies | |
| run: | | |
| pip3 install pytest | |
| - name: Run index generation tests | |
| run: | | |
| cd scripts | |
| pytest test_generate_index.py -v | |
| - name: Publish with optimistic concurrency (targeted mode) | |
| if: steps.payload.outputs.mode == 'targeted' | |
| run: | | |
| set -euo pipefail | |
| CHANNEL="${{ steps.payload.outputs.channel }}" | |
| COMPONENT="${{ steps.payload.outputs.component }}" | |
| DISTRO="${{ steps.payload.outputs.distro }}" | |
| REPO_URL="https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git" | |
| MAX_RETRIES=10 | |
| # Source helper functions | |
| source scripts/suffix-parsing-functions.sh | |
| source scripts/routing-functions.sh | |
| # Verify we have packages to publish | |
| if ! ls packages/*.deb 1>/dev/null 2>&1; then | |
| echo "No packages to publish" | |
| exit 0 | |
| fi | |
| echo "=== Packages to publish ===" | |
| ls -la packages/*.deb | |
| for attempt in $(seq 1 $MAX_RETRIES); do | |
| echo "" | |
| echo "=== Attempt $attempt of $MAX_RETRIES ===" | |
| # Fresh clone of gh-pages | |
| rm -rf apt-repo | |
| if git clone --branch gh-pages --single-branch --depth=1 "$REPO_URL" apt-repo 2>/dev/null; then | |
| echo "Cloned gh-pages branch" | |
| else | |
| echo "No gh-pages branch found (first run). Creating empty repository." | |
| mkdir -p apt-repo | |
| cd apt-repo | |
| git init | |
| git checkout -b gh-pages | |
| git remote add origin "$REPO_URL" | |
| cd .. | |
| fi | |
| # Route packages to pool | |
| already_exists=true | |
| for pkg in packages/*.deb; do | |
| filename=$(basename "$pkg") | |
| meta_file="${pkg}.meta" | |
| if [ ! -f "$meta_file" ]; then | |
| echo "Error: No metadata for $filename" | |
| exit 1 | |
| fi | |
| # Read metadata for routing targets | |
| unset distro component | |
| source "$meta_file" | |
| # Determine target distributions | |
| if [ "$distro" = "any" ]; then | |
| target_distros=("${SUPPORTED_DISTROS[@]}") | |
| else | |
| target_distros=("$distro") | |
| fi | |
| # Check if package already exists in all targets | |
| for target_distro in "${target_distros[@]}"; do | |
| target_dist="${target_distro}-${CHANNEL}" | |
| target_pool="apt-repo/pool/${target_dist}/${component}" | |
| if [ ! -f "$target_pool/$filename" ]; then | |
| already_exists=false | |
| break 2 | |
| fi | |
| done | |
| # Also check legacy pool for distro=any + component=hatlabs | |
| if [ "$already_exists" = "true" ] && [ "$distro" = "any" ] && [ "$component" = "hatlabs" ]; then | |
| if [ ! -f "apt-repo/pool/${CHANNEL}/main/$filename" ]; then | |
| already_exists=false | |
| break | |
| fi | |
| fi | |
| done | |
| if [ "$already_exists" = "true" ]; then | |
| echo "All packages already exist in pool — another run published them." | |
| exit 0 | |
| fi | |
| # Route packages into the cloned apt-repo | |
| for pkg in packages/*.deb; do | |
| echo "Routing: $(basename "$pkg")" | |
| if ! route_package "$pkg" "$CHANNEL"; then | |
| echo "Failed to route $(basename "$pkg")" | |
| exit 1 | |
| fi | |
| done | |
| # Build metadata for affected distributions | |
| cd apt-repo | |
| source ../scripts/build-metadata.sh | |
| if [ "$DISTRO" = "any" ]; then | |
| for d in "${SUPPORTED_DISTROS[@]}"; do | |
| dist="${d}-${CHANNEL}" | |
| build_component "$dist" "$COMPONENT" | |
| build_release "$dist" | |
| done | |
| # Also build legacy distribution for component=hatlabs | |
| if [ "$COMPONENT" = "hatlabs" ]; then | |
| build_component "$CHANNEL" "main" | |
| build_release "$CHANNEL" | |
| fi | |
| else | |
| TARGET_DIST="${DISTRO}-${CHANNEL}" | |
| build_component "$TARGET_DIST" "$COMPONENT" | |
| build_release "$TARGET_DIST" | |
| fi | |
| # Export GPG public key | |
| gpg --export --armor $GPG_KEY_ID > hat-labs-apt-key.asc | |
| gpg --export $GPG_KEY_ID > hat-labs-apt-key.gpg | |
| # Generate index pages | |
| FINGERPRINT=$(gpg --fingerprint $GPG_KEY_ID | grep -A1 "pub " | tail -n1 | tr -d ' ') | |
| python3 ../scripts/generate_index.py . --gpg-fingerprint "$FINGERPRINT" | |
| # Set CNAME for GitHub Pages | |
| echo "apt.hatlabs.fi" > CNAME | |
| # Commit | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add -A | |
| if git diff --cached --quiet; then | |
| echo "No changes to commit — repository already up to date." | |
| cd .. | |
| exit 0 | |
| fi | |
| git commit -m "Add packages from ${{ steps.payload.outputs.target_repo }}" | |
| # Push (non-force) | |
| if git push origin gh-pages; then | |
| echo "=== Successfully published on attempt $attempt ===" | |
| cd .. | |
| exit 0 | |
| fi | |
| cd .. | |
| # Push failed — another run pushed first | |
| if [ "$attempt" -lt "$MAX_RETRIES" ]; then | |
| # Linear backoff with jitter | |
| delay=$(( 2 * attempt + RANDOM % 5 )) | |
| echo "Push failed (non-fast-forward). Retrying in ${delay}s..." | |
| sleep "$delay" | |
| fi | |
| done | |
| echo "ERROR: Failed to publish after $MAX_RETRIES attempts" | |
| exit 1 | |
| - name: Build APT repository structure (full rebuild) | |
| if: steps.payload.outputs.mode == 'full' | |
| run: | | |
| cd apt-repo | |
| # Source shared build functions | |
| source ../scripts/build-metadata.sh | |
| echo "=== Full Rebuild Mode ===" | |
| echo "Building metadata for all distributions" | |
| for dist in ${{ steps.payload.outputs.all_distributions }}; do | |
| build_component "$dist" "main" | |
| build_component "$dist" "hatlabs" | |
| build_release "$dist" | |
| done | |
| echo "" | |
| echo "=== Repository structure ===" | |
| find dists -name "Packages" -o -name "Release" | sort | |
| - name: Export public GPG key (full rebuild) | |
| if: steps.payload.outputs.mode == 'full' | |
| run: | | |
| cd apt-repo | |
| gpg --export --armor $GPG_KEY_ID > hat-labs-apt-key.asc | |
| gpg --export $GPG_KEY_ID > hat-labs-apt-key.gpg | |
| FINGERPRINT=$(gpg --fingerprint $GPG_KEY_ID | grep -A1 "pub " | tail -n1 | tr -d ' ') | |
| echo "GPG_FINGERPRINT=$FINGERPRINT" >> $GITHUB_ENV | |
| echo "Key fingerprint: $FINGERPRINT" | |
| - name: Generate repository index pages (full rebuild) | |
| if: steps.payload.outputs.mode == 'full' | |
| run: | | |
| set -e | |
| python3 scripts/generate_index.py \ | |
| apt-repo \ | |
| --gpg-fingerprint "$GPG_FINGERPRINT" | |
| - name: Validate package inventory (full rebuild) | |
| if: steps.payload.outputs.mode == 'full' | |
| run: | | |
| echo "=== PACKAGE INVENTORY BEFORE DEPLOY ===" | |
| final_packages=0 | |
| for dist_dir in apt-repo/pool/*/; do | |
| if [ -d "$dist_dir" ]; then | |
| dist_name=$(basename "$dist_dir") | |
| for comp_dir in "$dist_dir"*/; do | |
| if [ -d "$comp_dir" ]; then | |
| comp_name=$(basename "$comp_dir") | |
| pkg_count=$(find "$comp_dir" -name "*.deb" 2>/dev/null | wc -l | tr -d ' ') | |
| if [ "$pkg_count" -gt 0 ]; then | |
| echo " $dist_name/$comp_name: $pkg_count packages" | |
| final_packages=$((final_packages + pkg_count)) | |
| fi | |
| fi | |
| done | |
| fi | |
| done | |
| echo " TOTAL: $final_packages packages ready to deploy" | |
| echo "===========================================" | |
| echo "" | |
| echo "Package inventory validation passed" | |
| - name: Deploy to GitHub Pages (full rebuild) | |
| if: steps.payload.outputs.mode == 'full' | |
| uses: peaceiris/actions-gh-pages@v4 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: ./apt-repo | |
| cname: apt.hatlabs.fi | |
| - name: Report status | |
| run: | | |
| echo "=== APT Repository Update Complete ===" | |
| echo "Repository deployed to: https://apt.hatlabs.fi" | |
| echo "" | |
| echo "Distribution Summary:" | |
| for dist in ${{ steps.payload.outputs.all_distributions }}; do | |
| dist_has_packages=false | |
| for component in main hatlabs; do | |
| pkgfile="apt-repo/dists/$dist/$component/binary-arm64/Packages" | |
| if [ -f "$pkgfile" ]; then | |
| count=$(grep -c '^Package:' "$pkgfile" 2>/dev/null || echo "0") | |
| echo " $dist/$component: $count packages" | |
| dist_has_packages=true | |
| fi | |
| done | |
| if [ "$dist_has_packages" = false ]; then | |
| echo " $dist: (not built this run)" | |
| fi | |
| done |