airgap: Script for building images.list for airgap solution

.github/workflows/airgap-bundle.yaml

@@ -0,0 +1,70 @@
+# Build airgap bundle (images + OCI charts) when a tag is pushed.
+# Step 1: collect chart OCI refs into charts.list
+# Step 2: collect image refs into images.list
+# Step 3: run make_bundle.sh (skopeo copy images + charts, then tar)
+name: Airgap bundle
+
+on:
+  push:
+    tags: ["*"]
+
+jobs:
+  airgap-bundle:
+    runs-on: ubuntu-latest
+    env:
+      REPO_ROOT: ${{ github.workspace }}
+      # Set repo variable OCI_CHARTS_REGISTRY to override (e.g. ghcr.io/owner/pelagia-charts)
+      REGISTRY: ${{ github.ref_type == 'tag' && vars.REGISTRY_PROD || vars.REGISTRY_CI }}
+      DEV_VERSION: ${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || '' }}
+      HELM_REPO: ${{ github.event_name == 'push' && 'pelagia' || 'pelagia/pr' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install PyYAML
+        run: pip install pyyaml
+
+      - name: Collect chart OCI refs
+        run: python3 build/scripts/collect_chart_oci_refs.py
+        env:
+          OCI_CHARTS_REGISTRY: ${{ env.REGISTRY }}/${{ env.HELM_REPO }}
+          OUTPUT_FILE: ${{ github.workspace }}/charts.list
+
+      - name: Collect chart images
+        run: python3 build/scripts/collect_chart_images.py
+        env:
+          IMAGE_REGISTRY: ${{ env.REGISTRY }}
+          OUTPUT_FILE: ${{ github.workspace }}/images.list
+
+      - name: Get latest tag
+        id: get_tag
+        run: |
+          git fetch --tags
+          LATEST_TAG=$(git describe --tags --abbrev=0)
+          echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT
+      - name: Build airgap bundle
+        run: bash build/scripts/make_bundle.sh
+        env:
+          FULL_IMAGES_LIST_FILE: ${{ github.workspace }}/images.list
+          FULL_CHARTS_LIST_FILE: ${{ github.workspace }}/charts.list
+          AIRGAP_BUNDLE_DIR: ${{ github.workspace }}/bundle/images
+          AIRGAP_BUNDLE_FILE: ${{ github.workspace }}/airgap-bundle-ceph.tar.gz
+          AIRGAP_BUNDLE_VERSION: ${{ steps.get_tag.outputs.tag }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          # Optional: set DOCKER_CONFIG_PATH if charts/images need private registry auth
+          # DOCKER_CONFIG_PATH: ${{ github.workspace }}/.docker/config.json
+
+      - name: Upload airgap bundle
+        uses: actions/upload-artifact@v4
+        with:
+          name: airgap-bundle-${{ github.ref_name }}
+          path: ${{ github.workspace }}/airgap-bundle-ceph.tar.gz

build/scripts/collect_chart_images.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""
+Collects all container images in format registry/repo/image:tag from the
+"images" section (including nested subsections) of values of all Helm charts
+under charts/ and writes them to images.list (unique, sorted).
+
+Parses values.yaml directly so all repository+tag pairs are collected even
+when tag is an object (e.g. latest/squid/tentacle).
+
+Registry prefix is taken from IMAGE_REGISTRY env (e.g. set from REGISTRY in CI).
+Requires: PyYAML (pip install pyyaml)
+Usage: run from repo root, or set REPO_ROOT env to the repo root.
+       Set IMAGE_REGISTRY for the image registry prefix (e.g. registry.example.com).
+       Override output with OUTPUT_FILE env (default: images.list in repo root).
+"""
+
+import os
+import sys
+from pathlib import Path
+
+try:
+    import yaml
+except ImportError:
+    sys.stderr.write("Error: PyYAML required. Install with: pip install pyyaml\n")
+    sys.exit(1)
+
+
+def collect_images_from_obj(obj: dict, registry: str, out: list[str]) -> None:
+    """Recursively collect registry/repo:tag from nodes that have repository and tag."""
+    if not isinstance(obj, dict):
+        return
+    if "repository" in obj and "tag" in obj:
+        repo = str(obj["repository"]).strip()
+        if not repo:
+            return
+        tag = obj["tag"]
+        if isinstance(tag, str):
+            t = tag.strip()
+            if t:
+                ref = f"{registry}/{repo}:{t}" if registry else f"{repo}:{t}"
+                out.append(ref)
+        elif isinstance(tag, dict):
+            for v in tag.values():
+                if isinstance(v, str):
+                    t = v.strip()
+                    if t:
+                        ref = f"{registry}/{repo}:{t}" if registry else f"{repo}:{t}"
+                        out.append(ref)
+    for v in obj.values():
+        collect_images_from_obj(v, registry, out)
+
+
+def parse_values_file(path: Path, registry: str) -> list[str]:
+    """Parse a values.yaml and return list of image references (registry from IMAGE_REGISTRY)."""
+    with open(path, encoding="utf-8") as f:
+        data = yaml.safe_load(f) or {}
+    out: list[str] = []
+    if "images" in data:
+        collect_images_from_obj(data["images"], registry, out)
+    return out
+
+
+def main() -> None:
+    script_dir = Path(__file__).resolve().parent
+    repo_root = Path(os.environ.get("REPO_ROOT", script_dir.parent.parent))
+    charts_dir = repo_root / "charts"
+    output_file = Path(os.environ.get("OUTPUT_FILE", str(repo_root / "images.list")))
+    registry = (os.environ.get("IMAGE_REGISTRY") or "").strip()
+
+    all_images: list[str] = []
+    for chart_path in sorted(charts_dir.iterdir()):
+        if not chart_path.is_dir():
+            continue
+        chart_yaml = chart_path / "Chart.yaml"
+        values_yaml = chart_path / "values.yaml"
+        if not chart_yaml.exists() or not values_yaml.exists():
+            continue
+        print(f"Processing chart: {chart_path}", file=sys.stderr)
+        try:
+            all_images.extend(parse_values_file(values_yaml, registry))
+        except Exception as e:
+            print(f"Warning: failed to parse {values_yaml}: {e}", file=sys.stderr)
+
+    lines = sorted(set(s for s in all_images if s.strip()))
+    output_file.write_text("\n".join(lines) + ("\n" if lines else ""))
+    print(f"Written {len(lines)} unique image(s) to {output_file}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()

build/scripts/collect_chart_oci_refs.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""
+Collects chart names from charts/*/Chart.yaml and writes refs to charts.list:
+  <registry>/<repo>/<chart-name>:<version>
+(no oci:// prefix; make_bundle.sh uses oci: transport when copying.)
+
+Chart version is taken from VERSION env if set, otherwise from `make get-version`
+(run from repo root; same version logic as the rest of the build).
+
+Requires: PyYAML (pip install pyyaml)
+Env:
+  REPO_ROOT           - repo root (default: parent of build/ parent)
+  OCI_CHARTS_REGISTRY - e.g. ghcr.io/owner/pelagia-charts (no oci:// prefix)
+  VERSION             - chart version (optional; if unset, runs make get-version)
+  OUTPUT_FILE         - output path (default: charts.list in repo root)
+"""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+try:
+    import yaml
+except ImportError:
+    sys.stderr.write("Error: PyYAML required. Install with: pip install pyyaml\n")
+    sys.exit(1)
+
+
+def main() -> None:
+    script_dir = Path(__file__).resolve().parent
+    repo_root = Path(os.environ.get("REPO_ROOT", script_dir.parent.parent))
+    charts_dir = repo_root / "charts"
+    registry = (os.environ.get("OCI_CHARTS_REGISTRY") or "").strip()
+    version = (os.environ.get("VERSION") or "").strip()
+    output_file = Path(os.environ.get("OUTPUT_FILE", str(repo_root / "charts.list")))
+
+    if not registry:
+        sys.stderr.write("Error: OCI_CHARTS_REGISTRY env is required (e.g. ghcr.io/owner/charts)\n")
+        sys.exit(1)
+    if not version:
+        try:
+            result = subprocess.run(
+                ["make", "get-version"],
+                cwd=repo_root,
+                capture_output=True,
+                text=True,
+                check=True,
+            )
+            version = (result.stdout or "").strip()
+        except (subprocess.CalledProcessError, FileNotFoundError) as e:
+            sys.stderr.write(f"Error: VERSION env unset and 'make get-version' failed: {e}\n")
+            sys.exit(1)
+        if not version:
+            sys.stderr.write("Error: VERSION env is required or run from repo with 'make get-version' available\n")
+            sys.exit(1)
+
+    refs: list[str] = []
+    for chart_path in sorted(charts_dir.iterdir()):
+        if not chart_path.is_dir():
+            continue
+        chart_yaml = chart_path / "Chart.yaml"
+        if not chart_yaml.exists():
+            continue
+        with open(chart_yaml, encoding="utf-8") as f:
+            data = yaml.safe_load(f) or {}
+        name = (data.get("name") or "").strip()
+        if not name:
+            continue
+        refs.append(f"{registry}/{name}:{version}")
+
+    output_file.write_text("\n".join(refs) + ("\n" if refs else ""))
+    print(f"Written {len(refs)} chart ref(s) to {output_file}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()

build/scripts/make_bundle.sh

@@ -2,11 +2,15 @@
 set -ex
 
 AIRGAP_BUNDLE_DIR=${AIRGAP_BUNDLE_DIR:-"./bundle/images/"}
+AIRGAP_BUNDLE_FILE=${AIRGAP_BUNDLE_FILE:-"airgap-bundle-ceph.tar.gz"}
+AIRGAP_BUNDLE_VERSION=${AIRGAP_BUNDLE_VERSION:-"latest"}
+
 FULL_IMAGES_LIST_FILE=${FULL_IMAGES_LIST_FILE:-"images.list"}
 FULL_CHARTS_LIST_FILE=${FULL_CHARTS_LIST_FILE:-"charts.list"}
 SKOPEO_IMG=${SKOPEO_IMG:-"quay.io/skopeo/stable:v1.18.0"}
 DOCKER_CONFIG_PATH=${DOCKER_CONFIG_PATH:-"${HOME}/.docker/config.json"}
-AIRGAP_BUNDLE_FILE=${AIRGAP_BUNDLE_FILE:-"airgap-bundle-ceph.tar.gz"}
+
+PUBLIC_BUCKET='s3://get-mirantis.com/pelagia'
 
 mkdir -p ${AIRGAP_BUNDLE_DIR}
 
@@ -27,3 +31,6 @@ for chart in $(cat ${FULL_CHARTS_LIST_FILE}); do
 done;
 
 tar -czf ${AIRGAP_BUNDLE_FILE} -C ${AIRGAP_BUNDLE_DIR} .
+
+# Upload to get.mirantis.com as a binary
+aws s3 cp "${AIRGAP_BUNDLE_FILE}" "${PUBLIC_BUCKET}/${AIRGAP_BUNDLE_VERSION}/"