From d41b3520b7f997c62a3545e9ac1a4e7ddbfac120 Mon Sep 17 00:00:00 2001 From: Peter Razumovsky Date: Wed, 25 Feb 2026 22:01:43 +0400 Subject: [PATCH] airgap: Script for building images.list for airgap solution Signed-off-by: Peter Razumovsky --- .github/workflows/airgap-bundle.yaml | 70 +++++++++++++++++++ build/scripts/collect_chart_images.py | 90 +++++++++++++++++++++++++ build/scripts/collect_chart_oci_refs.py | 77 +++++++++++++++++++++ build/scripts/make_bundle.sh | 9 ++- 4 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/airgap-bundle.yaml create mode 100755 build/scripts/collect_chart_images.py create mode 100755 build/scripts/collect_chart_oci_refs.py diff --git a/.github/workflows/airgap-bundle.yaml b/.github/workflows/airgap-bundle.yaml new file mode 100644 index 00000000..601f0e00 --- /dev/null +++ b/.github/workflows/airgap-bundle.yaml @@ -0,0 +1,70 @@ +# Build airgap bundle (images + OCI charts) when a tag is pushed. +# Step 1: collect chart OCI refs into charts.list +# Step 2: collect image refs into images.list +# Step 3: run make_bundle.sh (skopeo copy images + charts, then tar) +name: Airgap bundle + +on: + push: + tags: ["*"] + +jobs: + airgap-bundle: + runs-on: ubuntu-latest + env: + REPO_ROOT: ${{ github.workspace }} + # Set repo variable OCI_CHARTS_REGISTRY to override (e.g. ghcr.io/owner/pelagia-charts) + REGISTRY: ${{ github.ref_type == 'tag' && vars.REGISTRY_PROD || vars.REGISTRY_CI }} + DEV_VERSION: ${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || '' }} + HELM_REPO: ${{ github.event_name == 'push' && 'pelagia' || 'pelagia/pr' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install PyYAML + run: pip install pyyaml + + - name: Collect chart OCI refs + run: python3 build/scripts/collect_chart_oci_refs.py + env: + OCI_CHARTS_REGISTRY: ${{ env.REGISTRY }}/${{ env.HELM_REPO }} + OUTPUT_FILE: ${{ github.workspace }}/charts.list + + - name: Collect chart images + run: python3 build/scripts/collect_chart_images.py + env: + IMAGE_REGISTRY: ${{ env.REGISTRY }} + OUTPUT_FILE: ${{ github.workspace }}/images.list + + - name: Get latest tag + id: get_tag + run: | + git fetch --tags + LATEST_TAG=$(git describe --tags --abbrev=0) + echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT + - name: Build airgap bundle + run: bash build/scripts/make_bundle.sh + env: + FULL_IMAGES_LIST_FILE: ${{ github.workspace }}/images.list + FULL_CHARTS_LIST_FILE: ${{ github.workspace }}/charts.list + AIRGAP_BUNDLE_DIR: ${{ github.workspace }}/bundle/images + AIRGAP_BUNDLE_FILE: ${{ github.workspace }}/airgap-bundle-ceph.tar.gz + AIRGAP_BUNDLE_VERSION: ${{ steps.get_tag.outputs.tag }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + # Optional: set DOCKER_CONFIG_PATH if charts/images need private registry auth + # DOCKER_CONFIG_PATH: ${{ github.workspace }}/.docker/config.json + + - name: Upload airgap bundle + uses: actions/upload-artifact@v4 + with: + name: airgap-bundle-${{ github.ref_name }} + path: ${{ github.workspace }}/airgap-bundle-ceph.tar.gz diff --git a/build/scripts/collect_chart_images.py b/build/scripts/collect_chart_images.py new file mode 100755 index 00000000..60b48f32 --- /dev/null +++ b/build/scripts/collect_chart_images.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +""" +Collects all container images in format registry/repo/image:tag from the +"images" section (including nested subsections) of values of all Helm charts +under charts/ and writes them to images.list (unique, sorted). + +Parses values.yaml directly so all repository+tag pairs are collected even +when tag is an object (e.g. latest/squid/tentacle). + +Registry prefix is taken from IMAGE_REGISTRY env (e.g. set from REGISTRY in CI). +Requires: PyYAML (pip install pyyaml) +Usage: run from repo root, or set REPO_ROOT env to the repo root. + Set IMAGE_REGISTRY for the image registry prefix (e.g. registry.example.com). + Override output with OUTPUT_FILE env (default: images.list in repo root). +""" + +import os +import sys +from pathlib import Path + +try: + import yaml +except ImportError: + sys.stderr.write("Error: PyYAML required. Install with: pip install pyyaml\n") + sys.exit(1) + + +def collect_images_from_obj(obj: dict, registry: str, out: list[str]) -> None: + """Recursively collect registry/repo:tag from nodes that have repository and tag.""" + if not isinstance(obj, dict): + return + if "repository" in obj and "tag" in obj: + repo = str(obj["repository"]).strip() + if not repo: + return + tag = obj["tag"] + if isinstance(tag, str): + t = tag.strip() + if t: + ref = f"{registry}/{repo}:{t}" if registry else f"{repo}:{t}" + out.append(ref) + elif isinstance(tag, dict): + for v in tag.values(): + if isinstance(v, str): + t = v.strip() + if t: + ref = f"{registry}/{repo}:{t}" if registry else f"{repo}:{t}" + out.append(ref) + for v in obj.values(): + collect_images_from_obj(v, registry, out) + + +def parse_values_file(path: Path, registry: str) -> list[str]: + """Parse a values.yaml and return list of image references (registry from IMAGE_REGISTRY).""" + with open(path, encoding="utf-8") as f: + data = yaml.safe_load(f) or {} + out: list[str] = [] + if "images" in data: + collect_images_from_obj(data["images"], registry, out) + return out + + +def main() -> None: + script_dir = Path(__file__).resolve().parent + repo_root = Path(os.environ.get("REPO_ROOT", script_dir.parent.parent)) + charts_dir = repo_root / "charts" + output_file = Path(os.environ.get("OUTPUT_FILE", str(repo_root / "images.list"))) + registry = (os.environ.get("IMAGE_REGISTRY") or "").strip() + + all_images: list[str] = [] + for chart_path in sorted(charts_dir.iterdir()): + if not chart_path.is_dir(): + continue + chart_yaml = chart_path / "Chart.yaml" + values_yaml = chart_path / "values.yaml" + if not chart_yaml.exists() or not values_yaml.exists(): + continue + print(f"Processing chart: {chart_path}", file=sys.stderr) + try: + all_images.extend(parse_values_file(values_yaml, registry)) + except Exception as e: + print(f"Warning: failed to parse {values_yaml}: {e}", file=sys.stderr) + + lines = sorted(set(s for s in all_images if s.strip())) + output_file.write_text("\n".join(lines) + ("\n" if lines else "")) + print(f"Written {len(lines)} unique image(s) to {output_file}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/build/scripts/collect_chart_oci_refs.py b/build/scripts/collect_chart_oci_refs.py new file mode 100755 index 00000000..d2a88c10 --- /dev/null +++ b/build/scripts/collect_chart_oci_refs.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +""" +Collects chart names from charts/*/Chart.yaml and writes refs to charts.list: + //: +(no oci:// prefix; make_bundle.sh uses oci: transport when copying.) + +Chart version is taken from VERSION env if set, otherwise from `make get-version` +(run from repo root; same version logic as the rest of the build). + +Requires: PyYAML (pip install pyyaml) +Env: + REPO_ROOT - repo root (default: parent of build/ parent) + OCI_CHARTS_REGISTRY - e.g. ghcr.io/owner/pelagia-charts (no oci:// prefix) + VERSION - chart version (optional; if unset, runs make get-version) + OUTPUT_FILE - output path (default: charts.list in repo root) +""" + +import os +import subprocess +import sys +from pathlib import Path + +try: + import yaml +except ImportError: + sys.stderr.write("Error: PyYAML required. Install with: pip install pyyaml\n") + sys.exit(1) + + +def main() -> None: + script_dir = Path(__file__).resolve().parent + repo_root = Path(os.environ.get("REPO_ROOT", script_dir.parent.parent)) + charts_dir = repo_root / "charts" + registry = (os.environ.get("OCI_CHARTS_REGISTRY") or "").strip() + version = (os.environ.get("VERSION") or "").strip() + output_file = Path(os.environ.get("OUTPUT_FILE", str(repo_root / "charts.list"))) + + if not registry: + sys.stderr.write("Error: OCI_CHARTS_REGISTRY env is required (e.g. ghcr.io/owner/charts)\n") + sys.exit(1) + if not version: + try: + result = subprocess.run( + ["make", "get-version"], + cwd=repo_root, + capture_output=True, + text=True, + check=True, + ) + version = (result.stdout or "").strip() + except (subprocess.CalledProcessError, FileNotFoundError) as e: + sys.stderr.write(f"Error: VERSION env unset and 'make get-version' failed: {e}\n") + sys.exit(1) + if not version: + sys.stderr.write("Error: VERSION env is required or run from repo with 'make get-version' available\n") + sys.exit(1) + + refs: list[str] = [] + for chart_path in sorted(charts_dir.iterdir()): + if not chart_path.is_dir(): + continue + chart_yaml = chart_path / "Chart.yaml" + if not chart_yaml.exists(): + continue + with open(chart_yaml, encoding="utf-8") as f: + data = yaml.safe_load(f) or {} + name = (data.get("name") or "").strip() + if not name: + continue + refs.append(f"{registry}/{name}:{version}") + + output_file.write_text("\n".join(refs) + ("\n" if refs else "")) + print(f"Written {len(refs)} chart ref(s) to {output_file}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/build/scripts/make_bundle.sh b/build/scripts/make_bundle.sh index 8b38d990..da58e242 100644 --- a/build/scripts/make_bundle.sh +++ b/build/scripts/make_bundle.sh @@ -2,11 +2,15 @@ set -ex AIRGAP_BUNDLE_DIR=${AIRGAP_BUNDLE_DIR:-"./bundle/images/"} +AIRGAP_BUNDLE_FILE=${AIRGAP_BUNDLE_FILE:-"airgap-bundle-ceph.tar.gz"} +AIRGAP_BUNDLE_VERSION=${AIRGAP_BUNDLE_VERSION:-"latest"} + FULL_IMAGES_LIST_FILE=${FULL_IMAGES_LIST_FILE:-"images.list"} FULL_CHARTS_LIST_FILE=${FULL_CHARTS_LIST_FILE:-"charts.list"} SKOPEO_IMG=${SKOPEO_IMG:-"quay.io/skopeo/stable:v1.18.0"} DOCKER_CONFIG_PATH=${DOCKER_CONFIG_PATH:-"${HOME}/.docker/config.json"} -AIRGAP_BUNDLE_FILE=${AIRGAP_BUNDLE_FILE:-"airgap-bundle-ceph.tar.gz"} + +PUBLIC_BUCKET='s3://get-mirantis.com/pelagia' mkdir -p ${AIRGAP_BUNDLE_DIR} @@ -27,3 +31,6 @@ for chart in $(cat ${FULL_CHARTS_LIST_FILE}); do done; tar -czf ${AIRGAP_BUNDLE_FILE} -C ${AIRGAP_BUNDLE_DIR} . + +# Upload to get.mirantis.com as a binary +aws s3 cp "${AIRGAP_BUNDLE_FILE}" "${PUBLIC_BUCKET}/${AIRGAP_BUNDLE_VERSION}/"