PG-2558 Add percona-distribution-postgresql-upgrade container to OBS

percona_obs/cmd_sync.py

@@ -36,6 +36,7 @@
     apply_macro_substitution,
     auto_rootprj_env,
     build_package_meta,
+    expand_for_blocks,
     find_projects,
     is_package,
     load_macros,
@@ -181,6 +182,7 @@ def _copy_with_env_subst(
         if apply_macros:
             assert macros is not None
             text = apply_macro_substitution(text, macros, source=src)
+            text = expand_for_blocks(text)
         if apply_env:
             assert env_vars is not None
             text = apply_env_substitution(text, env_vars, source=src)

percona_obs/common.py

@@ -203,6 +203,30 @@ def _replace(m: re.Match) -> str:
     return _MACRO_RE.sub(_replace, text)
 
 
+_FOR_BLOCK_RE = re.compile(
+    r"\{\{FOR (\w+) IN ([^}]+)\}\}\n(.*?)\{\{ENDFOR\}\}\n?",
+    re.DOTALL,
+)
+
+
+def expand_for_blocks(text: str) -> str:
+    """Expand ``{{FOR var IN v1,v2,...}} ... {{ENDFOR}}`` blocks.
+
+    Repeats the enclosed text once per comma-separated value, substituting
+    ``${var}`` with each value in turn, and concatenates the results in
+    place of the block. This is pure text repetition resolved locally before
+    a file is synced to OBS, so OBS itself only ever sees the fully unrolled,
+    literal result — it has no loop construct of its own to expand tokens
+    like a shell ``${var}`` inside a ``RUN`` command.
+    """
+
+    def _expand(m: re.Match) -> str:
+        var, values, body = m.group(1), m.group(2).split(","), m.group(3)
+        return "".join(body.replace(f"${{{var}}}", v.strip()) for v in values)
+
+    return _FOR_BLOCK_RE.sub(_expand, text)
+
+
 def _macros_chain_files(project_path: Path) -> list[Path]:
     """Return the candidate macros.yaml paths from REPO_ROOT down to *project_path*.
 

root/ppg/common/containers/macros.yaml

@@ -0,0 +1,9 @@
+# Macros for the PPG common containers project (images shared across PG major versions).
+# PG_MAJOR_VERSION is the upgrade *target* (the newest supported version); the Dockerfile
+# installs the older major versions it can upgrade from via a hardcoded shell loop instead
+# of a macro, since macros here don't support lists. Bump PG_MAJOR_VERSION/PG_MINOR_VERSION
+# and extend that loop whenever a new PG major version becomes the target.
+- PG_MAJOR_VERSION: 18
+- PG_MINOR_VERSION: 6
+- PG_VERSION: %!{PG_MAJOR_VERSION}.%!{PG_MINOR_VERSION}
+- PPG_RELEASE: 1

root/ppg/staging/18/containers/percona-distribution-postgresql-upgrade/obs/Dockerfile → root/ppg/common/containers/percona-distribution-postgresql-upgrade/obs/Dockerfile

@@ -1,14 +1,14 @@
 #!UseOBSRepositories
 #!BuildVersion: %!{PG_VERSION}-%!{PPG_RELEASE}
-#!BuildTag: percona-distribution-postgresql-upgrade:%!{PG_VERSION}-17-%!{PPG_RELEASE}-<RELEASE>
-#!BuildTag: percona-distribution-postgresql-upgrade:%!{PG_VERSION}-17-%!{PPG_RELEASE}
-#!BuildTag: percona-distribution-postgresql-upgrade:%!{PG_VERSION}-17
-#!BuildTag: percona-distribution-postgresql-upgrade:%!{PG_MAJOR_VERSION}-17
+#!BuildTag: percona-distribution-postgresql-upgrade:%!{PG_VERSION}-%!{PPG_RELEASE}-<RELEASE>
+#!BuildTag: percona-distribution-postgresql-upgrade:%!{PG_VERSION}-%!{PPG_RELEASE}
+#!BuildTag: percona-distribution-postgresql-upgrade:%!{PG_VERSION}
+#!BuildTag: percona-distribution-postgresql-upgrade:%!{PG_MAJOR_VERSION}
 
 FROM percona-ubi-minimal:latest
 
 LABEL name="Percona Distribution for PostgreSQL Upgrade" \
-      description="Upgrade container for Percona Distribution for PostgreSQL. Facilitates major version upgrades from PostgreSQL 17 to %!{PG_MAJOR_VERSION} using pg_upgrade." \
+      description="Upgrade container for Percona Distribution for PostgreSQL. Facilitates major version upgrades from PostgreSQL 14, 15, 16, or 17 to %!{PG_MAJOR_VERSION} using pg_upgrade." \
       vendor="Percona" \
       summary="Percona Distribution for PostgreSQL major version upgrade container" \
       maintainer="Percona Development <info@percona.com>" \
@@ -70,26 +70,38 @@ RUN set -ex; \
         percona-pg_cron_${PPG_MAJOR_VERSION}; \
     microdnf clean all
 
-# Install PG%!{PG_PREV_MAJOR_VERSION} (source version for upgrade) with all extensions including PostGIS
+# Install every older PG major version this image can upgrade from, with the extensions
+# that exist for that version, so a single image can upgrade from any of them to
+# ${PPG_MAJOR_VERSION}. This block is expanded by percona-obs's sync tooling
+# (expand_for_blocks) into one literal, fully-expanded RUN per version before the file
+# ever reaches OBS — OBS's Dockerfile scanner statically resolves package names and
+# cannot expand a real shell loop variable, so what OBS sees is never this block itself.
+{{FOR pg_version IN 17,16,15,14}}
 RUN set -ex; \
     microdnf install -y \
-        percona-postgresql%!{PG_PREV_MAJOR_VERSION}-server \
-        percona-postgresql%!{PG_PREV_MAJOR_VERSION}-contrib \
-        percona-pg_tde%!{PG_PREV_MAJOR_VERSION} \
-        percona-pg_stat_monitor%!{PG_PREV_MAJOR_VERSION} \
-        percona-pg_repack%!{PG_PREV_MAJOR_VERSION} \
-        percona-pgaudit%!{PG_PREV_MAJOR_VERSION} \
-        percona-pgaudit%!{PG_PREV_MAJOR_VERSION}_set_user \
-        percona-pgvector_%!{PG_PREV_MAJOR_VERSION} \
-        percona-postgresql%!{PG_PREV_MAJOR_VERSION}-llvmjit \
-        percona-pgvector_%!{PG_PREV_MAJOR_VERSION}-llvmjit \
-        percona-wal2json%!{PG_PREV_MAJOR_VERSION} \
-        percona-postgis35_%!{PG_PREV_MAJOR_VERSION} \
-        percona-postgis35_%!{PG_PREV_MAJOR_VERSION}-client \
-        percona-postgis35_%!{PG_PREV_MAJOR_VERSION}-gui \
-        percona-postgis35_%!{PG_PREV_MAJOR_VERSION}-llvmjit \
-        percona-postgis35_%!{PG_PREV_MAJOR_VERSION}-utils \
-        percona-pg_cron_%!{PG_PREV_MAJOR_VERSION}; \
+        percona-postgresql${pg_version}-server \
+        percona-postgresql${pg_version}-contrib \
+        percona-postgresql${pg_version}-llvmjit \
+        percona-pg_stat_monitor${pg_version} \
+        percona-pg_repack${pg_version} \
+        percona-pgaudit${pg_version} \
+        percona-pgaudit${pg_version}_set_user \
+        percona-pgvector_${pg_version} \
+        percona-pgvector_${pg_version}-llvmjit \
+        percona-wal2json${pg_version} \
+        percona-postgis35_${pg_version} \
+        percona-postgis35_${pg_version}-client \
+        percona-postgis35_${pg_version}-gui \
+        percona-postgis35_${pg_version}-llvmjit \
+        percona-postgis35_${pg_version}-utils \
+        percona-pg_cron_${pg_version}; \
+    microdnf clean all
+{{ENDFOR}}
+
+# pg_tde is only packaged standalone from PG17 onward, so it's installed separately
+# here rather than inside the loop above.
+RUN set -ex; \
+    microdnf install -y percona-pg_tde17; \
     microdnf clean all; \
     rm -rf /var/cache/dnf /var/cache/yum
 

root/ppg/common/containers/project.yaml

@@ -0,0 +1,122 @@
+title: Percona Common Container Images for PostgreSQL
+description: |
+  This project contains container images shared across all Percona Distribution for PostgreSQL
+  major versions (currently 14-18), such as the major-version upgrade image.
+  We currently build our images based on UBI-8 and UBI-9 base containers.
+
+repositories:
+  - name: ubi8
+    paths:
+      - subproject: ppg:staging:18
+        repository: UBI_8
+      - subproject: ppg:staging:17
+        repository: UBI_8
+      - subproject: ppg:staging:16
+        repository: UBI_8
+      - subproject: ppg:staging:15
+        repository: UBI_8
+      - subproject: ppg:staging:14
+        repository: UBI_8
+      - subproject: ppg:common:deps
+        repository: UBI_8
+      - subproject: common:containers:ubi8
+        repository: images
+      - subproject: common:containers:ubi8
+        repository: UBI_8
+      - project: ${REMOTE_OBS_ORG_INTERCONNECT}Fedora:EPEL:8
+        repository: standard
+      - project: ${REMOTE_OBS_ORG_INTERCONNECT}RedHat:UBI-8
+        repository: appstream
+      - project: ${REMOTE_OBS_ORG_INTERCONNECT}RedHat:UBI-8
+        repository: baseos
+      - project: ${REMOTE_OBS_ORG_INTERCONNECT}RockyLinux:8
+        repository: appstream
+      - project: ${REMOTE_OBS_ORG_INTERCONNECT}RockyLinux:8
+        repository: baseos
+      - project: ${REMOTE_OBS_ORG_INTERCONNECT}RockyLinux:8
+        repository: devel
+    archs: [x86_64, aarch64]
+  - name: ubi9
+    paths:
+      - subproject: ppg:staging:18
+        repository: UBI_9
+      - subproject: ppg:staging:17
+        repository: UBI_9
+      - subproject: ppg:staging:16
+        repository: UBI_9
+      - subproject: ppg:staging:15
+        repository: UBI_9
+      - subproject: ppg:staging:14
+        repository: UBI_9
+      - subproject: ppg:common:deps
+        repository: UBI_9
+      - subproject: common:containers:ubi9
+        repository: images
+      - subproject: common:containers:ubi9
+        repository: UBI_9
+      - project: ${REMOTE_OBS_ORG_INTERCONNECT}Fedora:EPEL:9
+        repository: standard
+      - project: ${REMOTE_OBS_ORG_INTERCONNECT}RedHat:UBI-9
+        repository: standard
+    archs: [x86_64, aarch64]
+
+project-config: |
+  Type: docker
+  BuildEngine: podman
+  BuildFlags: sbom:spdx
+  BuildFlags: sbom:cyclonedx
+  PublishFlags: withsbom
+
+  Preinstall: skopeo
+
+  ExpandFlags: filterbasecontainerpkgs
+
+  Prefer: redhat-release
+  Prefer: glibc-minimal-langpack
+  Prefer: gpgme
+  Prefer: iptables-nft
+  Prefer: percona-postgresql%!{PG_MAJOR_VERSION}-libs
+  Ignore: rocky-release
+  Substitute: rocky-release redhat-release
+
+  BuildFlags: dockerarg:PPG_REPO=staging
+
+  %if "%_repository" == "ubi8"
+
+  ExpandFlags: module:container-tools-rhel8
+
+  BuildFlags: dockerarg:RHEL_VER=el8
+
+  %endif
+
+  %if "%_repository" == "ubi9"
+
+  BuildFlags: dockerarg:RHEL_VER=el9
+
+  %endif
+
+
+qa:
+  pipeline: docker-server-parallel-generic
+  parameters:
+    DOCKER_TAG: "%!{PG_MAJOR_VERSION}"
+    SERVER_VERSION: "%!{PG_VERSION}"
+    TESTING_BRANCH: ricardo-work
+    REPOSITORY: registry.opensuse.org/${OBS_CONTAINER_REGISTRY_ROOTPRJ}/ppg/common/containers/ubi8
+    WITH_POSTGIS:
+      - true
+      - false
+  matrix:
+    - WITH_POSTGIS
+
+  pipeline: docker-server-parallel-generic
+  parameters:
+    DOCKER_TAG: "%!{PG_MAJOR_VERSION}"
+    SERVER_VERSION: "%!{PG_VERSION}"
+    TESTING_BRANCH: ricardo-work
+    REPOSITORY: registry.opensuse.org/${OBS_CONTAINER_REGISTRY_ROOTPRJ}/ppg/common/containers/ubi9
+    WITH_POSTGIS:
+      - true
+      - false
+  matrix:
+    - WITH_POSTGIS

root/ppg/staging/18/containers/project.yaml

@@ -8,8 +8,6 @@ repositories:
     paths:
       - subproject: ppg:staging:%!{PG_MAJOR_VERSION}
         repository: UBI_8
-      - subproject: ppg:staging:%!{PG_PREV_MAJOR_VERSION}
-        repository: UBI_8
       - subproject: ppg:common:deps
         repository: UBI_8
       - subproject: common:containers:ubi8
@@ -33,8 +31,6 @@ repositories:
     paths:
       - subproject: ppg:staging:%!{PG_MAJOR_VERSION}
         repository: UBI_9
-      - subproject: ppg:staging:%!{PG_PREV_MAJOR_VERSION}
-        repository: UBI_9
       - subproject: ppg:common:deps
         repository: UBI_9
       - subproject: common:containers:ubi9

tests/test_for_block_expansion.py

@@ -0,0 +1,45 @@
+"""Unit tests for ``expand_for_blocks`` (percona_obs.common).
+
+Dockerfiles synced to OBS need every package name in a ``RUN <pkgmgr> install``
+line to be a literal token: OBS statically scans those lines to pre-resolve
+build dependencies, and cannot expand a real shell loop variable (see the
+percona-distribution-postgresql-upgrade Dockerfile, which needs a near-
+identical install block repeated once per older PG major version). This
+``{{FOR var IN v1,v2,...}} ... {{ENDFOR}}`` block lets that block be authored
+once and expanded into fully literal, repeated text before the file is synced,
+so OBS only ever sees plain text with no loop construct in it.
+"""
+
+from percona_obs.common import expand_for_blocks
+
+
+def test_no_for_block_is_unchanged():
+    text = "FROM foo\nRUN microdnf install -y bar\n"
+    assert expand_for_blocks(text) == text
+
+
+def test_single_substitution_repeated_per_value():
+    text = "{{FOR v IN 17,16,14}}\nRUN install pkg${v}\n{{ENDFOR}}\n"
+    assert expand_for_blocks(text) == (
+        "RUN install pkg17\nRUN install pkg16\nRUN install pkg14\n"
+    )
+
+
+def test_multiple_occurrences_of_var_in_one_block():
+    text = "{{FOR v IN 17,16}}\nRUN a${v} && b${v}\n{{ENDFOR}}\n"
+    assert expand_for_blocks(text) == "RUN a17 && b17\nRUN a16 && b16\n"
+
+
+def test_text_outside_block_is_preserved():
+    text = "before\n{{FOR v IN 1,2}}\nline${v}\n{{ENDFOR}}\nafter\n"
+    assert expand_for_blocks(text) == "before\nline1\nline2\nafter\n"
+
+
+def test_multiple_blocks_in_one_file():
+    text = "{{FOR v IN 1,2}}\nA${v}\n{{ENDFOR}}\n{{FOR v IN 9,8}}\nB${v}\n{{ENDFOR}}\n"
+    assert expand_for_blocks(text) == "A1\nA2\nB9\nB8\n"
+
+
+def test_values_may_have_surrounding_whitespace():
+    text = "{{FOR v IN 17, 16 , 14}}\npkg${v}\n{{ENDFOR}}\n"
+    assert expand_for_blocks(text) == "pkg17\npkg16\npkg14\n"