Skip to content

Commit db41203

Browse files
surbhat1595surbhat1595
committed
PG-2558 Add percona-distribution-postgresql-upgrade container to OBS
 Move percona-distribution-postgresql-upgrade container to ppg:common:containers Modify percona-distribution-postgresql-upgrade Dockerfile to include PG 14-18 versions
1 parent 487bbfe commit db41203

9 files changed

Lines changed: 237 additions & 27 deletions

File tree

‎percona_obs/cmd_sync.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
apply_macro_substitution,
3737
auto_rootprj_env,
3838
build_package_meta,
39+
expand_for_blocks,
3940
find_projects,
4041
is_package,
4142
load_macros,
@@ -181,6 +182,7 @@ def _copy_with_env_subst(
181182
if apply_macros:
182183
assert macros is not None
183184
text = apply_macro_substitution(text, macros, source=src)
185+
text = expand_for_blocks(text)
184186
if apply_env:
185187
assert env_vars is not None
186188
text = apply_env_substitution(text, env_vars, source=src)

‎percona_obs/common.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,30 @@ def _replace(m: re.Match) -> str:
203203
return _MACRO_RE.sub(_replace, text)
204204

205205

206+
_FOR_BLOCK_RE = re.compile(
207+
r"\{\{FOR (\w+) IN ([^}]+)\}\}\n(.*?)\{\{ENDFOR\}\}\n?",
208+
re.DOTALL,
209+
)
210+
211+
212+
def expand_for_blocks(text: str) -> str:
213+
"""Expand ``{{FOR var IN v1,v2,...}} ... {{ENDFOR}}`` blocks.
214+
215+
Repeats the enclosed text once per comma-separated value, substituting
216+
``${var}`` with each value in turn, and concatenates the results in
217+
place of the block. This is pure text repetition resolved locally before
218+
a file is synced to OBS, so OBS itself only ever sees the fully unrolled,
219+
literal result — it has no loop construct of its own to expand tokens
220+
like a shell ``${var}`` inside a ``RUN`` command.
221+
"""
222+
223+
def _expand(m: re.Match) -> str:
224+
var, values, body = m.group(1), m.group(2).split(","), m.group(3)
225+
return "".join(body.replace(f"${{{var}}}", v.strip()) for v in values)
226+
227+
return _FOR_BLOCK_RE.sub(_expand, text)
228+
229+
206230
def _macros_chain_files(project_path: Path) -> list[Path]:
207231
"""Return the candidate macros.yaml paths from REPO_ROOT down to *project_path*.
208232

‎root/ppg/common/containers/macros.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Macros for the PPG common containers project (images shared across PG major versions).
2+
# PG_MAJOR_VERSION is the upgrade *target* (the newest supported version); the Dockerfile
3+
# installs the older major versions it can upgrade from via a hardcoded shell loop instead
4+
# of a macro, since macros here don't support lists. Bump PG_MAJOR_VERSION/PG_MINOR_VERSION
5+
# and extend that loop whenever a new PG major version becomes the target.
6+
- PG_MAJOR_VERSION: 18
7+
- PG_MINOR_VERSION: 6
8+
- PG_VERSION: %!{PG_MAJOR_VERSION}.%!{PG_MINOR_VERSION}
9+
- PPG_RELEASE: 1

root/ppg/staging/18/containers/percona-distribution-postgresql-upgrade/obs/Dockerfile renamed to root/ppg/common/containers/percona-distribution-postgresql-upgrade/obs/Dockerfile

Lines changed: 35 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
#!UseOBSRepositories
22
#!BuildVersion: %!{PG_VERSION}-%!{PPG_RELEASE}
3-
#!BuildTag: percona-distribution-postgresql-upgrade:%!{PG_VERSION}-17-%!{PPG_RELEASE}-<RELEASE>
4-
#!BuildTag: percona-distribution-postgresql-upgrade:%!{PG_VERSION}-17-%!{PPG_RELEASE}
5-
#!BuildTag: percona-distribution-postgresql-upgrade:%!{PG_VERSION}-17
6-
#!BuildTag: percona-distribution-postgresql-upgrade:%!{PG_MAJOR_VERSION}-17
3+
#!BuildTag: percona-distribution-postgresql-upgrade:%!{PG_VERSION}-%!{PPG_RELEASE}-<RELEASE>
4+
#!BuildTag: percona-distribution-postgresql-upgrade:%!{PG_VERSION}-%!{PPG_RELEASE}
5+
#!BuildTag: percona-distribution-postgresql-upgrade:%!{PG_VERSION}
6+
#!BuildTag: percona-distribution-postgresql-upgrade:%!{PG_MAJOR_VERSION}
77

88
FROM percona-ubi-minimal:latest
99

1010
LABEL name="Percona Distribution for PostgreSQL Upgrade" \
11-
description="Upgrade container for Percona Distribution for PostgreSQL. Facilitates major version upgrades from PostgreSQL 17 to %!{PG_MAJOR_VERSION} using pg_upgrade." \
11+
description="Upgrade container for Percona Distribution for PostgreSQL. Facilitates major version upgrades from PostgreSQL 14, 15, 16, or 17 to %!{PG_MAJOR_VERSION} using pg_upgrade." \
1212
vendor="Percona" \
1313
summary="Percona Distribution for PostgreSQL major version upgrade container" \
1414
maintainer="Percona Development <info@percona.com>" \
@@ -70,26 +70,38 @@ RUN set -ex; \
7070
percona-pg_cron_${PPG_MAJOR_VERSION}; \
7171
microdnf clean all
7272

73-
# Install PG%!{PG_PREV_MAJOR_VERSION} (source version for upgrade) with all extensions including PostGIS
73+
# Install every older PG major version this image can upgrade from, with the extensions
74+
# that exist for that version, so a single image can upgrade from any of them to
75+
# ${PPG_MAJOR_VERSION}. This block is expanded by percona-obs's sync tooling
76+
# (expand_for_blocks) into one literal, fully-expanded RUN per version before the file
77+
# ever reaches OBS — OBS's Dockerfile scanner statically resolves package names and
78+
# cannot expand a real shell loop variable, so what OBS sees is never this block itself.
79+
{{FOR pg_version IN 17,16,15,14}}
7480
RUN set -ex; \
7581
microdnf install -y \
76-
percona-postgresql%!{PG_PREV_MAJOR_VERSION}-server \
77-
percona-postgresql%!{PG_PREV_MAJOR_VERSION}-contrib \
78-
percona-pg_tde%!{PG_PREV_MAJOR_VERSION} \
79-
percona-pg_stat_monitor%!{PG_PREV_MAJOR_VERSION} \
80-
percona-pg_repack%!{PG_PREV_MAJOR_VERSION} \
81-
percona-pgaudit%!{PG_PREV_MAJOR_VERSION} \
82-
percona-pgaudit%!{PG_PREV_MAJOR_VERSION}_set_user \
83-
percona-pgvector_%!{PG_PREV_MAJOR_VERSION} \
84-
percona-postgresql%!{PG_PREV_MAJOR_VERSION}-llvmjit \
85-
percona-pgvector_%!{PG_PREV_MAJOR_VERSION}-llvmjit \
86-
percona-wal2json%!{PG_PREV_MAJOR_VERSION} \
87-
percona-postgis35_%!{PG_PREV_MAJOR_VERSION} \
88-
percona-postgis35_%!{PG_PREV_MAJOR_VERSION}-client \
89-
percona-postgis35_%!{PG_PREV_MAJOR_VERSION}-gui \
90-
percona-postgis35_%!{PG_PREV_MAJOR_VERSION}-llvmjit \
91-
percona-postgis35_%!{PG_PREV_MAJOR_VERSION}-utils \
92-
percona-pg_cron_%!{PG_PREV_MAJOR_VERSION}; \
82+
percona-postgresql${pg_version}-server \
83+
percona-postgresql${pg_version}-contrib \
84+
percona-postgresql${pg_version}-llvmjit \
85+
percona-pg_stat_monitor${pg_version} \
86+
percona-pg_repack${pg_version} \
87+
percona-pgaudit${pg_version} \
88+
percona-pgaudit${pg_version}_set_user \
89+
percona-pgvector_${pg_version} \
90+
percona-pgvector_${pg_version}-llvmjit \
91+
percona-wal2json${pg_version} \
92+
percona-postgis35_${pg_version} \
93+
percona-postgis35_${pg_version}-client \
94+
percona-postgis35_${pg_version}-gui \
95+
percona-postgis35_${pg_version}-llvmjit \
96+
percona-postgis35_${pg_version}-utils \
97+
percona-pg_cron_${pg_version}; \
98+
microdnf clean all
99+
{{ENDFOR}}
100+
101+
# pg_tde is only packaged standalone from PG17 onward, so it's installed separately
102+
# here rather than inside the loop above.
103+
RUN set -ex; \
104+
microdnf install -y percona-pg_tde17; \
93105
microdnf clean all; \
94106
rm -rf /var/cache/dnf /var/cache/yum
95107

root/ppg/staging/18/containers/percona-distribution-postgresql-upgrade/obs/LICENSE renamed to root/ppg/common/containers/percona-distribution-postgresql-upgrade/obs/LICENSE

File renamed without changes.

root/ppg/staging/18/containers/percona-distribution-postgresql-upgrade/obs/upgrade-scripts.tar.gz renamed to root/ppg/common/containers/percona-distribution-postgresql-upgrade/obs/upgrade-scripts.tar.gz

File renamed without changes.

‎root/ppg/common/containers/project.yaml‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
title: Percona Common Container Images for PostgreSQL
2+
description: |
3+
This project contains container images shared across all Percona Distribution for PostgreSQL
4+
major versions (currently 14-18), such as the major-version upgrade image.
5+
We currently build our images based on UBI-8 and UBI-9 base containers.
6+
7+
repositories:
8+
- name: ubi8
9+
paths:
10+
- subproject: ppg:staging:18
11+
repository: UBI_8
12+
- subproject: ppg:staging:17
13+
repository: UBI_8
14+
- subproject: ppg:staging:16
15+
repository: UBI_8
16+
- subproject: ppg:staging:15
17+
repository: UBI_8
18+
- subproject: ppg:staging:14
19+
repository: UBI_8
20+
- subproject: ppg:common:deps
21+
repository: UBI_8
22+
- subproject: common:containers:ubi8
23+
repository: images
24+
- subproject: common:containers:ubi8
25+
repository: UBI_8
26+
- project: ${REMOTE_OBS_ORG_INTERCONNECT}Fedora:EPEL:8
27+
repository: standard
28+
- project: ${REMOTE_OBS_ORG_INTERCONNECT}RedHat:UBI-8
29+
repository: appstream
30+
- project: ${REMOTE_OBS_ORG_INTERCONNECT}RedHat:UBI-8
31+
repository: baseos
32+
- project: ${REMOTE_OBS_ORG_INTERCONNECT}RockyLinux:8
33+
repository: appstream
34+
- project: ${REMOTE_OBS_ORG_INTERCONNECT}RockyLinux:8
35+
repository: baseos
36+
- project: ${REMOTE_OBS_ORG_INTERCONNECT}RockyLinux:8
37+
repository: devel
38+
archs: [x86_64, aarch64]
39+
- name: ubi9
40+
paths:
41+
- subproject: ppg:staging:18
42+
repository: UBI_9
43+
- subproject: ppg:staging:17
44+
repository: UBI_9
45+
- subproject: ppg:staging:16
46+
repository: UBI_9
47+
- subproject: ppg:staging:15
48+
repository: UBI_9
49+
- subproject: ppg:staging:14
50+
repository: UBI_9
51+
- subproject: ppg:common:deps
52+
repository: UBI_9
53+
- subproject: common:containers:ubi9
54+
repository: images
55+
- subproject: common:containers:ubi9
56+
repository: UBI_9
57+
- project: ${REMOTE_OBS_ORG_INTERCONNECT}Fedora:EPEL:9
58+
repository: standard
59+
- project: ${REMOTE_OBS_ORG_INTERCONNECT}RedHat:UBI-9
60+
repository: standard
61+
archs: [x86_64, aarch64]
62+
63+
project-config: |
64+
Type: docker
65+
BuildEngine: podman
66+
BuildFlags: sbom:spdx
67+
BuildFlags: sbom:cyclonedx
68+
PublishFlags: withsbom
69+
70+
Preinstall: skopeo
71+
72+
ExpandFlags: filterbasecontainerpkgs
73+
74+
Prefer: redhat-release
75+
Prefer: glibc-minimal-langpack
76+
Prefer: gpgme
77+
Prefer: iptables-nft
78+
Prefer: percona-postgresql%!{PG_MAJOR_VERSION}-libs
79+
Ignore: rocky-release
80+
Substitute: rocky-release redhat-release
81+
82+
BuildFlags: dockerarg:PPG_REPO=staging
83+
84+
%if "%_repository" == "ubi8"
85+
86+
ExpandFlags: module:container-tools-rhel8
87+
88+
BuildFlags: dockerarg:RHEL_VER=el8
89+
90+
%endif
91+
92+
%if "%_repository" == "ubi9"
93+
94+
BuildFlags: dockerarg:RHEL_VER=el9
95+
96+
%endif
97+
98+
99+
qa:
100+
pipeline: docker-server-parallel-generic
101+
parameters:
102+
DOCKER_TAG: "%!{PG_MAJOR_VERSION}"
103+
SERVER_VERSION: "%!{PG_VERSION}"
104+
TESTING_BRANCH: ricardo-work
105+
REPOSITORY: registry.opensuse.org/${OBS_CONTAINER_REGISTRY_ROOTPRJ}/ppg/common/containers/ubi8
106+
WITH_POSTGIS:
107+
- true
108+
- false
109+
matrix:
110+
- WITH_POSTGIS
111+
112+
pipeline: docker-server-parallel-generic
113+
parameters:
114+
DOCKER_TAG: "%!{PG_MAJOR_VERSION}"
115+
SERVER_VERSION: "%!{PG_VERSION}"
116+
TESTING_BRANCH: ricardo-work
117+
REPOSITORY: registry.opensuse.org/${OBS_CONTAINER_REGISTRY_ROOTPRJ}/ppg/common/containers/ubi9
118+
WITH_POSTGIS:
119+
- true
120+
- false
121+
matrix:
122+
- WITH_POSTGIS

‎root/ppg/staging/18/containers/project.yaml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@ repositories:
88
paths:
99
- subproject: ppg:staging:%!{PG_MAJOR_VERSION}
1010
repository: UBI_8
11-
- subproject: ppg:staging:%!{PG_PREV_MAJOR_VERSION}
12-
repository: UBI_8
1311
- subproject: ppg:common:deps
1412
repository: UBI_8
1513
- subproject: common:containers:ubi8
@@ -33,8 +31,6 @@ repositories:
3331
paths:
3432
- subproject: ppg:staging:%!{PG_MAJOR_VERSION}
3533
repository: UBI_9
36-
- subproject: ppg:staging:%!{PG_PREV_MAJOR_VERSION}
37-
repository: UBI_9
3834
- subproject: ppg:common:deps
3935
repository: UBI_9
4036
- subproject: common:containers:ubi9

‎tests/test_for_block_expansion.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
"""Unit tests for ``expand_for_blocks`` (percona_obs.common).
2+
3+
Dockerfiles synced to OBS need every package name in a ``RUN <pkgmgr> install``
4+
line to be a literal token: OBS statically scans those lines to pre-resolve
5+
build dependencies, and cannot expand a real shell loop variable (see the
6+
percona-distribution-postgresql-upgrade Dockerfile, which needs a near-
7+
identical install block repeated once per older PG major version). This
8+
``{{FOR var IN v1,v2,...}} ... {{ENDFOR}}`` block lets that block be authored
9+
once and expanded into fully literal, repeated text before the file is synced,
10+
so OBS only ever sees plain text with no loop construct in it.
11+
"""
12+
13+
from percona_obs.common import expand_for_blocks
14+
15+
16+
def test_no_for_block_is_unchanged():
17+
text = "FROM foo\nRUN microdnf install -y bar\n"
18+
assert expand_for_blocks(text) == text
19+
20+
21+
def test_single_substitution_repeated_per_value():
22+
text = "{{FOR v IN 17,16,14}}\nRUN install pkg${v}\n{{ENDFOR}}\n"
23+
assert expand_for_blocks(text) == (
24+
"RUN install pkg17\nRUN install pkg16\nRUN install pkg14\n"
25+
)
26+
27+
28+
def test_multiple_occurrences_of_var_in_one_block():
29+
text = "{{FOR v IN 17,16}}\nRUN a${v} && b${v}\n{{ENDFOR}}\n"
30+
assert expand_for_blocks(text) == "RUN a17 && b17\nRUN a16 && b16\n"
31+
32+
33+
def test_text_outside_block_is_preserved():
34+
text = "before\n{{FOR v IN 1,2}}\nline${v}\n{{ENDFOR}}\nafter\n"
35+
assert expand_for_blocks(text) == "before\nline1\nline2\nafter\n"
36+
37+
38+
def test_multiple_blocks_in_one_file():
39+
text = "{{FOR v IN 1,2}}\nA${v}\n{{ENDFOR}}\n{{FOR v IN 9,8}}\nB${v}\n{{ENDFOR}}\n"
40+
assert expand_for_blocks(text) == "A1\nA2\nB9\nB8\n"
41+
42+
43+
def test_values_may_have_surrounding_whitespace():
44+
text = "{{FOR v IN 17, 16 , 14}}\npkg${v}\n{{ENDFOR}}\n"
45+
assert expand_for_blocks(text) == "pkg17\npkg16\npkg14\n"

0 commit comments

Comments
 (0)