Skip to content

Commit 011955a

Browse files
Merge pull request #424 from Quetzacoalt91/add-backlog-generation
Add generator of backlog based on Distribution API contents
2 parents 2bf8a5e + 5531a27 commit 011955a

14 files changed

Lines changed: 1548 additions & 78 deletions
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: Extract PrestaShop releases and sync backlog of images to generate
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
- cron: "0 2 * * *"
7+
8+
jobs:
9+
sync-releases:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: Checkout the repo
13+
uses: actions/checkout@v2
14+
- name: Set up Python 3.8
15+
uses: actions/setup-python@v2
16+
with:
17+
python-version: "3.8"
18+
- name: Install dependencies
19+
run: |
20+
python -m pip install --upgrade pip
21+
pip install -r requirements.txt
22+
- name: Update backlog
23+
run: python prestashop_docker.py backlog
24+
- name: Create Pull Request
25+
uses: peter-evans/create-pull-request@v4
26+
with:
27+
commit-message: Update versions.py with new images to generate
28+
title: Sync backlog of Docker images
29+
body: Update of the versions.py contents based on releases published on Distribution API.
30+
base: master
31+
branch: docker-images-backlog-update
32+
delete-branch: true

Dockerfile-nightly.model

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
FROM prestashop/base:$container_version
22
LABEL maintainer="PrestaShop Core Team <coreteam@prestashop.com>"
33

4-
ENV PS_VERSION $ps_version
4+
ENV PS_VERSION=$ps_version
55

66
ENV PATH /root/google-cloud-sdk/bin/:$$PATH
77

Dockerfile.model

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
FROM prestashop/base:$container_version
22
LABEL maintainer="PrestaShop Core Team <coreteam@prestashop.com>"
33

4-
ENV PS_VERSION $ps_version
4+
ENV PS_VERSION=$ps_version
55

66
# Get PrestaShop
77
ADD $ps_url /tmp/prestashop.zip

HOW-TO-USE.md

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,33 @@ Display the help:
1515

1616
```bash
1717
$ ./prestashop_docker.py --help
18-
usage: prestashop_docker.py [-h] [--debug] [--cache] [subcommand] ...
18+
usage: prestashop_docker.py [-h] [--debug] [--quiet] [--cache] [subcommand] ...
1919

2020
PrestaShop Docker manager.
2121

2222
positional arguments:
2323
[subcommand]
24-
tag Tag managment
24+
backlog Update backlog of stable versions of PrestaShop to build from data on Distribution API
2525
generate Generate Dockerfile
26+
tag Tag managment
2627

27-
optional arguments:
28+
options:
2829
-h, --help show this help message and exit
2930
--debug Use Debug
30-
--cache Enable cache
31+
--quiet Use Debug
32+
--cache Enable cache
33+
```
34+
35+
### Backlog
36+
37+
Updates the info about stable releases of PrestaShop inside versions.py, based on the contents of Ditribution API.
38+
39+
```bash
40+
./prestashop_docker.py backlog --help
41+
usage: prestashop_docker.py backlog [-h]
42+
43+
optional arguments:
44+
-h, --help show this help message and exit
3145
```
3246

3347
### Generate
@@ -63,7 +77,7 @@ optional arguments:
6377
6478
## Using docker compose
6579
66-
To generate the new base files, you need to update the `versions.py` file, add a section ith your new version along with the associated PHP versions, then run
80+
To generate the new base files, you need to update the `versions.py` file, add a section with your new version along with the associated PHP versions, then run
6781
6882
```php
6983
docker compose up generate

prestashop_docker.py

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@
22

33
import docker
44
from versions import VERSIONS
5+
from prestashop_docker.backlog import Backlog
56
from prestashop_docker.generator import Generator
67
from prestashop_docker.tag_manager import TagManager
78
from prestashop_docker.docker_api import DockerApi
9+
from prestashop_docker.distribution_api import DistributionApi
810
from prestashop_docker.version_manager import VersionManager
911
from os import path
1012
import argparse
@@ -74,20 +76,43 @@ def get_generate_parser(subparser):
7476
return generate_parser
7577

7678

79+
def get_backlog_parser(subparser):
80+
backlog_parser = subparser.add_parser(
81+
'backlog',
82+
help='Update backlog of stable versions of PrestaShop to build from data on Distribution API'
83+
)
84+
85+
return backlog_parser
86+
87+
7788
def main():
7889
parser = get_parser()
7990
subparser = get_subparser(parser)
80-
tag_parser = get_tag_parser(subparser)
91+
get_backlog_parser(subparser)
8192
get_generate_parser(subparser)
93+
tag_parser = get_tag_parser(subparser)
8294

8395
args = parser.parse_args()
8496

8597
logging.basicConfig()
8698
if args.debug:
8799
logging.getLogger().setLevel(logging.DEBUG)
88100

89-
if args.subcommand == 'generate':
101+
if args.subcommand == 'backlog':
102+
version_manager = VersionManager(path.join(path.dirname(path.realpath(__file__)), 'images'))
103+
backlog = Backlog(
104+
DockerApi(args.cache, args.debug),
105+
docker.from_env(),
106+
DistributionApi(version_manager),
107+
version_manager,
108+
previous_state_versions=VERSIONS,
109+
nightly_const=Generator.NIGHTLY
110+
)
111+
backlog.generate()
112+
elif args.subcommand == 'generate':
113+
version_manager = VersionManager(path.join(path.dirname(path.realpath(__file__)), 'images'))
90114
generator = Generator(
115+
DistributionApi(version_manager),
91116
path.join(path.dirname(path.realpath(__file__)), 'images'),
92117
open('./Dockerfile.model').read(),
93118
open('./Dockerfile-nightly.model').read(),

prestashop_docker/backlog.py

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
import re
2+
from packaging.version import Version
3+
4+
5+
class Backlog:
6+
MINIMUM_PRESTASHOP_VERSION_TO_BUILD = '1.7.7.0'
7+
8+
def __init__(self, docker_api, docker_client, distribution_api, version_manager, previous_state_versions, nightly_const):
9+
"""Constructor
10+
11+
@param docker_api: Customer docker API
12+
@type docker_api: DockerApi
13+
@param docker_client: Docker client
14+
@type docker_client: docker
15+
@param distribution_api: Distribution API
16+
@type distribution_api: DistributionApi
17+
@param version_manager: Version Manager
18+
@type version_manager: VersionManager
19+
@param previous_state_versions: Contents of the version.py file before regeneration
20+
@type previous_state_versions: dict
21+
@param nightly_const: Value of nightly case
22+
@type nightly_const: string
23+
"""
24+
self.docker_api = docker_api
25+
self.docker_client = docker_client
26+
self.distribution_api = distribution_api
27+
self.version_manager = version_manager
28+
self.previous_state_versions = previous_state_versions
29+
self.NIGHTLY = nightly_const
30+
31+
def generate(self):
32+
print('Refreshing contents of versions.py')
33+
print('Step 1/3: Fetching PHP tags from Docker Hub API')
34+
available_php_versions = self.get_available_php_versions()
35+
print('Step 2/3: Fetching PrestaShop releases from Distribution API')
36+
prestashop_data = self.distribution_api.fetch_prestashop_versions()
37+
38+
print('Step 3/3: Building list')
39+
versions_dict = self.parse_prestashop_versions(prestashop_data, available_php_versions)
40+
branches_dict = self.get_branches_and_nightly_from_existing_file()
41+
self.write_versions_py(versions_dict | branches_dict)
42+
43+
def get_available_php_versions(self):
44+
tags = self.docker_api.get_tags(image_name='library/php')
45+
available_versions = set()
46+
for tag in tags:
47+
match = re.match(r'^(\d+\.\d+)-apache$', tag['name'])
48+
if match:
49+
available_versions.add(match.group(1))
50+
return available_versions
51+
52+
# Branches and nightly entries are manually added in versions.py file.
53+
# Let's reuse the existing contents on each generation.
54+
def get_branches_and_nightly_from_existing_file(self):
55+
branches = {}
56+
for branch, php_versions in self.previous_state_versions.items():
57+
if branch == self.NIGHTLY or branch.endswith('x'):
58+
branches[branch] = (tuple(php_versions))
59+
return branches
60+
61+
def parse_prestashop_versions(self, prestashop_json, available_php_versions):
62+
versions = {}
63+
for entry in prestashop_json:
64+
65+
if Version(entry.get('version')) < Version(self.MINIMUM_PRESTASHOP_VERSION_TO_BUILD):
66+
continue
67+
68+
ps_version = self.version_manager.create_version_from_distribution_api(entry['version'], entry['distribution'], entry['distribution_version'])
69+
php_min = entry['php_min_version']
70+
php_max = entry['php_max_version']
71+
72+
min_major, min_minor = map(int, php_min.split('.')[:2])
73+
max_major, max_minor = map(int, php_max.split('.')[:2])
74+
75+
compatible_php_versions = []
76+
77+
current_major, current_minor = min_major, min_minor
78+
while (current_major, current_minor) <= (max_major, max_minor):
79+
version_str = f"{current_major}.{current_minor}"
80+
if version_str in available_php_versions:
81+
compatible_php_versions.append(version_str)
82+
# Increment minor version
83+
current_minor += 1
84+
if current_minor >= 10:
85+
current_minor = 0
86+
current_major += 1
87+
88+
versions[ps_version] = (tuple(compatible_php_versions))
89+
90+
return versions
91+
92+
def write_versions_py(self, versions, output_path='versions.py'):
93+
with open(output_path, 'w') as f:
94+
f.write("VERSIONS = {\n")
95+
for ps_version in sorted(versions):
96+
php_versions = versions[ps_version]
97+
f.write(f" '{ps_version}': (\n")
98+
for php_version in php_versions:
99+
f.write(f" '{php_version}',\n") # noqa: E231
100+
f.write(" ),\n")
101+
f.write("}\n")
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import requests
2+
3+
4+
class DistributionApi():
5+
PRESTASHOP_API_URL = "https://api.prestashop-project.org/prestashop"
6+
7+
def __init__(self, version_manager, data=None):
8+
self.version_manager = version_manager
9+
self.prestashop_versions = data
10+
11+
def fetch_prestashop_versions(self):
12+
if self.prestashop_versions is None:
13+
response = requests.get(self.PRESTASHOP_API_URL)
14+
response.raise_for_status()
15+
self.prestashop_versions = response.json()
16+
17+
return self.prestashop_versions
18+
19+
def get_download_url_of(self, version):
20+
prestashop_versions = self.fetch_prestashop_versions()
21+
22+
for entry in prestashop_versions:
23+
if version == self.version_manager.create_version_from_distribution_api(entry.get('version'), entry.get('distribution'), entry.get('distribution_version')):
24+
return entry.get('zip_download_url')

prestashop_docker/docker_api.py

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@
99
ssl._create_default_https_context = ssl._create_unverified_context
1010

1111

12-
# Docker api
13-
#
1412
class DockerApi():
1513
retries = 0
1614

@@ -23,14 +21,14 @@ def __init__(self, cache, debug):
2321
@type debug: bool
2422
"""
2523
self.sleep_time = 1
26-
self.url = 'https://hub.docker.com/v2/repositories/prestashop/prestashop'
24+
self.url = 'https://hub.docker.com/v2/repositories/'
2725
self.cache = cache
2826
self.is_debug = debug
2927

3028
if self.cache:
3129
requests_cache.install_cache('cache')
3230

33-
def get_tags(self):
31+
def get_tags(self, image_name='prestashop/prestashop'):
3432
"""Generate return tags
3533
3634
@return: The json content
@@ -42,7 +40,7 @@ def get_tags(self):
4240
)
4341

4442
data = self.execute(
45-
self.url + '/tags'
43+
self.url + image_name + '/tags?page_size=100'
4644
)
4745

4846
return data['results']

prestashop_docker/generator.py

Lines changed: 5 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
from errno import EEXIST
22
from os import path, makedirs
33
from string import Template
4-
from packaging import version
54
from . import CONTAINERS
65
from prestashop_docker.version_manager import VersionManager
76

87

98
class Generator:
109
NIGHTLY = 'nightly'
1110

12-
def __init__(self, directory_path, template, nightly_template, branch_template):
11+
def __init__(self, distribution_api, directory_path, template, nightly_template, branch_template):
1312
"""Constructor
1413
14+
@param distribution_api: Distrihbution API
15+
@type distribution_api: DistributionApi
1516
@param directory_path: Directory path
1617
@type directory_path: str
1718
@param template: Base template
@@ -21,10 +22,7 @@ def __init__(self, directory_path, template, nightly_template, branch_template):
2122
@param branch_template: Branch template
2223
@type branch_template: str
2324
"""
24-
self.download_url = 'https://www.prestashop.com/download/old/' \
25-
'prestashop_{}.zip'
26-
self.download_url_github = 'https://github.com/PrestaShop/PrestaShop/releases/download/{}/prestashop_{}.zip'
27-
self.download_url_github_classic = 'https://github.com/PrestaShopCorp/prestashop-classic/releases/download/{}/prestashop_{}.zip'
25+
self.distribution_api = distribution_api
2826
self.directory_path = directory_path
2927
self.template = Template(template)
3028
self.nightly_template = Template(nightly_template)
@@ -86,19 +84,7 @@ def generate_image(self, ps_version, container_version):
8684
node_version = 'v20.17.0'
8785

8886
with open(file_path, 'w+') as f:
89-
use_github_url = True
90-
# We use 1.7.8.8 as the comparison base because the 1.7.8.9 is not hosted on the .com anymore but until 1.7.8.8,
91-
# it still works so the .com url is used
92-
if split_version is not None and split_version['patch'] != 'x' and split_version['major'] == '1.7' and version.parse(ps_version) <= version.parse('1.7.8.8'):
93-
use_github_url = False
94-
95-
if use_github_url:
96-
if parsed_version['distribution'] == 'classic':
97-
ps_url = self.download_url_github_classic.format(ps_version.replace("-classic", ""), ps_version.replace("-classic", ""))
98-
else:
99-
ps_url = self.download_url_github.format(ps_version, ps_version)
100-
else:
101-
ps_url = self.download_url.format(ps_version)
87+
ps_url = self.distribution_api.get_download_url_of(ps_version)
10288
f.write(
10389
template.substitute(
10490
{

0 commit comments

Comments
 (0)