Skip to content

Commit e6ecec2

Browse files
glehmannglehmann
authored
pkg_in_pipe: write a machine readable version of the report (#839)
The report is written as a json file that other scripts can consume, e.g. to generate the release post or to move the plane cards of the builds to another state. Signed-off-by: Gaëtan Lehmann <gaetan.lehmann@vates.tech>
1 parent 356fba8 commit e6ecec2

5 files changed

Lines changed: 265 additions & 7 deletions

File tree

‎scripts/pkg_in_pipe/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
report.html
2+
report.json
23
.cache

‎scripts/pkg_in_pipe/README.md‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,19 @@ A plane token with enough rights to list the cards in the XCPNG project must be
1515
environment variable or the `--plane-token` command line option.
1616

1717
An extra `--generated-info` command line option may be used to add some info about the report generation process.
18-
18+
19+
A machine readable version of the report can also be generated with the `--json-output` option:
20+
21+
```sh
22+
pkg_in_pipe --json-output report.json report.html
23+
```
24+
25+
The json report can be validated against its schema with:
26+
27+
```sh
28+
python -m jsonschema -i report.json pkg_in_pipe.schema.json
29+
```
30+
1931
# Run in docker
2032

2133
Before running in docker, the docker image must be built with:

‎scripts/pkg_in_pipe/__pycache__/pkg_in_pipe.cpython-313.pyc‎

27.5 KB
Binary file not shown.

‎scripts/pkg_in_pipe/pkg_in_pipe.py‎

Lines changed: 66 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -213,24 +213,46 @@ def tag_priority(tag):
213213
else:
214214
return -1
215215

216-
def find_previous_build_commit(build_tag, build):
217-
"""Find the previous build in an higher priority koji tag and return its commit."""
216+
def find_previous_build_with_commit(build_tag, build):
217+
"""Find the previous build of the package in an higher priority koji tag and its source commit.
218+
219+
The builds whose source is unknown are skipped, so the most recent build providing a source commit
220+
is returned, as (build, commit). Return (None, None) when no such build exists.
221+
"""
218222
tagged = KOJI.listTagged(build_tag, package=build['package_name'], inherit=True)
219223
tagged = sorted(tagged, key=lambda t: (tag_priority(t['tag_name']), -t['build_id']))
220224
build_tag_priority = tag_priority(build_tag)
221225
tagged = [
222226
t for t in tagged if tag_priority(t['tag_name']) >= build_tag_priority and t['build_id'] < build['build_id']
223227
]
224-
225228
for tag in tagged:
226229
previous_build = get_koji_build(tag['build_id'])
227230
source = previous_build.get('source') or MISSING_SOURCES.get(previous_build['nvr'])
228231
if not source:
229232
continue
230-
return parse_source(source)[1]
233+
return previous_build, parse_source(source)[1]
231234

232-
return None
235+
return None, None
233236

237+
def released_tags(build_tag):
238+
"""Return the tags holding the released builds of the version, e.g. v8.3-updates and v8.3-base."""
239+
version = build_tag.split('-')[0][1:] # v8.3-candidates -> 8.3
240+
return [] if not version else [f'v{version}-updates', f'v{version}-base']
241+
242+
def find_released_build(build_tag, package_name):
243+
"""Find the newest released build of the package in the updates and base tags, updates first.
244+
245+
Some versions may not have an updates or base tag yet, which koji reports as an error, so a
246+
missing tag is treated as an empty one.
247+
"""
248+
for tag in released_tags(build_tag):
249+
try:
250+
tagged = KOJI.listTagged(tag, package=package_name)
251+
except koji.GenericError:
252+
continue
253+
if tagged:
254+
return get_koji_build(max(tagged, key=lambda t: t['build_id'])['build_id'])
255+
return None
234256

235257
def find_commits(gh, repo, start_sha, end_sha) -> list[Commit]:
236258
"""
@@ -386,6 +408,7 @@ def get_plane_issues_with_milestones(plane_token):
386408
'--package', '-p', dest='packages', help="The packages to include in the report", action='append', default=[]
387409
)
388410
parser.add_argument('--re-cache', help="Refresh the cache", action='store_true')
411+
parser.add_argument('--json-output', help="Also write a machine readable report in json format to this path")
389412
args = parser.parse_args()
390413

391414
CACHE = diskcache.Cache(args.cache)
@@ -420,6 +443,14 @@ def get_plane_issues_with_milestones(plane_token):
420443
with urlopen('https://github.com/xcp-ng/xcp/raw/refs/heads/master/scripts/rpm_owners/packages.json') as f:
421444
PACKAGES = json.load(f)
422445

446+
report_data = {
447+
'generated_at': started_at.isoformat(),
448+
'generated_info': args.generated_info,
449+
'warnings': {'plane': not issues, 'github': not GITHUB},
450+
'error': None,
451+
'tags': [],
452+
}
453+
423454
with io.StringIO() as out:
424455
print_header(out)
425456
if not issues:
@@ -433,6 +464,8 @@ def get_plane_issues_with_milestones(plane_token):
433464
KOJI = koji.ClientSession('https://kojihub.xcp-ng.org', config)
434465
KOJI.ssl_login(config['cert'], None, config['serverca'])
435466
for tag in tags:
467+
tag_data = {'tag': tag, 'builds': []}
468+
report_data['tags'].append(tag_data)
436469
tag_history = dict(
437470
(tl['build_id'], tl['create_ts'])
438471
for tl in KOJI.queryHistory(tag=tag, active=True)['tag_listing']
@@ -445,7 +478,8 @@ def get_plane_issues_with_milestones(plane_token):
445478
build = get_koji_build(tagged['build_id'])
446479
prs: list[PullRequest] = []
447480
maintained_by = None
448-
previous_build_sha = find_previous_build_commit(tag, build)
481+
(previous_build, previous_build_sha) = find_previous_build_with_commit(tag, build)
482+
released_build = find_released_build(tag, tagged['package_name'])
449483
if build['source'] is not None:
450484
(repo, sha) = parse_source(build['source'])
451485
prs = find_pull_requests(repo, sha, previous_build_sha)
@@ -455,12 +489,34 @@ def get_plane_issues_with_milestones(plane_token):
455489
print_table_line(
456490
temp_out, tagged['nvr'], build_url, build_issues, tagged['owner_name'], prs, maintained_by
457491
)
492+
tag_data['builds'].append({
493+
'nvr': tagged['nvr'],
494+
'previous_nvr': released_build['nvr'] if released_build is not None else None,
495+
'package': tagged['package_name'],
496+
'url': build_url,
497+
'built_by': tagged['owner_name'],
498+
'maintained_by': maintained_by,
499+
'issues': [{
500+
'sequence_id': i['sequence_id'],
501+
'url': f'https://project.vates.tech/vates-global/browse/XCPNG-{i["sequence_id"]}/',
502+
'milestones': sorted(set(i['milestones'])),
503+
} for i in build_issues],
504+
'pull_requests': [{
505+
'number': pr.number,
506+
'title': pr.title,
507+
'url': pr.html_url,
508+
'linked': issues_have_link(build_issues, pr.html_url),
509+
} for pr in prs],
510+
'build_linked': issues_have_link(build_issues, build_url),
511+
})
458512
print_table_footer(temp_out)
459513
out.write(temp_out.getvalue())
460514
except koji.GenericError:
515+
report_data['error'] = 'koji'
461516
print_koji_error(out)
462517
raise
463518
except Exception:
519+
report_data['error'] = 'unknown'
464520
print_generic_error(out)
465521
raise
466522
finally:
@@ -469,3 +525,7 @@ def get_plane_issues_with_milestones(plane_token):
469525
# write the actual output at once, in order to avoid a blank page during the processing
470526
with open(args.output, 'w') as f:
471527
f.write(out.getvalue())
528+
529+
if args.json_output:
530+
with open(args.json_output, 'w') as f:
531+
json.dump(report_data, f, indent=2)

‎scripts/pkg_in_pipe/pkg_in_pipe.schema.json‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
{
2+
"$schema": "https://json-schema.org/draft/2020-12/schema",
3+
"$id": "https://xcp-ng.org/schemas/pkg_in_pipe.json",
4+
"title": "XCP-ng package in the pipe report",
5+
"type": "object",
6+
"required": [
7+
"generated_at",
8+
"generated_info",
9+
"warnings",
10+
"error",
11+
"tags"
12+
],
13+
"properties": {
14+
"generated_at": {
15+
"description": "Timestamp of the start of the report generation, in ISO 8601 format",
16+
"type": "string",
17+
"format": "date-time"
18+
},
19+
"generated_info": {
20+
"description": "Message about the generation included in the report",
21+
"type": [
22+
"string",
23+
"null"
24+
]
25+
},
26+
"warnings": {
27+
"description": "Whether some external sources could not be reached",
28+
"type": "object",
29+
"required": [
30+
"plane",
31+
"github"
32+
],
33+
"properties": {
34+
"plane": {
35+
"description": "True if the issues could not be retrieved from plane",
36+
"type": "boolean"
37+
},
38+
"github": {
39+
"description": "True if github is not available and the pull requests may come from the cache",
40+
"type": "boolean"
41+
}
42+
}
43+
},
44+
"error": {
45+
"description": "Set when the report could not be fully generated",
46+
"enum": [
47+
null,
48+
"koji",
49+
"unknown"
50+
]
51+
},
52+
"tags": {
53+
"type": "array",
54+
"items": {
55+
"type": "object",
56+
"required": [
57+
"tag",
58+
"builds"
59+
],
60+
"properties": {
61+
"tag": {
62+
"description": "The koji tag this section is about, e.g. v8.3-incoming",
63+
"type": "string"
64+
},
65+
"builds": {
66+
"type": "array",
67+
"items": {
68+
"type": "object",
69+
"required": [
70+
"nvr",
71+
"package",
72+
"url",
73+
"built_by",
74+
"maintained_by",
75+
"issues",
76+
"pull_requests",
77+
"build_linked"
78+
],
79+
"properties": {
80+
"nvr": {
81+
"description": "The package name, version and release of the build",
82+
"type": "string"
83+
},
84+
"previous_nvr": {
85+
"description": "The NVR of the previously released build of the package (newest build in the updates or base tag), when one exists",
86+
"type": [
87+
"string",
88+
"null"
89+
]
90+
},
91+
"package": {
92+
"description": "The name of the package this build is about",
93+
"type": "string"
94+
},
95+
"url": {
96+
"description": "URL to the build on koji",
97+
"type": "string",
98+
"format": "uri"
99+
},
100+
"built_by": {
101+
"description": "The koji user who built the package",
102+
"type": "string"
103+
},
104+
"maintained_by": {
105+
"description": "The maintainer of the package, when known",
106+
"type": [
107+
"string",
108+
"null"
109+
]
110+
},
111+
"issues": {
112+
"description": "The plane cards related to this build or its pull requests",
113+
"type": "array",
114+
"items": {
115+
"type": "object",
116+
"required": [
117+
"sequence_id",
118+
"url",
119+
"milestones"
120+
],
121+
"properties": {
122+
"sequence_id": {
123+
"description": "The XCPNG sequence id of the card",
124+
"type": "string"
125+
},
126+
"url": {
127+
"description": "URL to the card on plane",
128+
"type": "string",
129+
"format": "uri"
130+
},
131+
"milestones": {
132+
"type": "array",
133+
"items": {
134+
"type": "string"
135+
},
136+
"uniqueItems": true
137+
}
138+
}
139+
}
140+
},
141+
"pull_requests": {
142+
"description": "The pull requests related to this build",
143+
"type": "array",
144+
"items": {
145+
"type": "object",
146+
"required": [
147+
"number",
148+
"title",
149+
"url",
150+
"linked"
151+
],
152+
"properties": {
153+
"number": {
154+
"description": "The pull request number on github",
155+
"type": "integer",
156+
"minimum": 1
157+
},
158+
"title": {
159+
"description": "The title of the pull request",
160+
"type": "string"
161+
},
162+
"url": {
163+
"description": "URL to the pull request on github",
164+
"type": "string",
165+
"format": "uri"
166+
},
167+
"linked": {
168+
"description": "True if the pull request is listed in one of the related cards",
169+
"type": "boolean"
170+
}
171+
}
172+
}
173+
},
174+
"build_linked": {
175+
"description": "True if the build is listed in one of the related cards",
176+
"type": "boolean"
177+
}
178+
}
179+
}
180+
}
181+
}
182+
}
183+
}
184+
}
185+
}

0 commit comments

Comments
 (0)