1+ #!/usr/bin/env python
2+ """Generate the package update section of the XCP-ng release post.
3+
4+ Read the json report generated by pkg_in_pipe.py and, for each package of a
5+ given koji tag, print the "Explain the change to users" sections of the pull
6+ requests related to the package builds.
7+ """
8+
9+ from __future__ import annotations
10+
11+ import argparse
12+ import os
13+ import re
14+ import signal
15+ import sys
16+ from collections import defaultdict
17+ from datetime import datetime
18+ from pathlib import Path
19+ from string import Template
20+ from textwrap import dedent
21+ from typing import Literal , cast
22+
23+ import diskcache # type: ignore[import-untyped]
24+ import requests
25+ from pydantic import BaseModel
26+ from tqdm import tqdm
27+
28+
29+ class Warnings (BaseModel ):
30+ plane : bool
31+ github : bool
32+
33+
34+ class Issue (BaseModel ):
35+ sequence_id : int
36+ url : str
37+ milestones : list [str ]
38+
39+
40+ class PullRequest (BaseModel ):
41+ number : int
42+ title : str
43+ url : str
44+ linked : bool
45+
46+
47+ class Build (BaseModel ):
48+ nvr : str
49+ previous_nvr : str | None = None
50+ package : str
51+ url : str
52+ built_by : str
53+ maintained_by : str | None
54+ issues : list [Issue ]
55+ pull_requests : list [PullRequest ]
56+ build_linked : bool
57+
58+
59+ class TagReport (BaseModel ):
60+ tag : str
61+ builds : list [Build ]
62+
63+
64+ class Report (BaseModel ):
65+ generated_at : datetime
66+ generated_info : str | None
67+ warnings : Warnings
68+ error : Literal ['koji' , 'unknown' ] | None
69+ tags : list [TagReport ]
70+
71+
72+ PR_URL_RE = re .compile (r'^https://github\.com/([^/]+)/([^/]+)/pull/(\d+)$' )
73+
74+ RETENTION_TIME = 24 * 60 * 60 # 24 hours
75+
76+
77+ def find_tag_report (report : Report , tag : str ) -> TagReport :
78+ for tag_report in report .tags :
79+ if tag_report .tag == tag :
80+ return tag_report
81+ raise SystemExit (f'error: the tag { tag } )
82+
83+
84+ def prs_by_package (tag_report : TagReport ) -> dict [str , list [PullRequest ]]:
85+ prs_by_package : dict [str , list [PullRequest ]] = defaultdict (list )
86+ for build in tag_report .builds :
87+ for pr in build .pull_requests :
88+ if all (existing .url != pr .url for existing in prs_by_package [build .package ]):
89+ prs_by_package [build .package ].append (pr )
90+ return dict (prs_by_package )
91+
92+
93+ def fetch_pr_description (
94+ url : str , token : str | None , re_cache : bool , cache : diskcache .Cache
95+ ) -> str | None :
96+ cache_key = f'pr-body-1-{ url }
97+ if not re_cache and cache_key in cache :
98+ return cast (str | None , cache [cache_key ])
99+ match = PR_URL_RE .fullmatch (url )
100+ if match is None :
101+ raise RuntimeError (f'not a github pull request url: { url } )
102+ owner , repo , number = cast (tuple [str , str , str ], match .groups ())
103+ headers = {'Accept' : 'application/vnd.github+json' }
104+ if token :
105+ headers ['Authorization' ] = f'Bearer { token }
106+ response = requests .get (f'https://api.github.com/repos/{ owner } { repo } { number } , headers = headers , timeout = 30 )
107+ if response .status_code != 200 :
108+ raise RuntimeError (f'got a { response .status_code } )
109+ body = cast (str | None , response .json ().get ('body' ))
110+ cache .set (cache_key , body , expire = RETENTION_TIME )
111+ return body
112+
113+
114+ def fetch_pr_descriptions (
115+ prs : list [PullRequest ], token : str | None , re_cache : bool , cache : diskcache .Cache
116+ ) -> list [tuple [PullRequest , str ]]:
117+ descriptions : list [tuple [PullRequest , str ]] = []
118+ for pr in prs :
119+ try :
120+ description = fetch_pr_description (pr .url , token , re_cache , cache )
121+ except (RuntimeError , requests .RequestException ) as e :
122+ print (f'warning: could not fetch the description of { pr .url } { e } , file = sys .stderr )
123+ continue
124+ if description is None :
125+ print (f'warning: the pull request { pr .url } , file = sys .stderr )
126+ continue
127+ descriptions .append ((pr , description ))
128+ return descriptions
129+
130+
131+ USER_SECTION_RE = re .compile (r'(?i)^#{1,6}\s*Explain the change to users\s*$' )
132+ HEADING_RE = re .compile (r'^#{1,6}(\s|$)' )
133+ CODE_FENCE_RE = re .compile (r'^\s*(```+|~~~+)' )
134+
135+
136+ def extract_user_section (description : str ) -> list [str ] | None :
137+ """Return the lines of the "Explain the change to users" section of a pull request description.
138+
139+ The section is returned verbatim, except for the HTML comments (template boilerplate)
140+ and the blank lines around them, which are removed.
141+ """
142+ in_code = False
143+ section : list [str ] | None = None
144+ for line in description .splitlines ():
145+ fence = CODE_FENCE_RE .match (line )
146+ if fence is not None :
147+ in_code = not in_code
148+ if section is not None :
149+ if not in_code and HEADING_RE .match (line ):
150+ break
151+ section .append (line )
152+ elif not in_code and USER_SECTION_RE .match (line ):
153+ section = []
154+ if section is None :
155+ return None
156+ section = strip_html_comments (section )
157+ while section and not section [0 ].strip ():
158+ section .pop (0 )
159+ while section and not section [- 1 ].strip ():
160+ section .pop ()
161+ return section or None
162+
163+
164+ def strip_html_comments (lines : list [str ]) -> list [str ]:
165+ """Remove the HTML comments (template boilerplate) and the blank lines around them."""
166+ res : list [str ] = []
167+ in_comment = False
168+ skip_blanks = False
169+ for line in lines :
170+ if in_comment :
171+ if '-->' in line :
172+ in_comment = False
173+ skip_blanks = True
174+ continue
175+ if '<!--' in line :
176+ in_comment = '-->' not in line
177+ skip_blanks = True
178+ while res and not res [- 1 ].strip ():
179+ res .pop ()
180+ continue
181+ if skip_blanks and not line .strip ():
182+ continue
183+ skip_blanks = False
184+ res .append (line )
185+ while res and not res [- 1 ].strip ():
186+ res .pop ()
187+ return res
188+
189+
190+ def warn (pbar : tqdm , message : str ) -> None :
191+ pbar .clear ()
192+ print (f'warning: { message } , file = sys .stderr )
193+
194+
195+ def collect_sections (descriptions : list [tuple [PullRequest , str ]], pbar : tqdm ) -> list [str ]:
196+ lines : list [str ] = []
197+ for pr , description in descriptions :
198+ section = extract_user_section (description )
199+ if section is None :
200+ warn (pbar , f'no "Explain the change to users" section in { pr .url } )
201+ continue
202+ lines .extend (section )
203+ return lines
204+
205+
206+ def verbatim_lines (package : str , lines : list [str ]) -> list [str ]:
207+ if len (lines ) > 1 :
208+ return [f'- `{ package } ] + [f'\t { line } if line else '' for line in lines ]
209+ if lines :
210+ return [f'- `{ package } { lines [0 ]} ]
211+ return [f'- `{ package } ]
212+
213+
214+ def versions_by_package (tag_report : TagReport ) -> list [tuple [str , str | None , str ]]:
215+ """Return the latest build of each package, as (package, previous_nvr, nvr), sorted by package name.
216+
217+ The report lists the builds of a tag newest first, so the first occurrence of a package is its latest build.
218+ """
219+ latest : dict [str , tuple [str | None , str ]] = {}
220+ for build in tag_report .builds :
221+ latest .setdefault (build .package , (build .previous_nvr , build .nvr ))
222+ return [(package , previous_nvr , nvr ) for package , (previous_nvr , nvr ) in sorted (latest .items ())]
223+
224+
225+ def version_release (package : str , nvr : str ) -> str :
226+ """Return the version-release part of a package nvr."""
227+ return nvr .removeprefix (package + '-' )
228+
229+
230+ def format_versions (versions : list [tuple [str , str | None , str ]]) -> str :
231+ lines = []
232+ for package , previous_nvr , nvr in versions :
233+ new_version = version_release (package , nvr )
234+ item = f'{ version_release (package , previous_nvr )} { new_version } if previous_nvr is not None else new_version
235+ lines .append (f'* `{ package } { item } )
236+ return '\n ' .join (lines )
237+
238+
239+ POST_TEMPLATE = dedent ('''\
240+
241+
242+ This batch of updates contains mostly fixes, tools version update, a some improvements.
243+
244+ ## What changed
245+
246+ $what_changed
247+
248+ ## Versions
249+
250+ $versions
251+
252+ ## Test on XCP-ng $version
253+
254+ ```bash
255+ yum clean metadata --enablerepo=xcp-ng-testing,xcp-ng-candidates
256+ yum update --enablerepo=xcp-ng-testing,xcp-ng-candidates
257+ reboot
258+ ```
259+
260+ The usual update rules apply: pool coordinator first, etc.
261+
262+ ## What to test
263+
264+ As usual, normal use and anything else you want to test.
265+
266+ ## Test window before official release of the updates
267+
268+ **X days**
269+
270+ We would like to thank users who shared feedback since our last call for testing:
271+ ''' )
272+
273+
274+ def format_post (what_changed : str , versions : str , version : str ) -> str :
275+ return Template (POST_TEMPLATE ).substitute (what_changed = what_changed , versions = versions , version = version )
276+
277+
278+ def parse_args () -> argparse .Namespace :
279+ parser = argparse .ArgumentParser (description = 'Generate the package update section of the XCP-ng release post' )
280+ parser .add_argument ('--report' , help = 'The json report generated by pkg_in_pipe.py' , default = 'report.json' )
281+ parser .add_argument ('--tag' , help = 'The koji tag to include in the release post' , default = 'v8.3-ci' )
282+ parser .add_argument ('--version' , help = 'The XCP-ng version of the release post, e.g. 8.3' , default = None )
283+ parser .add_argument ('--cache' , help = 'The cache path' , default = '/tmp/pkg_in_pipe.cache' )
284+ parser .add_argument ('--re-cache' , help = 'Refresh the cache' , action = 'store_true' )
285+ parser .add_argument (
286+ '--github-token' , help = 'The token used to access the Github api' , default = os .environ .get ('GITHUB_TOKEN' )
287+ )
288+ return parser .parse_args ()
289+
290+
291+ def main () -> None :
292+ signal .signal (signal .SIGPIPE , signal .SIG_DFL )
293+ args = parse_args ()
294+ cache = diskcache .Cache (args .cache )
295+ report = Report .model_validate_json (Path (args .report ).read_text ())
296+ tag_report = find_tag_report (report , args .tag )
297+ what_changed : list [str ] = []
298+ with tqdm (
299+ sorted (prs_by_package (tag_report ).items ()),
300+ desc = 'packages' ,
301+ unit = 'package' ,
302+ file = sys .stderr ,
303+ leave = False ,
304+ ) as pbar :
305+ for package , prs in pbar :
306+ descriptions = fetch_pr_descriptions (prs , args .github_token , args .re_cache , cache )
307+ if not descriptions :
308+ warn (pbar , f'no description available for the pull requests of { package } )
309+ continue
310+ lines = collect_sections (descriptions , pbar )
311+ if not lines :
312+ warn (pbar , f'no "Explain the change to users" section available for the pull requests of { package } )
313+ what_changed .extend (verbatim_lines (package , lines ))
314+ what_changed .append ('' )
315+ version = args .version or args .tag [1 :].split ('-' )[0 ]
316+ print (format_post ('\n ' .join (what_changed ).rstrip (), format_versions (versions_by_package (tag_report )), version ))
317+
318+
319+ if __name__ == '__main__' :
320+ main ()
0 commit comments