Skip to content

Commit e830ce2

Browse files
authored
Merge pull request #748 from xcp-ng/gln/koji-build-cache-qtkz
Cache koji builds
2 parents 85cebfa + 08078e0 commit e830ce2

1 file changed

Lines changed: 27 additions & 20 deletions

File tree

‎scripts/pkg_in_pipe/pkg_in_pipe.py‎

Lines changed: 27 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -206,17 +206,17 @@ def tag_priority(tag):
206206
tag = tag.split('-')[-1]
207207
return TAG_ORDER.index(tag)
208208

209-
def find_previous_build_commit(session, build_tag, build):
209+
def find_previous_build_commit(build_tag, build):
210210
"""Find the previous build in an higher priority koji tag and return its commit."""
211-
tagged = session.listTagged(build_tag, package=build['package_name'], inherit=True)
211+
tagged = KOJI.listTagged(build_tag, package=build['package_name'], inherit=True)
212212
tagged = sorted(tagged, key=lambda t: (tag_priority(t['tag_name']), -t['build_id']))
213213
build_tag_priority = tag_priority(build_tag)
214214
tagged = [
215215
t for t in tagged if tag_priority(t['tag_name']) >= build_tag_priority and t['build_id'] < build['build_id']
216216
]
217217
if not tagged:
218218
return None
219-
previous_build = session.getBuild(tagged[0]['build_id'])
219+
previous_build = get_koji_build(tagged[0]['build_id'])
220220
if not previous_build.get('source'):
221221
return None
222222
return parse_source(previous_build['source'])[1]
@@ -240,21 +240,21 @@ def find_commits(gh, repo, start_sha, end_sha) -> list[Commit]:
240240
CACHE.set(cache_key, commits, expire=RETENTION_TIME)
241241
return commits
242242

243-
def find_pull_requests(gh, repo, start_sha, end_sha):
243+
def find_pull_requests(repo, start_sha, end_sha):
244244
"""Find the pull requests for the commits in the [start_sha,end_sha[ range."""
245245
prs = set()
246-
for commit in find_commits(gh, repo, start_sha, end_sha):
246+
for commit in find_commits(GITHUB, repo, start_sha, end_sha):
247247
cache_key = f'commit-prs-4-{commit.sha}'
248248
if not args.re_cache and cache_key in CACHE:
249249
prs.update(cast(list[PullRequest], CACHE[cache_key]))
250-
elif gh:
250+
elif GITHUB:
251251
commit_prs = list(commit.get_pulls())
252252
if not commit_prs:
253253
# github is not properly reporting some PRs. Try to workaround that problem by getting
254254
# the PR number from the commit message
255255
group = re.match(r'Merge pull request #(\d+) from ', commit.commit.message)
256256
if group:
257-
pr = gh.get_repo(repo).get_pull(int(group[1]))
257+
pr = GITHUB.get_repo(repo).get_pull(int(group[1]))
258258
prs.add(pr)
259259
# github sometimes return a PR which doesn't match the given commit, so we check if the commit
260260
# is actually in the PR
@@ -263,6 +263,14 @@ def find_pull_requests(gh, repo, start_sha, end_sha):
263263
prs.update(commit_prs)
264264
return sorted(prs, key=lambda p: p.number, reverse=True)
265265

266+
def get_koji_build(build_id) -> dict:
267+
cache_key = f'koji-build-{build_id}'
268+
if not args.re_cache and cache_key in CACHE:
269+
return cast(dict, CACHE[cache_key])
270+
else:
271+
build = KOJI.getBuild(build_id)
272+
CACHE.set(cache_key, build, expire=RETENTION_TIME)
273+
return build
266274

267275
def get_plane_issues(plane_token):
268276
"""Fetch all issues from Plane API with cursor-based pagination."""
@@ -384,14 +392,13 @@ def get_plane_issues_with_milestones(plane_token):
384392
raise
385393

386394
# connect to github
395+
GITHUB = None
387396
if args.github_token:
388-
gh = github.Github(auth=github.Auth.Token(args.github_token))
397+
GITHUB = github.Github(auth=github.Auth.Token(args.github_token))
389398
try:
390-
gh.get_repo('xcp-ng/xcp') # check that the token is valid
399+
GITHUB.get_repo('xcp-ng/xcp') # check that the token is valid
391400
except BadCredentialsException:
392-
gh = None
393-
else:
394-
gh = None
401+
GITHUB = None
395402

396403
# load the packages maintainers
397404
with urlopen('https://github.com/xcp-ng/xcp/raw/refs/heads/master/scripts/rpm_owners/packages.json') as f:
@@ -401,31 +408,31 @@ def get_plane_issues_with_milestones(plane_token):
401408
print_header(out)
402409
if not issues:
403410
print_plane_warning(out)
404-
if not gh:
411+
if not GITHUB:
405412
print_github_warning(out)
406413
with io.StringIO() as temp_out:
407414
try:
408415
# open koji session
409416
config = koji.read_config("koji")
410-
session = koji.ClientSession('https://kojihub.xcp-ng.org', config)
411-
session.ssl_login(config['cert'], None, config['serverca'])
417+
KOJI = koji.ClientSession('https://kojihub.xcp-ng.org', config)
418+
KOJI.ssl_login(config['cert'], None, config['serverca'])
412419
for tag in tags:
413420
tag_history = dict(
414421
(tl['build_id'], tl['create_ts'])
415-
for tl in session.queryHistory(tag=tag, active=True)['tag_listing']
422+
for tl in KOJI.queryHistory(tag=tag, active=True)['tag_listing']
416423
)
417424
print_table_header(temp_out, tag)
418-
taggeds = session.listTagged(tag)
425+
taggeds = KOJI.listTagged(tag)
419426
taggeds = (t for t in taggeds if t['package_name'] in args.packages or args.packages == [])
420427
taggeds = sorted(taggeds, key=lambda t: (tag_history[t['build_id']], t['build_id']), reverse=True)
421428
for tagged in taggeds:
422-
build = session.getBuild(tagged['build_id'])
429+
build = get_koji_build(tagged['build_id'])
423430
prs: list[PullRequest] = []
424431
maintained_by = None
425-
previous_build_sha = find_previous_build_commit(session, tag, build)
432+
previous_build_sha = find_previous_build_commit(tag, build)
426433
if build['source'] is not None:
427434
(repo, sha) = parse_source(build['source'])
428-
prs = find_pull_requests(gh, repo, sha, previous_build_sha)
435+
prs = find_pull_requests(repo, sha, previous_build_sha)
429436
maintained_by = PACKAGES.get(tagged['package_name'], {}).get('maintainer')
430437
build_url = f'https://koji.xcp-ng.org/buildinfo?buildID={tagged["build_id"]}'
431438
build_issues = filter_issues(issues, [build_url] + [pr.html_url for pr in prs])

0 commit comments

Comments
 (0)