Skip to content

pkg_in_pipe: write a machine readable version of the report #839

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
glehmann wants to merge 1 commit into
base: gln/pkg-in-pipe-updates-psmx
Choose a base branch
from
+222 −8
Open

pkg_in_pipe: write a machine readable version of the report #839

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions scripts/pkg_in_pipe/.gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
report.html
report.json
.cache
14 changes: 13 additions & 1 deletion scripts/pkg_in_pipe/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,19 @@ A plane token with enough rights to list the cards in the XCPNG project must be
environment variable or the `--plane-token` command line option.

An extra `--generated-info` command line option may be used to add some info about the report generation process.


A machine readable version of the report can also be generated with the `--json-output` option:

```sh
pkg_in_pipe --json-output report.json report.html
```

The json report can be validated against its schema with:

```sh
python -m jsonschema -i report.json pkg_in_pipe.schema.json
```

# Run in docker

Before running in docker, the docker image must be built with:
Expand Down
72 changes: 65 additions & 7 deletions scripts/pkg_in_pipe/pkg_in_pipe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,8 +208,8 @@ def tag_priority(tag):
else:
return -1

def find_previous_build_commit(build_tag, build):
"""Find the previous build in an higher priority koji tag and return its commit."""
def find_previous_build(build_tag, build):
"""Find the previous build of the package in an higher priority koji tag and return it."""
tagged = KOJI.listTagged(build_tag, package=build['package_name'], inherit=True)
tagged = sorted(tagged, key=lambda t: (tag_priority(t['tag_name']), -t['build_id']))
build_tag_priority = tag_priority(build_tag)
Expand All @@ -218,10 +218,27 @@ def find_previous_build_commit(build_tag, build):
]
if not tagged:
return None
previous_build = get_koji_build(tagged[0]['build_id'])
if not previous_build.get('source'):
return None
return parse_source(previous_build['source'])[1]
return get_koji_build(tagged[0]['build_id'])

def released_tags(build_tag):
"""Return the tags holding the released builds of the version, e.g. v8.3-updates and v8.3-base."""
version = build_tag.split('-')[0][1:] # v8.3-candidates -> 8.3
return [] if not version else [f'v{version}-updates', f'v{version}-base']

def find_released_build(build_tag, package_name):
"""Find the newest released build of the package in the updates and base tags, updates first.

Some versions may not have an updates or base tag yet, which koji reports as an error, so a
missing tag is treated as an empty one.
"""
for tag in released_tags(build_tag):
try:
tagged = KOJI.listTagged(tag, package=package_name)
except koji.GenericError:
continue
if tagged:
return get_koji_build(max(tagged, key=lambda t: t['build_id'])['build_id'])
return None

def find_commits(gh, repo, start_sha, end_sha) -> list[Commit]:
"""
Expand Down Expand Up @@ -377,6 +394,7 @@ def get_plane_issues_with_milestones(plane_token):
'--package', '-p', dest='packages', help="The packages to include in the report", action='append', default=[]
)
parser.add_argument('--re-cache', help="Refresh the cache", action='store_true')
parser.add_argument('--json-output', help="Also write a machine readable report in json format to this path")
args = parser.parse_args()

CACHE = diskcache.Cache(args.cache)
Expand Down Expand Up @@ -411,6 +429,14 @@ def get_plane_issues_with_milestones(plane_token):
with urlopen('https://github.com/xcp-ng/xcp/raw/refs/heads/master/scripts/rpm_owners/packages.json') as f:
PACKAGES = json.load(f)

report_data = {
'generated_at': started_at.isoformat(),
'generated_info': args.generated_info,
'warnings': {'plane': not issues, 'github': not GITHUB},
'error': None,
'tags': [],
}

with io.StringIO() as out:
print_header(out)
if not issues:
Expand All @@ -424,6 +450,8 @@ def get_plane_issues_with_milestones(plane_token):
KOJI = koji.ClientSession('https://kojihub.xcp-ng.org', config)
KOJI.ssl_login(config['cert'], None, config['serverca'])
for tag in tags:
tag_data = {'tag': tag, 'builds': []}
report_data['tags'].append(tag_data)
tag_history = dict(
(tl['build_id'], tl['create_ts'])
for tl in KOJI.queryHistory(tag=tag, active=True)['tag_listing']
Expand All @@ -436,7 +464,11 @@ def get_plane_issues_with_milestones(plane_token):
build = get_koji_build(tagged['build_id'])
prs: list[PullRequest] = []
maintained_by = None
previous_build_sha = find_previous_build_commit(tag, build)
previous_build = find_previous_build(tag, build)
previous_build_sha = None
if previous_build is not None and previous_build.get('source') is not None:
previous_build_sha = parse_source(previous_build['source'])[1]
released_build = find_released_build(tag, tagged['package_name'])
if build['source'] is not None:
(repo, sha) = parse_source(build['source'])
prs = find_pull_requests(repo, sha, previous_build_sha)
Expand All @@ -446,12 +478,34 @@ def get_plane_issues_with_milestones(plane_token):
print_table_line(
temp_out, tagged['nvr'], build_url, build_issues, tagged['owner_name'], prs, maintained_by
)
tag_data['builds'].append({
'nvr': tagged['nvr'],
'previous_nvr': released_build['nvr'] if released_build is not None else None,
'package': tagged['package_name'],
'url': build_url,
'built_by': tagged['owner_name'],
'maintained_by': maintained_by,
'issues': [{
'sequence_id': i['sequence_id'],
'url': f'https://project.vates.tech/vates-global/browse/XCPNG-{i["sequence_id"]}/',
'milestones': sorted(set(i['milestones'])),
} for i in build_issues],
'pull_requests': [{
'number': pr.number,
'title': pr.title,
'url': pr.html_url,
'linked': issues_have_link(build_issues, pr.html_url),
} for pr in prs],
'build_linked': issues_have_link(build_issues, build_url),
})
print_table_footer(temp_out)
out.write(temp_out.getvalue())
except koji.GenericError:
report_data['error'] = 'koji'
print_koji_error(out)
raise
except Exception:
report_data['error'] = 'unknown'
print_generic_error(out)
raise
finally:
Expand All @@ -460,3 +514,7 @@ def get_plane_issues_with_milestones(plane_token):
# write the actual output at once, in order to avoid a blank page during the processing
with open(args.output, 'w') as f:
f.write(out.getvalue())

if args.json_output:
with open(args.json_output, 'w') as f:
json.dump(report_data, f, indent=2)
143 changes: 143 additions & 0 deletions scripts/pkg_in_pipe/pkg_in_pipe.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://xcp-ng.org/schemas/pkg_in_pipe.json",
"title": "XCP-ng package in the pipe report",
"type": "object",
"required": ["generated_at", "generated_info", "warnings", "error", "tags"],
"properties": {
"generated_at": {
"description": "Timestamp of the start of the report generation, in ISO 8601 format",
"type": "string",
"format": "date-time"
},
"generated_info": {
"description": "Message about the generation included in the report",
"type": ["string", "null"]
},
"warnings": {
"description": "Whether some external sources could not be reached",
"type": "object",
"required": ["plane", "github"],
"properties": {
"plane": {
"description": "True if the issues could not be retrieved from plane",
"type": "boolean"
},
"github": {
"description": "True if github is not available and the pull requests may come from the cache",
"type": "boolean"
}
}
},
"error": {
"description": "Set when the report could not be fully generated",
"enum": [null, "koji", "unknown"]
},
"tags": {
"type": "array",
"items": {
"type": "object",
"required": ["tag", "builds"],
"properties": {
"tag": {
"description": "The koji tag this section is about, e.g. v8.3-incoming",
"type": "string"
},
"builds": {
"type": "array",
"items": {
"type": "object",
"required": [
"nvr", "package", "url", "built_by", "maintained_by", "issues", "pull_requests",
"build_linked"
],
"properties": {
"nvr": {
"description": "The package name, version and release of the build",
"type": "string"
},
"previous_nvr": {
"description": "The NVR of the previously released build of the package (newest build in the updates or base tag), when one exists",
"type": ["string", "null"]
},
"package": {
"description": "The name of the package this build is about",
"type": "string"
},
"url": {
"description": "URL to the build on koji",
"type": "string",
"format": "uri"
},
"built_by": {
"description": "The koji user who built the package",
"type": "string"
},
"maintained_by": {
"description": "The maintainer of the package, when known",
"type": ["string", "null"]
},
"issues": {
"description": "The plane cards related to this build or its pull requests",
"type": "array",
"items": {
"type": "object",
"required": ["sequence_id", "url", "milestones"],
"properties": {
"sequence_id": {
"description": "The XCPNG sequence id of the card",
"type": "string"
},
"url": {
"description": "URL to the card on plane",
"type": "string",
"format": "uri"
},
"milestones": {
"type": "array",
"items": {"type": "string"},
"uniqueItems": true
}
}
}
},
"pull_requests": {
"description": "The pull requests related to this build",
"type": "array",
"items": {
"type": "object",
"required": ["number", "title", "url", "linked"],
"properties": {
"number": {
"description": "The pull request number on github",
"type": "integer",
"minimum": 1
},
"title": {
"description": "The title of the pull request",
"type": "string"
},
"url": {
"description": "URL to the pull request on github",
"type": "string",
"format": "uri"
},
"linked": {
"description": "True if the pull request is listed in one of the related cards",
"type": "boolean"
}
}
}
},
"build_linked": {
"description": "True if the build is listed in one of the related cards",
"type": "boolean"
}
}
}
}
}
}
}
}
}