pkg_in_pipe: write a machine readable version of the report

The report is written as a json file that other scripts can consume,
e.g. to generate the release post or to move the plane cards of the
builds to another state.

Signed-off-by: Gaëtan Lehmann <gaetan.lehmann@vates.tech>

Author: glehmann

scripts/pkg_in_pipe/.gitignore

@@ -1,2 +1,3 @@
 report.html
+report.json
 .cache

scripts/pkg_in_pipe/README.md

@@ -15,7 +15,19 @@ A plane token with enough rights to list the cards in the XCPNG project must be
 environment variable or the `--plane-token` command line option.
 
 An extra `--generated-info` command line option may be used to add some info about the report generation process.
- 
+
+A machine readable version of the report can also be generated with the `--json-output` option:
+
+```sh
+pkg_in_pipe --json-output report.json report.html
+```
+
+The json report can be validated against its schema with:
+
+```sh
+python -m jsonschema -i report.json pkg_in_pipe.schema.json
+```
+
 # Run in docker
 
 Before running in docker, the docker image must be built with:

scripts/pkg_in_pipe/pkg_in_pipe.py

@@ -208,8 +208,8 @@ def tag_priority(tag):
     else:
         return -1
 
-def find_previous_build_commit(build_tag, build):
-    """Find the previous build in an higher priority koji tag and return its commit."""
+def find_previous_build(build_tag, build):
+    """Find the previous build of the package in an higher priority koji tag and return it."""
     tagged = KOJI.listTagged(build_tag, package=build['package_name'], inherit=True)
     tagged = sorted(tagged, key=lambda t: (tag_priority(t['tag_name']), -t['build_id']))
     build_tag_priority = tag_priority(build_tag)
@@ -218,10 +218,27 @@ def find_previous_build_commit(build_tag, build):
     ]
     if not tagged:
         return None
-    previous_build = get_koji_build(tagged[0]['build_id'])
-    if not previous_build.get('source'):
-        return None
-    return parse_source(previous_build['source'])[1]
+    return get_koji_build(tagged[0]['build_id'])
+
+def released_tags(build_tag):
+    """Return the tags holding the released builds of the version, e.g. v8.3-updates and v8.3-base."""
+    version = build_tag.split('-')[0][1:]  # v8.3-candidates -> 8.3
+    return [] if not version else [f'v{version}-updates', f'v{version}-base']
+
+def find_released_build(build_tag, package_name):
+    """Find the newest released build of the package in the updates and base tags, updates first.
+
+    Some versions may not have an updates or base tag yet, which koji reports as an error, so a
+    missing tag is treated as an empty one.
+    """
+    for tag in released_tags(build_tag):
+        try:
+            tagged = KOJI.listTagged(tag, package=package_name)
+        except koji.GenericError:
+            continue
+        if tagged:
+            return get_koji_build(max(tagged, key=lambda t: t['build_id'])['build_id'])
+    return None
 
 def find_commits(gh, repo, start_sha, end_sha) -> list[Commit]:
     """
@@ -377,6 +394,7 @@ def get_plane_issues_with_milestones(plane_token):
     '--package', '-p', dest='packages', help="The packages to include in the report", action='append', default=[]
 )
 parser.add_argument('--re-cache', help="Refresh the cache", action='store_true')
+parser.add_argument('--json-output', help="Also write a machine readable report in json format to this path")
 args = parser.parse_args()
 
 CACHE = diskcache.Cache(args.cache)
@@ -411,6 +429,14 @@ def get_plane_issues_with_milestones(plane_token):
 with urlopen('https://github.com/xcp-ng/xcp/raw/refs/heads/master/scripts/rpm_owners/packages.json') as f:
     PACKAGES = json.load(f)
 
+report_data = {
+    'generated_at': started_at.isoformat(),
+    'generated_info': args.generated_info,
+    'warnings': {'plane': not issues, 'github': not GITHUB},
+    'error': None,
+    'tags': [],
+}
+
 with io.StringIO() as out:
     print_header(out)
     if not issues:
@@ -424,6 +450,8 @@ def get_plane_issues_with_milestones(plane_token):
             KOJI = koji.ClientSession('https://kojihub.xcp-ng.org', config)
             KOJI.ssl_login(config['cert'], None, config['serverca'])
             for tag in tags:
+                tag_data = {'tag': tag, 'builds': []}
+                report_data['tags'].append(tag_data)
                 tag_history = dict(
                     (tl['build_id'], tl['create_ts'])
                     for tl in KOJI.queryHistory(tag=tag, active=True)['tag_listing']
@@ -436,7 +464,11 @@ def get_plane_issues_with_milestones(plane_token):
                     build = get_koji_build(tagged['build_id'])
                     prs: list[PullRequest] = []
                     maintained_by = None
-                    previous_build_sha = find_previous_build_commit(tag, build)
+                    previous_build = find_previous_build(tag, build)
+                    previous_build_sha = None
+                    if previous_build is not None and previous_build.get('source') is not None:
+                        previous_build_sha = parse_source(previous_build['source'])[1]
+                    released_build = find_released_build(tag, tagged['package_name'])
                     if build['source'] is not None:
                         (repo, sha) = parse_source(build['source'])
                         prs = find_pull_requests(repo, sha, previous_build_sha)
@@ -446,12 +478,34 @@ def get_plane_issues_with_milestones(plane_token):
                     print_table_line(
                         temp_out, tagged['nvr'], build_url, build_issues, tagged['owner_name'], prs, maintained_by
                     )
+                    tag_data['builds'].append({
+                        'nvr': tagged['nvr'],
+                        'previous_nvr': released_build['nvr'] if released_build is not None else None,
+                        'package': tagged['package_name'],
+                        'url': build_url,
+                        'built_by': tagged['owner_name'],
+                        'maintained_by': maintained_by,
+                        'issues': [{
+                            'sequence_id': i['sequence_id'],
+                            'url': f'https://project.vates.tech/vates-global/browse/XCPNG-{i["sequence_id"]}/',
+                            'milestones': sorted(set(i['milestones'])),
+                        } for i in build_issues],
+                        'pull_requests': [{
+                            'number': pr.number,
+                            'title': pr.title,
+                            'url': pr.html_url,
+                            'linked': issues_have_link(build_issues, pr.html_url),
+                        } for pr in prs],
+                        'build_linked': issues_have_link(build_issues, build_url),
+                    })
                 print_table_footer(temp_out)
             out.write(temp_out.getvalue())
         except koji.GenericError:
+            report_data['error'] = 'koji'
             print_koji_error(out)
             raise
         except Exception:
+            report_data['error'] = 'unknown'
             print_generic_error(out)
             raise
         finally:
@@ -460,3 +514,7 @@ def get_plane_issues_with_milestones(plane_token):
     # write the actual output at once, in order to avoid a blank page during the processing
     with open(args.output, 'w') as f:
         f.write(out.getvalue())
+
+    if args.json_output:
+        with open(args.json_output, 'w') as f:
+            json.dump(report_data, f, indent=2)

scripts/pkg_in_pipe/pkg_in_pipe.schema.json

@@ -0,0 +1,143 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://xcp-ng.org/schemas/pkg_in_pipe.json",
+    "title": "XCP-ng package in the pipe report",
+    "type": "object",
+    "required": ["generated_at", "generated_info", "warnings", "error", "tags"],
+    "properties": {
+        "generated_at": {
+            "description": "Timestamp of the start of the report generation, in ISO 8601 format",
+            "type": "string",
+            "format": "date-time"
+        },
+        "generated_info": {
+            "description": "Message about the generation included in the report",
+            "type": ["string", "null"]
+        },
+        "warnings": {
+            "description": "Whether some external sources could not be reached",
+            "type": "object",
+            "required": ["plane", "github"],
+            "properties": {
+                "plane": {
+                    "description": "True if the issues could not be retrieved from plane",
+                    "type": "boolean"
+                },
+                "github": {
+                    "description": "True if github is not available and the pull requests may come from the cache",
+                    "type": "boolean"
+                }
+            }
+        },
+        "error": {
+            "description": "Set when the report could not be fully generated",
+            "enum": [null, "koji", "unknown"]
+        },
+        "tags": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "required": ["tag", "builds"],
+                "properties": {
+                    "tag": {
+                        "description": "The koji tag this section is about, e.g. v8.3-incoming",
+                        "type": "string"
+                    },
+                    "builds": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "required": [
+                                "nvr", "package", "url", "built_by", "maintained_by", "issues", "pull_requests",
+                                "build_linked"
+                            ],
+                            "properties": {
+                                "nvr": {
+                                    "description": "The package name, version and release of the build",
+                                    "type": "string"
+                                },
+                                "previous_nvr": {
+                                    "description": "The NVR of the previously released build of the package (newest build in the updates or base tag), when one exists",
+                                    "type": ["string", "null"]
+                                },
+                                "package": {
+                                    "description": "The name of the package this build is about",
+                                    "type": "string"
+                                },
+                                "url": {
+                                    "description": "URL to the build on koji",
+                                    "type": "string",
+                                    "format": "uri"
+                                },
+                                "built_by": {
+                                    "description": "The koji user who built the package",
+                                    "type": "string"
+                                },
+                                "maintained_by": {
+                                    "description": "The maintainer of the package, when known",
+                                    "type": ["string", "null"]
+                                },
+                                "issues": {
+                                    "description": "The plane cards related to this build or its pull requests",
+                                    "type": "array",
+                                    "items": {
+                                        "type": "object",
+                                        "required": ["sequence_id", "url", "milestones"],
+                                        "properties": {
+                                            "sequence_id": {
+                                                "description": "The XCPNG sequence id of the card",
+                                                "type": "string"
+                                            },
+                                            "url": {
+                                                "description": "URL to the card on plane",
+                                                "type": "string",
+                                                "format": "uri"
+                                            },
+                                            "milestones": {
+                                                "type": "array",
+                                                "items": {"type": "string"},
+                                                "uniqueItems": true
+                                            }
+                                        }
+                                    }
+                                },
+                                "pull_requests": {
+                                    "description": "The pull requests related to this build",
+                                    "type": "array",
+                                    "items": {
+                                        "type": "object",
+                                        "required": ["number", "title", "url", "linked"],
+                                        "properties": {
+                                            "number": {
+                                                "description": "The pull request number on github",
+                                                "type": "integer",
+                                                "minimum": 1
+                                            },
+                                            "title": {
+                                                "description": "The title of the pull request",
+                                                "type": "string"
+                                            },
+                                            "url": {
+                                                "description": "URL to the pull request on github",
+                                                "type": "string",
+                                                "format": "uri"
+                                            },
+                                            "linked": {
+                                                "description": "True if the pull request is listed in one of the related cards",
+                                                "type": "boolean"
+                                            }
+                                        }
+                                    }
+                                },
+                                "build_linked": {
+                                    "description": "True if the build is listed in one of the related cards",
+                                    "type": "boolean"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}