WIP yum_repo_status: generate a table showing how much outdated 8.3 is from xs8

FIXME:
- include successive xs8 update waves
- migrate infos from Sam's wiki page to package_status.csv

- show binrpms?
- include "current" el7 versions
- link proprietary packages replaced by ours
- should take import_reason from provenance.csv as tooltip?
- include upstream releases from GH?

Author: ydirson

Diff for: scripts/rpmwatcher/package_status.csv

@@ -0,0 +1,60 @@
+SRPM_name;status;comment
+auto-cert-kit;ignored;unused, why?
+automake16;ignored;unused, why?
+bpftool;ignored;unused, why?
+capstone;ignored;unused, why?
+citrix-crypto-module;ignored;proprietary, patched FIPS openssl
+compiler-rt18;ignored;unused, why?
+dlm;ignored;previous dependency for corosync
+emu-manager;ignored;proprietary, replaced by xcp-emu-manager
+epel-release;ignored;unused, why?
+forkexecd;ignored;now in xapi
+fuse;;breq for e2fsprogs 1.47
+gfs2-utils;ignored;unsupported fs
+glib2;ignored;same version as el7, why?
+golang;ignored;for newer xe-guest-utilities
+hcp_nss;ignored;unused, “enforce any permitted user login as root”
+hwloc;ignored;unused, why?
+libbpf;ignored;unused, why?
+libcgroup;ignored;unused, same version as el7, why?
+libhbalinux;;unused? el7 fork, “Fix crash in fcoeadm/elxhbamgr on certain machines”
+libnbd;ignored;unused, dep for xapi-storage-plugins
+linuxconsoletools;ignored;unused, same version as el7, why?
+mbootpack;;for secureboot?
+message-switch;ignored;now in xapi
+mpdecimal;ignored;unused, why?
+ninja-build;ignored;unused
+pbis-open;ignored;unused, likely linked to upgrade-pbis-to-winbind
+pbis-open-upgrade;ignored;unused, likely linked to upgrade-pbis-to-winbind
+pvsproxy;ignored;proprietary
+python-monotonic;ignored;previous dependency for sm
+python-tqdm;ignored;unused, dependency for pvsproxy
+rrdd-plugins;ignored;now in xapi
+ruby;ignored;unused, why?
+sbd;;unused? "storage-based death functionality"
+sm-cli;ignored;now in xapi
+secureboot-certificates;ignored;proprietary, needs alternative?
+security-tools;ignored;proprietary, pool_secret tool
+sm-transport-lib;ignored;proprietary
+squeezed;ignored;now in xapi
+tix;ignored;unused, why?
+upgrade-pbis-to-winbind;ignored;proprietary
+v6d;ignored;proprietary, replaced by xcp-featured
+varstored-guard;ignored;now in xapi
+vendor-update-keys;ignored;proprietary
+vgpu;ignored;proprietary
+vhd-tool;ignored;now in xapi
+wsproxy;ignored;now in xapi
+xapi-clusterd;ignored;proprietary
+xapi-nbd;ignored;now in xapi
+xapi-storage;ignored;now in xapi
+xapi-storage-plugins;ignored;proprietarized, forked as xcp-ng-xapi-storage
+xapi-storage-script;ignored;now in xapi
+xcp-networkd;ignored;now in xapi
+xcp-rrdd;ignored;now in xapi
+xencert;;"automated testkit for certifying storage hardware with XenServer"
+xenopsd;ignored;now in xapi
+xenserver-release;forked;xcp-ng-release
+xenserver-snmp-agent;ignored;proprietary, SNMP MIB
+xenserver-telemetry;ignored;proprietary, xapi plugin
+xs-clipboardd;ignored;proprietary, replaced by xcp-clipboardd

Diff for: scripts/rpmwatcher/repoquery.py

@@ -12,22 +12,59 @@
 skip_if_unavailable=False
 """
 
+XCPNG_YUMREPO_USER_TMPL = """
+[xcpng-{section}{suffix}]
+name=xcpng - {section}{suffix}
+baseurl=https://koji.xcp-ng.org/repos/user/8/{version}/{section}/{rpmarch}/
+gpgkey=https://xcp-ng.org/RPM-GPG-KEY-xcpng
+failovermethod=priority
+skip_if_unavailable=False
+"""
+
 def setup_xcpng_yum_repos(*, yum_repo_d, sections, bin_arch, version):
     with open(os.path.join(yum_repo_d, "xcpng.repo"), "w") as yumrepoconf:
         for section in sections:
+            # HACK: use USER_TMPL if section ends with a number
+            if section[-1].isdigit():
+                tmpl = XCPNG_YUMREPO_USER_TMPL
+            else:
+                tmpl = XCPNG_YUMREPO_TMPL
+
             # binaries
-            block = XCPNG_YUMREPO_TMPL.format(rpmarch=bin_arch,
-                                              section=section,
-                                              version=version,
-                                              suffix='',
-                                              )
-            yumrepoconf.write(block)
+            if bin_arch:
+                block = tmpl.format(rpmarch=bin_arch,
+                                    section=section,
+                                    version=version,
+                                    suffix='',
+                                    )
+                yumrepoconf.write(block)
             # sources
-            block = XCPNG_YUMREPO_TMPL.format(rpmarch='Source',
-                                              section=section,
-                                              version=version,
-                                              suffix='-src',
-                                              )
+            block = tmpl.format(rpmarch='Source',
+                                section=section,
+                                version=version,
+                                suffix='-src',
+                                )
+            yumrepoconf.write(block)
+
+
+XS8_YUMREPO_TMPL = """
+[xs8-{section}]
+name=XS8 - {section}
+baseurl=http://10.1.0.94/repos/XS8/{section}/xs8p-{section}/
+failovermethod=priority
+skip_if_unavailable=False
+
+[xs8-{section}-src]
+name=XS8 - {section} source
+baseurl=http://10.1.0.94/repos/XS8/{section}/xs8p-{section}-source/
+failovermethod=priority
+skip_if_unavailable=False
+"""
+
+def setup_xs8_yum_repos(*, yum_repo_d, sections):
+    with open(os.path.join(yum_repo_d, "xs8.repo"), "w") as yumrepoconf:
+        for section in sections:
+            block = XS8_YUMREPO_TMPL.format(section=section)
             yumrepoconf.write(block)
 
 DNF_BASE_CMD = None
@@ -47,7 +84,7 @@ def run_repoquery(args):
     cmd = DNF_BASE_CMD + ['repoquery'] + args
     logging.debug('$ %s', ' '.join(cmd))
     ret = subprocess.check_output(cmd, universal_newlines=True).strip().split()
-    logging.debug('> %s', ret)
+    logging.debug('> %s', '\n'.join(ret))
     return ret
 
 SRPM_BINRPMS_CACHE = {}         # binrpm-nevr -> srpm-nevr
@@ -141,3 +178,36 @@ def is_pristine_upstream(rpmname):
     if re.search(UPSTREAM_REGEX, rpmname):
         return True
     return False
+
+def rpm_parse_nevr(nevr, suffix):
+    "Parse into (name, epoch:version, release) stripping suffix from release"
+    m = re.match(RPM_NVR_SPLIT_REGEX, nevr)
+    assert m, f"{nevr} does not match NEVR pattern"
+    n, ev, r = m.groups()
+    if ":" in ev:
+        e, v = ev.split(":")
+    else:
+        e, v = "0", ev
+    if r.endswith(suffix):
+        r = r[:-len(suffix)]
+    return (n, e, v, r)
+
+def all_binrpms():
+    args = [
+        '--disablerepo=*-src',
+        '--qf=%{name}-%{evr}', # to avoid getting the arch
+        '--latest-limit=1',    # only most recent for each package
+        '*',
+    ]
+    ret = set(run_repoquery(args))
+    return ret
+
+def all_srpms():
+    args = [
+        '--disablerepo=*', '--enablerepo=*-src',
+        '--qf=%{name}-%{evr}', # to avoid getting the arch
+        '--latest-limit=1',    # only most recent for each package
+        '*',
+    ]
+    ret = set(run_repoquery(args))
+    return ret

Diff for: scripts/rpmwatcher/style.css

@@ -0,0 +1,53 @@
+table {
+    border-collapse: collapse;
+    border-spacing: 0;
+}
+td {
+    padding: 2px 5px;
+    /* background-color: lightblue; */
+}
+
+tr.header {
+    vertical-align: top;
+}
+tr.ignored {
+    color: #888888;
+    background-color: #cccccc;
+}
+tr.notused { /* still to check */
+    color: red;
+    background-color: #cccccc;
+}
+
+th.xs {
+    background-color: #ddddff;
+}
+th.xcp {
+    background-color: #ddffdd;
+}
+
+.nosource {
+    color: red;
+    background-color: black;
+}
+
+.unexpected {
+    color: red;
+}
+
+.outdated {
+    background-color: red;
+}
+
+.uptodate {
+    background-color: lightgreen;
+}
+
+.better {
+    background-color: #77cc00;
+}
+
+.upstream {
+    /* must not be background-color, can exist in outdated/uptodate/better */
+    color: blue;
+}