refresh-plugin-download-links.py

#!/usr/bin/env python3
"""Refresh plugin download-link tables in plugin-catalog docs.

Starting with EMQX 5.10 (enterprise) and 6.2 (unified), plugins live in
the emqx.git monorepo under `plugins/<name>/` and each has a `VERSION`
file. For every stable release tag in the chosen series, the CI
publishes a tarball at:

    https://packages.emqx.io/emqx-plugins/<tag>/<plugin>-<plugin-ver>.tar.gz

where `<tag>` is the literal git tag — bare semver for 6.x (e.g. `6.2.0`)
and `e`-prefixed for 5.x enterprise (e.g. `e5.10.4`).

This script walks the stable tags of one series (e.g. 5.10 or 6.2),
reads each plugin's VERSION file, and regenerates a "Download" table
inside every matching markdown file under the version-scoped catalog dir
`en_US|zh_CN|ja_JP/extensions/plugin-catalog/<major>.<minor>/` (e.g.
`.../plugin-catalog/6.0/`). Pages are namespaced by series so that
release branches never edit the same page files, keeping
release-6.N -> release-6.(N+1) sync merges free of content conflicts on
the plugin pages. The series defaults to the current docs-root branch
(`release-N.M`); override with `--series N.M`.

If an `en_US` catalog page does not yet exist for a discovered plugin,
the script bootstraps it by copying the plugin's `README.md` from the
latest stable tag of the series, then splices the Download table onto
the end. Missing `zh_CN`/`ja_JP` pages are left alone (the established
workflow hand-translates them from the seeded `en_US` page); pass
`--bootstrap-locales en_US,zh_CN` to opt zh_CN in as well.

Markdown filenames use dashes (e.g. `emqx-bridge-mqtt-dq.md`) whereas
plugin directory names use underscores (e.g. `emqx_bridge_mqtt_dq`);
the script maps between them by replacing `-` with `_`.

Generated content is bounded by HTML-comment markers so the script is
idempotent. If the markers are missing, a new Download section is
appended at the end of the file. When the markers are already present,
the script preserves the existing prose (heading, intro, column
headers) and only appends data rows for release tags that are not yet
listed — translator-polished wording in locales such as `ja_JP` is
left intact instead of being overwritten by the `LOCALES` constants.
"""

from __future__ import annotations

import argparse
import os
import re
import shutil
import subprocess
import sys
import tempfile
from dataclasses import dataclass
from pathlib import Path
from typing import Iterable

DEFAULT_EMQX_REMOTE = "https://github.com/emqx/emqx.git"
PACKAGE_URL_TEMPLATE = (
    "https://packages.emqx.io/emqx-plugins/{tag}/{plugin}-{plugin_ver}.tar.gz"
)

# Stable release tags. Two conventions are in use:
#   - 6.x:  bare semver, e.g. "6.2.0"
#   - 5.x:  enterprise prefix, e.g. "e5.10.4"
# Pre-release suffixes (-rc.1 / -alpha.1 / -M2) are excluded.
STABLE_TAG_RE = re.compile(r"^(?P<prefix>e)?(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)$")
SERIES_RE = re.compile(r"^(\d+)\.(\d+)$")
BRANCH_RE = re.compile(r"^release-(\d+)\.?(\d+)$")

BEGIN_MARKER = "<!-- PLUGIN-DOWNLOADS:BEGIN (auto-generated, do not edit) -->"
END_MARKER = "<!-- PLUGIN-DOWNLOADS:END -->"

LOCALES: dict[str, dict[str, str]] = {
    "en_US": {
        "heading": "## Download",
        "intro": "Tarballs for each EMQX release:",
        "col_emqx": "EMQX Version",
        "col_plugin": "Plugin Version",
        "col_link": "Package",
    },
    "zh_CN": {
        "heading": "## 下载",
        "intro": "各 EMQX 版本对应的插件安装包：",
        "col_emqx": "EMQX 版本",
        "col_plugin": "插件版本",
        "col_link": "安装包",
    },
    "ja_JP": {
        "heading": "## ダウンロード",
        "intro": "各 EMQX リリースに対応するプラグインパッケージ:",
        "col_emqx": "EMQX バージョン",
        "col_plugin": "プラグインバージョン",
        "col_link": "パッケージ",
    },
}

CATALOG_BASE = Path("extensions/plugin-catalog")


def catalog_subpath(series: tuple[int, int]) -> Path:
    """Version-scoped catalog dir, e.g. extensions/plugin-catalog/6.0.

    Pages live under a `<major>.<minor>` subdir so that release branches
    (release-6.0, release-6.1, ...) never edit the same page files; this
    keeps release-6.N -> release-6.(N+1) sync merges free of content
    conflicts on the plugin pages.
    """
    return CATALOG_BASE / f"{series[0]}.{series[1]}"


@dataclass(frozen=True)
class ReleaseTag:
    raw: str  # original tag, used verbatim in URL: "6.2.0" or "e5.10.4"
    major: int
    minor: int
    patch: int

    @property
    def display_version(self) -> str:
        """Human-readable semver shown in the table (no 'e' prefix)."""
        return f"{self.major}.{self.minor}.{self.patch}"

    @property
    def sort_key(self) -> tuple[int, int, int]:
        return (self.major, self.minor, self.patch)


def run_git(repo: Path, *args: str) -> str:
    result = subprocess.run(
        ["git", "-C", str(repo), *args],
        check=True,
        capture_output=True,
        text=True,
    )
    return result.stdout


def try_git(repo: Path, *args: str) -> str | None:
    """Run git; return None if it fails (e.g. file missing at tag)."""
    result = subprocess.run(
        ["git", "-C", str(repo), *args],
        capture_output=True,
        text=True,
    )
    if result.returncode != 0:
        return None
    return result.stdout


def ensure_repo(supplied: Path | None, workdir: Path) -> Path:
    """Return a usable emqx repo; clone into workdir if none supplied."""
    if supplied is not None:
        if not (supplied / ".git").exists():
            sys.exit(f"error: {supplied} is not a git repository")
        print(f"Using existing emqx clone: {supplied}", file=sys.stderr)
        run_git(supplied, "fetch", "--tags", "--quiet")
        return supplied

    target = workdir / "emqx"
    print(f"Cloning {DEFAULT_EMQX_REMOTE} into {target} ...", file=sys.stderr)
    subprocess.run(
        ["git", "clone", "--filter=blob:none", "--no-checkout", DEFAULT_EMQX_REMOTE, str(target)],
        check=True,
    )
    return target


def list_release_tags(repo: Path, series: tuple[int, int]) -> list[ReleaseTag]:
    """Return stable tags matching the given (major, minor) series.

    `series` is e.g. (5, 10) or (6, 2). Only tags using the conventional
    prefix for that major are accepted (5.x → 'e' prefix; 6.x → no prefix).
    """
    out = run_git(repo, "tag", "--list")
    want_major, want_minor = series
    tags: list[ReleaseTag] = []
    for line in out.splitlines():
        raw = line.strip()
        m = STABLE_TAG_RE.match(raw)
        if not m:
            continue
        has_prefix = bool(m.group("prefix"))
        major = int(m.group("major"))
        minor = int(m.group("minor"))
        patch = int(m.group("patch"))
        if (major, minor) != (want_major, want_minor):
            continue
        # Enforce prefix convention: 5.x uses 'e', 6.x uses none.
        if major == 5 and not has_prefix:
            continue
        if major >= 6 and has_prefix:
            continue
        tags.append(ReleaseTag(raw=raw, major=major, minor=minor, patch=patch))
    tags.sort(key=lambda t: t.sort_key)
    return tags


def list_plugins_at(repo: Path, tag: str) -> list[str]:
    """Return plugin directory names present under `plugins/` at the tag."""
    out = try_git(repo, "ls-tree", "--name-only", f"{tag}:plugins")
    if out is None:
        return []
    return sorted(p.strip() for p in out.splitlines() if p.strip())


def read_plugin_version(repo: Path, tag: str, plugin: str) -> str | None:
    out = try_git(repo, "show", f"{tag}:plugins/{plugin}/VERSION")
    if out is None:
        return None
    version = out.strip()
    return version or None


def read_plugin_readme(repo: Path, tag: str, plugin: str) -> str | None:
    out = try_git(repo, "show", f"{tag}:plugins/{plugin}/README.md")
    if out is None:
        return None
    return out


def build_rows(
    repo: Path, tags: Iterable[ReleaseTag]
) -> dict[str, list[tuple[ReleaseTag, str]]]:
    """Return plugin -> [(tag, plugin_ver), ...] sorted by tag asc."""
    rows: dict[str, list[tuple[ReleaseTag, str]]] = {}
    for tag in tags:
        plugins = list_plugins_at(repo, tag.raw)
        for plugin in plugins:
            pver = read_plugin_version(repo, tag.raw, plugin)
            if not pver:
                continue
            rows.setdefault(plugin, []).append((tag, pver))
    return rows


def collect_readmes(
    repo: Path, rows: dict[str, list[tuple[ReleaseTag, str]]]
) -> dict[str, str]:
    """Return plugin -> README.md content from the latest tag where it appears."""
    readmes: dict[str, str] = {}
    for plugin, entries in rows.items():
        latest_tag = entries[-1][0].raw
        readme = read_plugin_readme(repo, latest_tag, plugin)
        if readme:
            readmes[plugin] = readme
    return readmes


def render_section(
    locale: str, plugin: str, entries: list[tuple[ReleaseTag, str]]
) -> str:
    labels = LOCALES[locale]
    lines: list[str] = []
    lines.append(BEGIN_MARKER)
    lines.append("")
    lines.append(labels["heading"])
    lines.append("")
    lines.append(labels["intro"])
    lines.append("")
    lines.append(
        f"| {labels['col_emqx']} | {labels['col_plugin']} | {labels['col_link']} |"
    )
    lines.append("|---|---|---|")
    for tag, plugin_ver in entries:
        url = PACKAGE_URL_TEMPLATE.format(
            tag=tag.raw, plugin=plugin, plugin_ver=plugin_ver
        )
        filename = f"{plugin}-{plugin_ver}.tar.gz"
        lines.append(
            f"| {tag.display_version} | {plugin_ver} | [{filename}]({url}) |"
        )
    lines.append("")
    lines.append(END_MARKER)
    return "\n".join(lines)


# A markdown table row whose first column is an EMQX version like "5.10.4"
# or "6.2.0" and third column is the package link `[name](url)`. The middle
# column (plugin version) is matched loosely.
DATA_ROW_RE = re.compile(
    r"^\|\s*(?P<display_version>\d+\.\d+\.\d+)\s*\|\s*[^|]+\|\s*\[[^\]]+\]\([^)]+\)\s*\|\s*$"
)
SEPARATOR_ROW_RE = re.compile(r"^\|\s*-+\s*(?:\|\s*-+\s*)+\|\s*$")


def parse_existing_versions(block: str) -> set[str]:
    """Return EMQX versions already listed as data rows inside an existing block."""
    return {
        m.group("display_version")
        for line in block.splitlines()
        if (m := DATA_ROW_RE.match(line)) is not None
    }


def render_data_row(plugin: str, tag: ReleaseTag, plugin_ver: str) -> str:
    url = PACKAGE_URL_TEMPLATE.format(
        tag=tag.raw, plugin=plugin, plugin_ver=plugin_ver
    )
    filename = f"{plugin}-{plugin_ver}.tar.gz"
    return f"| {tag.display_version} | {plugin_ver} | [{filename}]({url}) |"


def append_new_rows(
    block: str, plugin: str, missing: list[tuple[ReleaseTag, str]]
) -> str:
    """Append rows for `missing` tags into an existing block, preserving prose."""
    lines = block.split("\n")
    insert_after = -1
    for i, line in enumerate(lines):
        if DATA_ROW_RE.match(line) or SEPARATOR_ROW_RE.match(line):
            insert_after = i
    new_rows = [render_data_row(plugin, tag, pver) for tag, pver in missing]
    if insert_after >= 0:
        return "\n".join(lines[: insert_after + 1] + new_rows + lines[insert_after + 1 :])
    last_nonblank = len(lines) - 1
    while last_nonblank >= 0 and lines[last_nonblank].strip() == "":
        last_nonblank -= 1
    table = ["|---|---|---|"] + new_rows
    return "\n".join(lines[: last_nonblank + 1] + table + lines[last_nonblank + 1 :])


def splice_section(
    original: str,
    locale: str,
    plugin: str,
    entries: list[tuple[ReleaseTag, str]],
) -> str:
    """Update or insert the Download section, preserving translator-polished prose."""
    begin_idx = original.find(BEGIN_MARKER)
    end_idx = original.find(END_MARKER)
    if begin_idx == -1 or end_idx == -1 or end_idx <= begin_idx:
        new_section = render_section(locale, plugin, entries)
        trimmed = original.rstrip() + "\n\n"
        return trimmed + new_section + "\n"

    block_start = begin_idx + len(BEGIN_MARKER)
    existing_block = original[block_start:end_idx]
    existing_versions = parse_existing_versions(existing_block)
    missing = [
        (tag, pver)
        for tag, pver in entries
        if tag.display_version not in existing_versions
    ]
    if not missing:
        return original
    updated_block = append_new_rows(existing_block, plugin, missing)
    return original[:block_start] + updated_block + original[end_idx:]


def md_filename_for(plugin: str) -> str:
    return plugin.replace("_", "-") + ".md"


def update_catalog_files(
    docs_root: Path,
    rows: dict[str, list[tuple[ReleaseTag, str]]],
    readmes: dict[str, str],
    locales: list[str],
    subpath: Path,
    bootstrap_locales: set[str],
    dry_run: bool,
) -> int:
    changed = 0
    for plugin, entries in sorted(rows.items()):
        fname = md_filename_for(plugin)
        for locale in locales:
            path = docs_root / locale / subpath / fname
            existed = path.exists()
            new_section = render_section(locale, plugin, entries)
            if existed:
                original = path.read_text(encoding="utf-8")
            elif locale in bootstrap_locales and plugin in readmes:
                original = readmes[plugin].rstrip() + "\n"
            else:
                continue
            updated = splice_section(original, new_section)
            if existed and updated == original:
                print(f"unchanged: {path.relative_to(docs_root)}", file=sys.stderr)
                continue
            changed += 1
            verb = "create" if not existed else "update"
            if dry_run:
                print(f"would {verb}: {path.relative_to(docs_root)}", file=sys.stderr)
            else:
                path.parent.mkdir(parents=True, exist_ok=True)
                path.write_text(updated, encoding="utf-8")
                print(f"{verb}d: {path.relative_to(docs_root)}", file=sys.stderr)
    return changed


def detect_series_from_branch(docs_root: Path) -> tuple[int, int] | None:
    """Read the docs-root git branch and parse 'release-N.M' / 'release-NM'."""
    out = try_git(docs_root, "rev-parse", "--abbrev-ref", "HEAD")
    if out is None:
        return None
    branch = out.strip()
    m = BRANCH_RE.match(branch)
    if not m:
        return None
    return int(m.group(1)), int(m.group(2))


def resolve_series(flag: str | None, docs_root: Path) -> tuple[int, int]:
    if flag is not None:
        m = SERIES_RE.match(flag)
        if not m:
            sys.exit(f"error: --series must look like 'N.M' (got {flag!r})")
        return int(m.group(1)), int(m.group(2))
    detected = detect_series_from_branch(docs_root)
    if detected is None:
        sys.exit(
            "error: could not auto-detect release series from docs-root branch; "
            "pass --series N.M (e.g. --series 5.10)"
        )
    return detected


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Refresh plugin download-link tables in plugin-catalog docs.",
    )
    parser.add_argument(
        "--emqx-repo",
        type=Path,
        default=None,
        help="Path to an existing emqx.git clone (tags will be fetched). "
             "Default: clone a fresh blobless copy into a temp dir.",
    )
    parser.add_argument(
        "--docs-root",
        type=Path,
        default=Path(__file__).resolve().parent,
        help="Root of emqx-docs (must contain en_US/zh_CN/ja_JP dirs). "
             "Default: directory containing this script.",
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="Show which files would change without writing.",
    )
    parser.add_argument(
        "--series",
        default=None,
        help="Release series to scan, e.g. '5.10' or '6.2'. "
             "Default: auto-detect from the docs-root branch name (release-N.M).",
    )
    parser.add_argument(
        "--bootstrap-locales",
        default="en_US",
        help="Comma-separated list of locales whose missing catalog pages "
             "should be seeded from the plugin README (latest stable tag). "
             "Locales not in this list silently skip missing files. "
             "Only locales whose plugin-catalog dir already exists are "
             "considered. Default: en_US.",
    )
    args = parser.parse_args()

    docs_root: Path = args.docs_root.resolve()
    present_locales = [
        loc for loc in LOCALES if (docs_root / loc / CATALOG_BASE).is_dir()
    ]
    if not present_locales:
        sys.exit(
            f"error: no plugin-catalog dirs found under {docs_root} "
            f"(looked for {'/'.join(LOCALES)}/{CATALOG_BASE})"
        )

    series = resolve_series(args.series, docs_root)
    subpath = catalog_subpath(series)
    print(
        f"Targeting release series {series[0]}.{series[1]} "
        f"(catalog dir: {subpath})",
        file=sys.stderr,
    )

    tmp_ctx: tempfile.TemporaryDirectory | None = None
    try:
        if args.emqx_repo is None:
            tmp_ctx = tempfile.TemporaryDirectory(prefix="emqx-plugins-")
            workdir = Path(tmp_ctx.name)
        else:
            workdir = Path(".")
        repo = ensure_repo(args.emqx_repo.resolve() if args.emqx_repo else None, workdir)

        tags = list_release_tags(repo, series)
        if not tags:
            print(
                f"warning: no stable tags found for series {series[0]}.{series[1]}",
                file=sys.stderr,
            )
            return 0
        print(
            f"Found {len(tags)} stable tag(s): "
            f"{tags[0].raw} ... {tags[-1].raw}",
            file=sys.stderr,
        )

        rows = build_rows(repo, tags)
        if not rows:
            print("warning: no plugins discovered in any tag", file=sys.stderr)
            return 0
        print(f"Discovered {len(rows)} plugin(s): {', '.join(sorted(rows))}", file=sys.stderr)

        bootstrap_locales = {
            loc.strip() for loc in args.bootstrap_locales.split(",") if loc.strip()
        }
        unknown = bootstrap_locales - set(LOCALES)
        if unknown:
            sys.exit(
                f"error: --bootstrap-locales contains unknown locale(s): "
                f"{', '.join(sorted(unknown))}; valid: {', '.join(LOCALES)}"
            )

        readmes = collect_readmes(repo, rows) if bootstrap_locales else {}
        missing_readmes = sorted(set(rows) - set(readmes))
        if bootstrap_locales and missing_readmes:
            print(
                f"warning: no README.md found for: {', '.join(missing_readmes)} "
                f"(missing pages in {sorted(bootstrap_locales)} will be skipped)",
                file=sys.stderr,
            )

        changed = update_catalog_files(
            docs_root, rows, readmes, present_locales, subpath,
            bootstrap_locales, args.dry_run
        )
        print(f"Done. {changed} file(s) {'to change' if args.dry_run else 'changed'}.", file=sys.stderr)
        return 0
    finally:
        if tmp_ctx is not None:
            tmp_ctx.cleanup()


if __name__ == "__main__":
    raise SystemExit(main())