Resolve all URLs for markdown

Author: snejus

.github/workflows/ci.yaml

@@ -34,7 +34,8 @@ jobs:
         run: |
           sudo apt update
           sudo apt install ffmpeg gobject-introspection libgirepository1.0-dev pandoc
-          poetry install --with=release --extras=replaygain --extras=reflink
+          poetry install --with=release --extras=docs --extras=replaygain --extras=reflink
+          poe docs
 
       - name: Install Python dependencies
         run: poetry install --only=main,test --extras=autobpm

.github/workflows/make_release.yaml

@@ -65,6 +65,7 @@ jobs:
       - name: Obtain the changelog
         id: generate_changelog
         run: |
+          poe docs
           {
             echo 'changelog<<EOF'
             poe --quiet changelog

extra/release.py

@@ -6,40 +6,136 @@
 
 import re
 import subprocess
+from contextlib import redirect_stdout
 from datetime import datetime, timezone
 from functools import partial
+from io import StringIO
 from pathlib import Path
-from typing import Callable
+from typing import Callable, NamedTuple
 
 import click
 import tomli
 from packaging.version import Version, parse
+from sphinx.ext import intersphinx
 from typing_extensions import TypeAlias
 
 BASE = Path(__file__).parent.parent.absolute()
 PYPROJECT = BASE / "pyproject.toml"
 CHANGELOG = BASE / "docs" / "changelog.rst"
 DOCS = "https://beets.readthedocs.io/en/stable"
 
-version_header = r"\d+\.\d+\.\d+ \([^)]+\)"
+VERSION_HEADER = r"\d+\.\d+\.\d+ \([^)]+\)"
 RST_LATEST_CHANGES = re.compile(
-    rf"{version_header}\n--+\s+(.+?)\n\n+{version_header}", re.DOTALL
+    rf"{VERSION_HEADER}\n--+\s+(.+?)\n\n+{VERSION_HEADER}", re.DOTALL
 )
+
 Replacement: TypeAlias = "tuple[str, str | Callable[[re.Match[str]], str]]"
-RST_REPLACEMENTS: list[Replacement] = [
-    (r"(?<=\n) {3,4}(?=\*)", "  "),  # fix indent of nested bullet points ...
-    (r"(?<=\n) {5,6}(?=[\w:`])", "    "),  # ... and align wrapped text indent
-    (r"(?<=[\s(])(`[^`]+`)(?!_)", r"`\1`"),  # double quotes for inline code
-    (r":bug:`(\d+)`", r":bug: (#\1)"),  # Issue numbers.
-    (r":user:`(\w+)`", r"\@\1"),  # Users.
-]
+
+
+class Ref(NamedTuple):
+    """A reference to documentation with ID, path, and optional title."""
+
+    id: str
+    path: str | None
+    title: str | None
+
+    @classmethod
+    def from_line(cls, line: str) -> Ref:
+        """Create Ref from a Sphinx objects.inv line.
+
+        Each line has the following structure:
+        <id>    [optional title : ] <relative-url-path>
+
+        """
+        if len(line_parts := line.split(" ", 1)) == 1:
+            return cls(line, None, None)
+
+        id, path_with_name = line_parts
+        parts = [p.strip() for p in path_with_name.split(":", 1)]
+
+        if len(parts) == 1:
+            path, name = parts[0], None
+        else:
+            name, path = parts
+
+        return cls(id, path, name)
+
+    @property
+    def url(self) -> str:
+        """Full documentation URL."""
+        return f"{DOCS}/{self.path}"
+
+    @property
+    def name(self) -> str:
+        """Display name (title if available, otherwise ID)."""
+        return self.title or self.id
+
+
+def get_refs() -> dict[str, Ref]:
+    """Parse Sphinx objects.inv and return dict of documentation references."""
+    objects_filepath = Path("docs/_build/html/objects.inv")
+    if not objects_filepath.exists():
+        raise ValueError("Documentation does not exist. Run 'poe docs' first.")
+
+    captured_output = StringIO()
+
+    with redirect_stdout(captured_output):
+        intersphinx.inspect_main([str(objects_filepath)])
+
+    return {
+        r.id: r
+        for ln in captured_output.getvalue().split("\n")
+        if ln.startswith("\t") and (r := Ref.from_line(ln.strip()))
+    }
+
+
+def create_rst_replacements() -> list[Replacement]:
+    """Generate list of pattern replacements for RST changelog."""
+    refs = get_refs()
+
+    def make_ref_link(ref_id: str, name: str | None = None) -> str:
+        ref = refs[ref_id]
+        return rf"`{name or ref.name} <{ref.url}>`_"
+
+    commands = "|".join(r.split("-")[0] for r in refs if r.endswith("-cmd"))
+    plugins = "|".join(
+        r.split("/")[-1] for r in refs if r.startswith("plugins/")
+    )
+    return [
+        # Fix nested bullet points indent: use 2 spaces consistently
+        (r"(?<=\n) {3,4}(?=\*)", "  "),
+        # Fix nested text indent: use 4 spaces consistently
+        (r"(?<=\n) {5,6}(?=[\w:`])", "    "),
+        # Replace Sphinx :ref: and :doc: directives by documentation URLs
+        #   :ref:`/plugins/autobpm` -> [AutoBPM Plugin](DOCS/plugins/autobpm.html)
+        (
+            r":(?:ref|doc):`+(?:([^`<]+)<)?/?([\w./_-]+)>?`+",
+            lambda m: make_ref_link(m[2], m[1]),
+        ),
+        # Convert command references to documentation URLs
+        #   `beet move` or `move` command -> [import](DOCS/reference/cli.html#import)
+        (
+            rf"`+beet ({commands})`+|`+({commands})`+(?= command)",
+            lambda m: make_ref_link(f"{m[1] or m[2]}-cmd"),
+        ),
+        # Convert plugin references to documentation URLs
+        #   `fetchart` plugin -> [fetchart](DOCS/plugins/fetchart.html)
+        (rf"`+({plugins})`+", lambda m: make_ref_link(f"plugins/{m[1]}")),
+        # Add additional backticks around existing backticked text to ensure it
+        # is rendered as inline code in Markdown
+        (r"(?<=[\s])(`[^`]+`)(?!_)", r"`\1`"),
+        # Convert bug references to GitHub issue links
+        (r":bug:`(\d+)`", r":bug: (#\1)"),
+        # Convert user references to GitHub @mentions
+        (r":user:`(\w+)`", r"\@\1"),
+    ]
+
+
 MD_REPLACEMENTS: list[Replacement] = [
     (r"^  (- )", r"\1"),  # remove indent from top-level bullet points
     (r"^ +(  - )", r"\1"),  # adjust nested bullet points indent
     (r"^(\w[^\n]{,80}):(?=\n\n[^ ])", r"### \1"),  # format section headers
     (r"^(\w[^\n]{81,}):(?=\n\n[^ ])", r"**\1**"),  # and bolden too long ones
-    (r"^- `/?plugins/(\w+)`:?", rf"- Plugin [\1]({DOCS}/plugins/\1.html):"),
-    (r"^- `(\w+)-cmd`:?", rf"- Command [\1]({DOCS}/reference/cli.html#\1):"),
     (r"### [^\n]+\n+(?=### )", ""),  # remove empty sections
 ]
 order_bullet_points = partial(
@@ -123,7 +219,7 @@ def rst2md(text: str) -> str:
     """Use Pandoc to convert text from ReST to Markdown."""
     return (
         subprocess.check_output(
-            ["pandoc", "--from=rst", "--to=gfm", "--wrap=none"],
+            ["pandoc", "--from=rst", "--to=gfm+hard_line_breaks"],
             input=text.encode(),
         )
         .decode()
@@ -132,7 +228,6 @@ def rst2md(text: str) -> str:
 
 
 def get_changelog_contents() -> str | None:
-    return CHANGELOG.read_text()
     if m := RST_LATEST_CHANGES.search(CHANGELOG.read_text()):
         return m.group(1)
 
@@ -141,8 +236,8 @@ def get_changelog_contents() -> str | None:
 
 def changelog_as_markdown(rst: str) -> str:
     """Get the latest changelog entry as hacked up Markdown."""
-    for pattern, repl in RST_REPLACEMENTS:
-        rst = re.sub(pattern, repl, rst, flags=re.M)
+    for pattern, repl in create_rst_replacements():
+        rst = re.sub(pattern, repl, rst, flags=re.M | re.DOTALL)
 
     md = rst2md(rst)
 
@@ -170,7 +265,10 @@ def bump(version: Version) -> None:
 def changelog():
     """Get the most recent version's changelog as Markdown."""
     if changelog := get_changelog_contents():
-        print(changelog_as_markdown(changelog))
+        try:
+            print(changelog_as_markdown(changelog))
+        except ValueError as e:
+            raise click.exceptions.UsageError(str(e))
 
 
 if __name__ == "__main__":

test/test_release.py

@@ -6,7 +6,8 @@
 
 import pytest
 
-from extra.release import changelog_as_markdown
+release = pytest.importorskip("extra.release")
+
 
 pytestmark = pytest.mark.skipif(
     not (
@@ -69,8 +70,8 @@ def rst_changelog():
 def md_changelog():
     return r"""### New features
 
-- Command [list](https://beets.readthedocs.io/en/stable/reference/cli.html#list): Update.
-- Plugin [substitute](https://beets.readthedocs.io/en/stable/plugins/substitute.html): Some substitute multi-line change. :bug: (\#5467)
+- [Substitute Plugin](https://beets.readthedocs.io/en/stable/plugins/substitute.html): Some substitute multi-line change. :bug: (\#5467)
+- [list](https://beets.readthedocs.io/en/stable/reference/cli.html#list-cmd) Update.
 
 You can do something with this command:
 
@@ -102,6 +103,6 @@ def md_changelog():
 
 
 def test_convert_rst_to_md(rst_changelog, md_changelog):
-    actual = changelog_as_markdown(rst_changelog)
+    actual = release.changelog_as_markdown(rst_changelog)
 
     assert actual == md_changelog