fix: add opt-in ASCII slug generation (#97)

Author: tylerdave

docs/Configuration.md

@@ -21,6 +21,7 @@ Configuration lives in `rockgarden.toml` at the site root.
 | `url_style`   | `str`  | `"slug"`    | How filenames become URLs. `"slug"` (lowercase, dashes), `"preserve-case"` (original casing, dashes), or `"preserve"` (original casing and spacing). See URL Styles below. |
 | `base_url`    | `str`  | `""`        | Full base URL (e.g. `https://example.com/docs`). Used for feeds, sitemap, and deriving `base_path` when not set explicitly. Trailing slash is stripped automatically. |
 | `base_path`   | `str`  | `""`        | URL path prefix for subdirectory deploys (e.g. `/docs`). When set, all generated URLs are prefixed with this path. If not set, derived from `base_url`. Trailing slash is stripped automatically. |
+| `ascii_urls`  | `bool` | `false`     | Transliterate Unicode characters to ASCII in URLs and tag slugs. `Café` → `cafe`, `東京` → `dong-jing`. Applies to all URL styles. |
 
 ### URL Styles
 

pyproject.toml

@@ -30,6 +30,7 @@ dependencies = [
     "pyyaml>=6.0",
     "markdownify>=0.14.1",
     "pygments>=2.17.0",
+    "python-slugify>=8.0.0",
     "watchfiles>=1.0.0",
 ]
 

src/rockgarden/config.py

@@ -18,6 +18,7 @@ class SiteConfig(BaseModel):
     base_url: str = ""
     base_path: str = ""
     url_style: str = "slug"
+    ascii_urls: bool = False
 
     @field_validator("url_style", mode="after")
     @classmethod

src/rockgarden/content/loader.py

@@ -57,26 +57,33 @@ def should_ignore(path: Path, source: Path, patterns: list[str]) -> bool:
     return False
 
 
-def path_to_slug(path: Path, source: Path, url_style: str = "slug") -> str:
+def path_to_slug(
+    path: Path,
+    source: Path,
+    url_style: str = "slug",
+    ascii_urls: bool = False,
+) -> str:
     """Convert a file path to a URL slug.
 
     Args:
         path: The file path.
         source: The source root directory.
         url_style: URL style ("slug", "preserve-case", or "preserve").
+        ascii_urls: When True, transliterate Unicode to ASCII.
 
     Returns:
         The slug (e.g., 'index', 'npcs/olvir').
     """
     rel_path = str(path.relative_to(source))
-    return generate_slug(rel_path, style=url_style)
+    return generate_slug(rel_path, style=url_style, ascii_urls=ascii_urls)
 
 
 def load_page(
     path: Path,
     source: Path,
     dates_config: DatesConfig | None = None,
     url_style: str = "slug",
+    ascii_urls: bool = False,
 ) -> Page:
     """Load a single page from a markdown file.
 
@@ -97,7 +104,7 @@ def load_page(
     if custom_slug := metadata.get("slug"):
         slug = custom_slug
     else:
-        slug = path_to_slug(path, source, url_style)
+        slug = path_to_slug(path, source, url_style, ascii_urls)
 
     modified = _resolve_frontmatter_date(metadata, dates_config.modified_date_fields)
     if modified is None and dates_config.modified_date_fallback:
@@ -120,6 +127,7 @@ def load_content(
     ignore_patterns: list[str],
     dates_config: DatesConfig | None = None,
     url_style: str = "slug",
+    ascii_urls: bool = False,
 ) -> list[Page]:
     """Discover and load all markdown files from source directory.
 
@@ -137,7 +145,7 @@ def load_content(
         if should_ignore(path, source, ignore_patterns):
             continue
 
-        page = load_page(path, source, dates_config, url_style)
+        page = load_page(path, source, dates_config, url_style, ascii_urls)
         pages.append(page)
 
     return pages

src/rockgarden/output/builder.py

@@ -420,7 +420,11 @@ def build_site(
     run_hooks(config.hooks.pre_build, "pre_build", cwd=site_root, env_vars=hook_env)
 
     pages = load_content(
-        source, config.build.ignore_patterns, config.dates, config.site.url_style
+        source,
+        config.build.ignore_patterns,
+        config.dates,
+        config.site.url_style,
+        config.site.ascii_urls,
     )
     clean_urls = config.site.clean_urls
     base_path = config.site.base_path or get_base_path(config.site.base_url)
@@ -789,11 +793,18 @@ def build_site(
 
     # Generate tag index pages if enabled
     if config.theme.tag_index:
-        tags = collect_tags(pages)
+        tags = collect_tags(pages, config.site.ascii_urls)
         if tags:
             tag_layout = resolve_layout({}, config.theme.default_layout)
             build_tag_pages(
-                tags, env, site_config, output, clean_urls, base_path, tag_layout
+                tags,
+                env,
+                site_config,
+                output,
+                clean_urls,
+                base_path,
+                tag_layout,
+                config.site.ascii_urls,
             )
 
     # Generate sitemap if base_url is configured

src/rockgarden/output/tags.py

@@ -8,7 +8,7 @@
 from rockgarden.urls import get_url, normalize_tag
 
 
-def collect_tags(pages: list[Page]) -> dict[str, list[Page]]:
+def collect_tags(pages: list[Page], ascii_urls: bool = False) -> dict[str, list[Page]]:
     """Return a mapping of normalized tag slug → list of pages with that tag.
 
     Pages are included in the order they appear in the input list. Tags with
@@ -20,7 +20,7 @@ def collect_tags(pages: list[Page]) -> dict[str, list[Page]]:
         if isinstance(raw_tags, str):
             raw_tags = [raw_tags]
         for tag in raw_tags:
-            slug = normalize_tag(tag)
+            slug = normalize_tag(tag, ascii_urls)
             if slug:
                 tags.setdefault(slug, []).append(page)
     return dict(sorted(tags.items()))
@@ -34,6 +34,7 @@ def build_tag_pages(
     clean_urls: bool = True,
     base_path: str = "",
     layout_template: str = "layouts/default.html",
+    ascii_urls: bool = False,
 ) -> None:
     """Generate /tags/<slug>/ and /tags/ pages in the output directory."""
     tag_index_template = env.get_template("tag_index.html")
@@ -47,7 +48,11 @@ def _page_entry(p: Page) -> dict:
             "title": p.title,
             "subtitle": p.frontmatter.get("subtitle", ""),
             "url": get_url(p.slug, clean_urls, base_path),
-            "tags": [normalize_tag(t) for t in raw_tags if normalize_tag(t)],
+            "tags": [
+                normalize_tag(t, ascii_urls)
+                for t in raw_tags
+                if normalize_tag(t, ascii_urls)
+            ],
         }
 
     def _sorted_entries(pages: list[Page]) -> list[dict]:

src/rockgarden/render/engine.py

@@ -85,7 +85,8 @@ def create_engine(
     )
     env.filters["format_datetime"] = _make_format_datetime(config.dates.timezone)
     clean_urls = config.site.clean_urls
-    env.globals["normalize_tag"] = normalize_tag
+    ascii_urls = config.site.ascii_urls
+    env.globals["normalize_tag"] = lambda tag: normalize_tag(tag, ascii_urls=ascii_urls)
     env.globals["tag_url"] = lambda slug: get_tag_url(slug, clean_urls, base_path)
     env.globals["tags_root_url"] = get_tags_root_url(clean_urls, base_path)
     env.globals["icon"] = _icon

src/rockgarden/urls.py

@@ -3,6 +3,8 @@
 import re
 from urllib.parse import quote, urlparse
 
+from slugify import slugify as _ascii_slugify
+
 
 def slugify_heading(text: str) -> str:
     """Convert heading text to a URL-friendly slug.
@@ -16,18 +18,20 @@ def slugify_heading(text: str) -> str:
     return text
 
 
-def normalize_tag(tag: str) -> str:
+def normalize_tag(tag: str, ascii_urls: bool = False) -> str:
     """Normalize a tag to a URL-safe slug.
 
-    Strips leading '#', lowercases, and replaces any character that is not
-    alphanumeric, hyphen, or underscore with a hyphen. Prevents path traversal
-    via tags containing '/' or '..'.
+    Strips leading '#', lowercases, and replaces non-word characters with
+    hyphens. When *ascii_urls* is True, transliterates Unicode to ASCII.
 
     Tags 'Python', '#python', and 'python' all normalize to 'python'.
     Obsidian nested tags like 'character/pc' normalize to 'character-pc'.
     """
-    slug = tag.lstrip("#").lower()
-    slug = re.sub(r"[^a-z0-9_-]", "-", slug)
+    slug = tag.lstrip("#")
+    if ascii_urls:
+        return _ascii_slugify(slug, allow_unicode=False)
+    slug = slug.lower()
+    slug = re.sub(r"[^\w-]", "-", slug)
     slug = re.sub(r"-+", "-", slug)
     return slug.strip("-")
 
@@ -46,25 +50,45 @@ def get_base_path(base_url: str) -> str:
     return path
 
 
-def generate_slug(relative_path: str, style: str = "slug") -> str:
+def generate_slug(
+    relative_path: str, style: str = "slug", ascii_urls: bool = False
+) -> str:
     """Generate a slug from a relative file path.
 
     Args:
         relative_path: Relative path from source root (e.g., "NPCs/Olvir.md").
         style: "slug" (lowercase, dashes), "preserve-case" (original casing,
             dashes), or "preserve" (original casing and spacing).
+        ascii_urls: When True, transliterate Unicode to ASCII.
 
     Returns:
         Slug string. For slug: "npcs/olvir-the-wise".
         For preserve-case: "NPCs/Olvir-the-Wise".
         For preserve: "NPCs/Olvir the Wise".
     """
     slug = re.sub(r"\.md$", "", relative_path, flags=re.IGNORECASE)
+
+    if ascii_urls:
+        segments = slug.split("/")
+        if style == "slug":
+            segments = [_ascii_slugify(s, allow_unicode=False) for s in segments]
+        elif style == "preserve":
+            segments = [
+                _ascii_slugify(s, allow_unicode=False, lowercase=False, separator=" ")
+                for s in segments
+            ]
+        else:
+            segments = [
+                _ascii_slugify(s, allow_unicode=False, lowercase=False)
+                for s in segments
+            ]
+        slug = "/".join(segments)
+
     if style == "preserve":
         return slug
     slug = re.sub(r"[ _]+", "-", slug)
     slug = re.sub(r"-+", "-", slug)
-    if style == "slug":
+    if style == "slug" and not ascii_urls:
         slug = slug.lower()
     return slug
 

tests/test_urls.py

@@ -406,3 +406,86 @@ def test_preserve_case_build_output(self, tmp_path):
 
         html = (output / "index.html").read_text()
         assert "/Getting-Started/" in html
+
+
+class TestGenerateSlugAscii:
+    def test_accented_latin(self):
+        assert generate_slug("Café.md", ascii_urls=True) == "cafe"
+
+    def test_cjk(self):
+        assert generate_slug("東京.md", ascii_urls=True) == "dong-jing"
+
+    def test_cyrillic(self):
+        assert generate_slug("Москва.md", ascii_urls=True) == "moskva"
+
+    def test_nested_path(self):
+        assert generate_slug("Cities/Zürich.md", ascii_urls=True) == "cities/zurich"
+
+    def test_plain_ascii_unaffected(self):
+        assert generate_slug("about.md", ascii_urls=True) == "about"
+
+    def test_preserve_case_with_ascii(self):
+        result = generate_slug("Café.md", style="preserve-case", ascii_urls=True)
+        assert result == "Cafe"
+
+    def test_preserve_with_ascii(self):
+        result = generate_slug("Café Latte.md", style="preserve", ascii_urls=True)
+        assert result == "Cafe Latte"
+
+    def test_ascii_false_preserves_unicode(self):
+        assert generate_slug("Café.md", ascii_urls=False) == "café"
+
+
+class TestNormalizeTagUnicode:
+    """Tests for the bug fix: Unicode preserved when ascii_urls=False."""
+
+    def test_accented_preserved(self):
+        from rockgarden.urls import normalize_tag
+
+        assert normalize_tag("#Café") == "café"
+
+    def test_cjk_preserved(self):
+        from rockgarden.urls import normalize_tag
+
+        result = normalize_tag("#東京")
+        assert result == "東京"
+
+    def test_basic_tag_unchanged(self):
+        from rockgarden.urls import normalize_tag
+
+        assert normalize_tag("#python") == "python"
+
+    def test_nested_tag(self):
+        from rockgarden.urls import normalize_tag
+
+        assert normalize_tag("character/pc") == "character-pc"
+
+
+class TestNormalizeTagAscii:
+    def test_accented_transliterated(self):
+        from rockgarden.urls import normalize_tag
+
+        assert normalize_tag("#Café", ascii_urls=True) == "cafe"
+
+    def test_cjk_transliterated(self):
+        from rockgarden.urls import normalize_tag
+
+        assert normalize_tag("#東京", ascii_urls=True) == "dong-jing"
+
+    def test_cyrillic_transliterated(self):
+        from rockgarden.urls import normalize_tag
+
+        assert normalize_tag("#Москва", ascii_urls=True) == "moskva"
+
+    def test_basic_tag_unchanged(self):
+        from rockgarden.urls import normalize_tag
+
+        assert normalize_tag("#python", ascii_urls=True) == "python"
+
+
+class TestAsciiUrlsConfig:
+    def test_default_false(self):
+        assert SiteConfig().ascii_urls is False
+
+    def test_set_true(self):
+        assert SiteConfig(ascii_urls=True).ascii_urls is True