Add ability to publish our docs in markdown format

.changes/next-release/feature-fcb484ef5c3589dd9a8ec2ae20d1975e25d51fb6.json

@@ -0,0 +1,7 @@
+{
+  "type": "feature",
+  "description": "Dev/yasmewad/markdown docs publishing",
+  "pull_requests": [
+    "[#3097](https://github.com/smithy-lang/smithy/pull/3097)"
+  ]
+}

docs/Makefile

@@ -18,13 +18,21 @@ html1:
 html2:
 	@source build/venv/bin/activate && $(SPHINXBUILD) -M html "source-2.0" "build/2.0" $(SPHINXOPTS) -W --keep-going -n $(O)
 
+md1:
+	@source build/venv/bin/activate && $(SPHINXBUILD) -M markdown "source-1.0" "build/1.0-md" $(SPHINXOPTS) -W --keep-going -n $(O)
+
+md2:
+	@source build/venv/bin/activate && $(SPHINXBUILD) -M markdown "source-2.0" "build/2.0-md" $(SPHINXOPTS) -W --keep-going -n $(O)
+
 build-landing-page:
 	cd landing-page && npm run build
 
 llms-txt: html2
 	python3 generate_llms_txt.py
 
-html: html2 html1 build-landing-page merge-versions llms-txt
+markdown: md2 md1 merge-markdown
+
+html: html2 html1 build-landing-page merge-versions markdown llms-txt
 
 serve: 
 	npx http-server build/html
@@ -36,6 +44,11 @@ merge-versions:
 	cp -R root/* "build/html"
 	cp -R landing-page/dist/* "build/html/"
 
+merge-markdown:
+	mkdir -p build/html/1.0/markdown build/html/2.0/markdown
+	cp -R "build/1.0-md/markdown/." "build/html/1.0/markdown/"
+	cp -R "build/2.0-md/markdown/." "build/html/2.0/markdown/"
+
 openhtml:
 	open "build/html/index.html"
 

docs/conf.py

@@ -13,6 +13,7 @@
     "myst_parser",
     "sphinx_copybutton",
     "sphinx_substitution_extensions",
+    "sphinx_markdown_builder",
     "smithy",
 ]
 templates_path = ["../_templates", "../root"]
@@ -127,6 +128,7 @@ def __load_java_version():
 def setup(sphinx):
     sphinx.add_lexer("smithy", SmithyLexer)
     sphinx.connect("source-read", source_read_handler)
+    sphinx.connect("builder-inited", _builder_inited_handler)
     for placeholder, replacement in replacements:
         print("Finding and replacing '" + placeholder + "' with '" + replacement + "'")
 
@@ -135,3 +137,97 @@ def setup(sphinx):
 def source_read_handler(app, docname, source):
     for placeholder, replacement in replacements:
         source[0] = source[0].replace(placeholder, replacement)
+
+
+# -- Markdown builder: register missing node handlers -----------------------
+
+def _patch_markdown_translator(app):
+    """Add handlers for node types not supported by sphinx-markdown-builder."""
+    try:
+        from sphinx_markdown_builder.translator import MarkdownTranslator
+    except ImportError:
+        return
+    from docutils import nodes as _nodes
+
+    def _make_admonition_visit(label):
+        def visit(self, node):
+            self._push_box(label)
+        return visit
+
+    def _admonition_depart(self, node):
+        self._pop_context(node)
+
+    for node_type, label in [
+        ("tip", "TIP"),
+        ("danger", "DANGER"),
+        ("caution", "CAUTION"),
+        ("admonition", "NOTE"),
+    ]:
+        visit_name = f"visit_{node_type}"
+        depart_name = f"depart_{node_type}"
+        if not hasattr(MarkdownTranslator, visit_name):
+            setattr(MarkdownTranslator, visit_name, _make_admonition_visit(label))
+            setattr(MarkdownTranslator, depart_name, _admonition_depart)
+
+    # productionlist: render grammar rules as a code block
+    def visit_productionlist(self, node):
+        self._push_status(escape_text=False)
+        self.add("```", prefix_eol=2, suffix_eol=1)
+        for production in node.children:
+            token_name = production.get("tokenname", "")
+            rule_text = production.astext()
+            if token_name:
+                self.add(f"{token_name} ::= {rule_text}", prefix_eol=1)
+            else:
+                self.add(f"         {rule_text}", prefix_eol=1)
+        self.add("```", prefix_eol=1, suffix_eol=2)
+        self._pop_status()
+        raise _nodes.SkipNode
+
+    if not hasattr(MarkdownTranslator, "visit_productionlist"):
+        MarkdownTranslator.visit_productionlist = visit_productionlist
+        MarkdownTranslator.depart_productionlist = lambda self, node: None
+
+    # caption: render as bold text
+    def visit_caption(self, node):
+        self.add("**", prefix_eol=2)
+
+    def depart_caption(self, node):
+        self.add("**", suffix_eol=2)
+
+    if not hasattr(MarkdownTranslator, "visit_caption"):
+        MarkdownTranslator.visit_caption = visit_caption
+        MarkdownTranslator.depart_caption = depart_caption
+
+    # hlist / hlistcol: pass through to children (they contain bullet_lists)
+    def _pass(self, node):
+        pass
+
+    if not hasattr(MarkdownTranslator, "visit_hlist"):
+        MarkdownTranslator.visit_hlist = _pass
+        MarkdownTranslator.depart_hlist = _pass
+
+    if not hasattr(MarkdownTranslator, "visit_hlistcol"):
+        MarkdownTranslator.visit_hlistcol = _pass
+        MarkdownTranslator.depart_hlistcol = _pass
+
+    # label (non-footnote, e.g. tab labels from sphinx_inline_tabs): skip
+    from sphinx_markdown_builder.contexts import FootNoteContext as _FootNoteContext
+
+    def visit_label(self, node):
+        if isinstance(self.ctx, _FootNoteContext):
+            self.footnote_ctx.visit_label()
+        else:
+            raise _nodes.SkipNode
+
+    def depart_label(self, node):
+        if isinstance(self.ctx, _FootNoteContext):
+            self.footnote_ctx.depart_label()
+
+    MarkdownTranslator.visit_label = visit_label
+    MarkdownTranslator.depart_label = depart_label
+
+
+def _builder_inited_handler(app):
+    if app.builder.name == "markdown":
+        _patch_markdown_translator(app)

docs/generate_llms_txt.py

@@ -11,6 +11,7 @@
 import sys
 
 BASE_URL = "https://smithy.io/2.0"
+MARKDOWN_BASE_URL = "https://smithy.io/2.0/markdown"
 SOURCE_DIR = "source-2.0"
 
 # Sections in display order. Keys are directory prefixes relative to SOURCE_DIR;
@@ -49,10 +50,17 @@ def extract_title(filepath):
 def rst_path_to_url(rst_path):
     """Convert a source-relative RST path to a smithy.io URL."""
     rel = rst_path.replace(os.sep, "/")
-    html = rel.removesuffix(".rst") + ".html"
+    html = rel[:-4] + ".html"  # strip .rst, add .html
     return f"{BASE_URL}/{html}"
 
 
+def rst_path_to_md_url(rst_path):
+    """Convert a source-relative RST path to a smithy.io markdown URL."""
+    rel = rst_path.replace(os.sep, "/")
+    md = rel[:-4] + ".md"  # strip .rst, add .md
+    return f"{MARKDOWN_BASE_URL}/{md}"
+
+
 def collect_pages(source_dir):
     """Walk source_dir and return {relative_path: title} for all RST files."""
     pages = {}
@@ -88,6 +96,11 @@ def generate(source_dir, output_path):
         " language. Smithy models define a service as a collection of resources,"
         " operations, and shapes.",
         "",
+        "## Content Formats",
+        "",
+        "Each page is available in HTML and Markdown. For AI/LLM consumption,",
+        "use the Markdown URLs listed below (more token-efficient, no HTML parsing).",
+        "",
     ]
 
     for prefix, heading in SECTIONS:
@@ -103,8 +116,8 @@ def generate(source_dir, output_path):
 
         for rel_path in sorted(section_pages):
             title = section_pages[rel_path]
-            url = rst_path_to_url(rel_path)
-            lines.append(f"- [{title}]({url})")
+            md_url = rst_path_to_md_url(rel_path)
+            lines.append(f"- [{title}]({md_url})")
 
         lines.append("")
 

docs/landing-page/index.html

@@ -31,6 +31,12 @@
     />
 
     <title>Smithy</title>
+    <link
+      rel="alternate"
+      type="text/plain"
+      href="/2.0/llms.txt"
+      title="LLM-readable documentation"
+    />
   </head>
   <body class="dark">
     <div id="root"></div>

docs/pyproject.toml

@@ -24,7 +24,10 @@ dependencies = [
     "standard-imghdr==3.13.0",
 
     # Allows writing docs in markdown
-    "myst-parser==5.0.0"
+    "myst-parser==5.0.0",
+
+    # Allows building docs as markdown output
+    "sphinx-markdown-builder==0.6.10"
 ]
 
 [project.entry-points."pygments.lexers"]

docs/smithy/redirects.py

@@ -40,8 +40,9 @@ def generate_redirects(app):
         return
 
     if not (type(app.builder) == StandaloneHTMLBuilder or type(app.builder) == DirectoryHTMLBuilder):
-        app.warn("The 'sphinxcontib-redirects' plugin is only supported "
-                 "by the 'html' and 'dirhtml' builder, but you are using '%s'. Skipping..." % type(app.builder))
+        LOGGER.info("The 'sphinxcontib-redirects' plugin is only supported "
+                    "by the 'html' and 'dirhtml' builder, but you are using '%s'. Skipping..." % type(app.builder))
+        return
 
     dirhtml = False
     if type(app.builder) == DirectoryHTMLBuilder: