DOC-13760 save ONLY markdown summary

Reduce memory usage by *only* saving the Markdown version in this run.
We can do this as a separate build, and scp the output to the same bucket.

e.g. this content would be overlaid on the *standard* build (which would
only need to add the <link href="..."> URL feature.

Rewrite relative hrefs to .md target.

Author: osfameron

lib/markdown-for-llm.js

@@ -5,9 +5,20 @@ const {JSDOM} = require('jsdom')
 
 function overrideElementProcessing (element) {
 
-  if (element.tagName?.toLowerCase() === 'a' 
-   && element.className === 'anchor' ) {
+  if (element.tagName?.toLowerCase() === 'a') {
+    if (element.className === 'anchor' ) {
       return [{type: 'custom', blank: true}]
+    }
+    let href = element.getAttribute('href')
+    const hasProtocol = /^[a-z]+:\/\//i
+    console.log(href)
+    if (href && !href.match(hasProtocol)) {
+      // convert internal links to markdown format
+      href = href.replace(/\.html/, '.md')
+      const content = toMarkdown(element.innerHTML || href)
+      console.log("LINK", href, content)
+      return [{type: 'link', href, content}]
+    }
   }
 
   if (element.classList?.contains("admonitionblock")) {
@@ -51,28 +62,57 @@ function markdownify(page, siteCatalog) {
     const html = page.contents.toString()
     const markdown = `# ${page.asciidoc.doctitle}\n\n` + toMarkdown(html)
 
-    const path = page.out.path.replace(/\.html$/, '.md')
-    
-    // tell docs-ui to output <link rel="alternate" ...> for the markdown page.
-    page.asciidoc.attributes["page-markdown-alt"] = `${page.out.rootPath}/${path}`
+    page.out.path = page.out.path.replace(/\.html$/, '.md')
+    if (page.out.url) {
+      page.out.url = page.out.url.replace(/\.html$/, '.md')
+    }
 
-    siteCatalog.addFile({
-      contents: Buffer.from(markdown),
-      out: { path }
-    })
+    page.contents = Buffer.from(markdown)
+
+    // tell docs-ui to output <link rel="alternate" ...> for the markdown page.
+    // (no longer relevant here - we need to do this in the MAIN build)
+    // page.asciidoc.attributes["page-markdown-alt"] = `${page.out.rootPath}/${path}`
 }
 
 module.exports.register = function ({ playbook, config }) {
-    const logger = this.getLogger('markdown-for-llm')
+  this.once('contextStarted', () => {
+    const { createPageComposer: _createPageComposerDelegate } = this.getFunctions()
 
-    this.on('navigationBuilt', ({ playbook, siteAsciiDocConfig, siteCatalog, uiCatalog, contentCatalog }) => {
-
-        logger.info('Compiling Markdown summaries')
-
-        for (const page of contentCatalog.getPages()) {
-          if (page.pub) {
-            markdownify(page, siteCatalog)
-          }
+    this.replaceFunctions({
+      // see https://gitlab.com/antora/antora/-/blob/v3.1.x/packages/page-composer/lib/create-page-composer.js
+      createPageComposer (playbook, contentCatalog, uiCatalog) {
+        function composePage (file, _contentCatalog, _navigationCatalog) {
+          // instead of wrapping the file in a layout, just
+          // return the file as-is
+          return file
         }
+        const create404Page = (siteAsciiDocConfig) =>
+          composePage({
+            asciidoc: siteAsciiDocConfig,
+            mediaType: 'text/html',
+            out: { path: '404.html' },
+            pub: {},
+            src: { stem: '404' },
+            title: siteAsciiDocConfig?.attributes['404-page-title'] || 'PageNot Found',
+        })
+
+        const ret = Object.assign(composePage, {composePage, create404Page} )
+        return ret
+      }
     })
+  })
+
+  this.once('documentsConverted', async ({ playbook, contentCatalog, siteCatalog }) => {
+    const logger = this.getLogger('llm-summaries')
+
+    const pages = contentCatalog.getPages(
+      (page) =>
+        page.mediaType === 'text/html'
+        && page.pub
+        && page.out)
+
+    for (const page of pages) {
+      markdownify(page, siteCatalog)
+    }
+  })
 }