enhance LLMs build process: add URL fixing script, update build command, and implement verification steps in CI workflow

Author: fassko

.github/workflows/deploy.yml

@@ -25,8 +25,42 @@ jobs:
           cache: "npm"
       - name: Install dependencies
         run: npm ci
-      - name: Build website
-        run: npm run build
+      - name: Build website with LLMs generation
+        run: npm run build:llms-fixed
+      - name: Verify LLMs files
+        run: |
+          echo "Verifying LLMs files generation..."
+          
+          # Check if LLMs files were generated
+          if [ ! -f "build/llms.txt" ] || [ ! -f "build/llms-full.txt" ]; then
+            echo "❌ LLMs files not found!"
+            exit 1
+          fi
+          
+          # Check file sizes
+          llms_txt_size=$(wc -c < build/llms.txt)
+          llms_full_size=$(wc -c < build/llms-full.txt)
+          echo "llms.txt: $llms_txt_size bytes"
+          echo "llms-full.txt: $llms_full_size bytes"
+          
+          # Verify URLs are correct (no prepended numbers)
+          if grep -q "https://dev\.flare\.network/docs/[^)]*/\d+-" build/llms.txt; then
+            echo "❌ Found URLs with prepended numbers!"
+            exit 1
+          else
+            echo "✅ All URLs are correctly formatted"
+          fi
+          
+          # Check content structure
+          sections=$(grep -c "^- \[" build/llms.txt || true)
+          echo "Found $sections sections in llms.txt"
+          
+          if [ "$sections" -lt 10 ]; then
+            echo "❌ Too few sections found!"
+            exit 1
+          fi
+          
+          echo "✅ LLMs files verified successfully"
       - name: Upload Build Artifact
         uses: actions/upload-pages-artifact@v3
         with:

docusaurus.config.ts

@@ -249,14 +249,19 @@ const config: Config = {
       "docusaurus-plugin-llms",
       {
         ignoreFiles: ["**/node_modules/**", "**/.git/**", "**/*.txt"],
-        // Include order for better organization
         includeOrder: [
-          "network",
-          "ftso",
-          "fdc",
-          "fassets",
-          "run-node",
-          "support",
+          "**/network/**/*.mdx",
+          "**/network/**/*.md",
+          "**/ftso/**/*.mdx",
+          "**/ftso/**/*.md",
+          "**/fdc/**/*.mdx",
+          "**/fdc/**/*.md",
+          "**/fassets/**/*.mdx",
+          "**/fassets/**/*.md",
+          "**/run-node/**/*.mdx",
+          "**/run-node/**/*.md",
+          "**/support/**/*.mdx",
+          "**/support/**/*.md",
         ],
         // Content cleaning options for better LLM consumption
         excludeImports: true,

package.json

@@ -19,7 +19,8 @@
     "format:diff": "prettier --list-different .",
     "format:check": "prettier --check \"**/*.{js,jsx,ts,tsx,md,mdx}\"",
     "lint": "npx eslint .",
-    "build:llms": "npm run build"
+    "fix-llms-urls": "node scripts/fix-llms-urls.js",
+    "build:llms-fixed": "npm run build && npm run fix-llms-urls"
   },
   "dependencies": {
     "@docusaurus/core": "^3.8.1",

scripts/fix-llms-urls.js

@@ -0,0 +1,66 @@
+#!/usr/bin/env node
+
+/**
+ * Script to fix URLs in generated LLMs files to match Docusaurus routing
+ * 
+ * This script removes prepended numbers from URLs to match how Docusaurus
+ * generates routes (e.g., "0-overview" becomes "overview")
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const config = {
+  buildDir: 'build',
+  filesToFix: ['llms.txt', 'llms-full.txt'],
+  baseUrl: 'https://dev.flare.network/docs/',
+};
+
+function fixUrlsInFile(filePath) {
+  try {
+    console.log(`Fixing URLs in: ${filePath}`);
+    
+    const content = fs.readFileSync(filePath, 'utf8');
+    
+    // Fix URLs by removing prepended numbers
+    // Pattern: https://dev.flare.network/docs/network/0-overview -> https://dev.flare.network/docs/network/overview
+    const fixedContent = content.replace(
+      /(https:\/\/dev\.flare\.network\/docs\/[^)]+\/)\d+-([^)]+)/g,
+      '$1$2'
+    );
+    
+    fs.writeFileSync(filePath, fixedContent);
+    
+    console.log(`✅ Fixed URLs in: ${filePath}`);
+    
+  } catch (error) {
+    console.error(`❌ Error fixing URLs in ${filePath}:`, error.message);
+  }
+}
+
+function main() {
+  console.log('🔧 Fixing URLs in LLMs files...');
+  
+  if (!fs.existsSync(config.buildDir)) {
+    console.error(`❌ Build directory not found: ${config.buildDir}`);
+    return;
+  }
+  
+  config.filesToFix.forEach(fileName => {
+    const filePath = path.join(config.buildDir, fileName);
+    
+    if (fs.existsSync(filePath)) {
+      fixUrlsInFile(filePath);
+    } else {
+      console.warn(`⚠️  File not found: ${filePath}`);
+    }
+  });
+  
+  console.log('✅ URL fixing completed!');
+}
+
+if (require.main === module) {
+  main();
+}
+
+module.exports = { fixUrlsInFile, main };