feat: support robots

Author: Priestch

packages/markopress/src/config/validation.ts

@@ -140,11 +140,23 @@ const SitemapConfigSchema = z.object({
   transformItems: z.any().optional(),
 }).passthrough();
 
+/**
+ * Robots configuration schema
+ */
+const RobotsConfigSchema = z.object({
+  userAgent: z.union([z.string(), z.array(z.string())]).optional(),
+  allow: z.array(z.string()).optional(),
+  disallow: z.array(z.string()).optional(),
+  crawlDelay: z.number().nonnegative().optional(),
+  sitemap: z.string().optional(),
+}).passthrough();
+
 /**
  * SEO plugin configuration schema
  */
 const SeoConfigSchema = z.object({
   sitemap: SitemapConfigSchema.optional(),
+  robots: RobotsConfigSchema.optional(),
 }).passthrough().optional();
 
 /**

packages/markopress/src/plugins/seo/index.test.ts

@@ -1,6 +1,7 @@
 import { describe, it, expect, vi } from 'vitest';
 import { seoPlugin } from './index.js';
-import type { BuildContext } from '../../plugin/types.js';
+import * as robotsModule from './robots.js';
+import * as sitemapModule from './sitemap.js';
 
 describe('seoPlugin', () => {
   it('should have correct plugin name', () => {
@@ -14,22 +15,30 @@ describe('seoPlugin', () => {
     expect(typeof plugin.postBuild).toBe('function');
   });
 
-  it('should skip sitemap if not configured', async () => {
+  it('should skip when not configured', async () => {
     const plugin = seoPlugin();
     const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+    const sitemapSpy = vi.spyOn(sitemapModule, 'generateSitemap').mockResolvedValue(undefined);
+    const robotsSpy = vi.spyOn(robotsModule, 'generateRobots').mockResolvedValue(undefined);
 
     const mockContext = {
       config: {},
     } as any;
 
     await plugin.postBuild!(mockContext as any);
 
-    expect(consoleSpy).toHaveBeenCalledWith('[seo] Sitemap not configured, skipping');
+    expect(consoleSpy).toHaveBeenCalledWith('[seo] No SEO generation configured, skipping');
+    expect(sitemapSpy).not.toHaveBeenCalled();
+    expect(robotsSpy).not.toHaveBeenCalled();
 
     consoleSpy.mockRestore();
+    sitemapSpy.mockRestore();
+    robotsSpy.mockRestore();
   });
 
   it('should call generateSitemap when configured', async () => {
+    const sitemapSpy = vi.spyOn(sitemapModule, 'generateSitemap').mockResolvedValue(undefined);
+
     const plugin = seoPlugin({
       sitemap: { hostname: 'https://example.com' },
     });
@@ -47,5 +56,38 @@ describe('seoPlugin', () => {
 
     // Should not throw
     await expect(plugin.postBuild!(mockContext as any)).resolves.toBeUndefined();
+    expect(sitemapSpy).toHaveBeenCalledWith(mockContext, { hostname: 'https://example.com' });
+
+    sitemapSpy.mockRestore();
+  });
+
+  it('should call generateRobots when configured', async () => {
+    const robotsSpy = vi.spyOn(robotsModule, 'generateRobots').mockResolvedValue(undefined);
+
+    const plugin = seoPlugin({
+      robots: { disallow: ['/admin'] },
+    });
+
+    const mockContext = {
+      config: {
+        seo: {
+          robots: {
+            disallow: ['/admin'],
+          },
+        },
+      },
+      outDir: '/tmp/dist',
+      routes: {},
+      allContent: {},
+    } as any;
+
+    await expect(plugin.postBuild!(mockContext as any)).resolves.toBeUndefined();
+    expect(robotsSpy).toHaveBeenCalledWith(
+      mockContext,
+      { disallow: ['/admin'] },
+      undefined
+    );
+
+    robotsSpy.mockRestore();
   });
 });

packages/markopress/src/plugins/seo/index.ts

@@ -1,9 +1,10 @@
 import type { MarkoPressPlugin } from '../../plugin/types.js';
 import type { SeoPluginFactoryOptions } from './types.js';
 import { generateSitemap } from './sitemap.js';
+import { generateRobots } from './robots.js';
 
 /**
- * SEO Plugin - Generates sitemap.xml for SEO
+ * SEO Plugin - Generates sitemap.xml and robots.txt
  *
  * @example
  * // In markopress config
@@ -23,18 +24,27 @@ export function seoPlugin(options?: SeoPluginFactoryOptions): MarkoPressPlugin {
 
     async postBuild(ctx) {
       const { config } = ctx;
+      const seoConfig = (options || (config as any).seo || {}) as any;
+      const userSeoConfig = (config as any).seo;
 
-      // Get SEO config from user config
-      const seoConfig = (config as any).seo;
+      const hasSitemap = Boolean(seoConfig?.sitemap || userSeoConfig?.sitemap);
+      const hasRobots = Boolean(seoConfig?.robots || userSeoConfig?.robots);
 
-      // Skip if sitemap not enabled
-      if (!seoConfig?.sitemap) {
-        console.log('[seo] Sitemap not configured, skipping');
+      // Skip if no seo features are enabled
+      if (!hasSitemap && !hasRobots) {
+        console.log('[seo] No SEO generation configured, skipping');
         return;
       }
 
-      // Generate sitemap
-      await generateSitemap(ctx, seoConfig.sitemap);
+      if (seoConfig?.sitemap || userSeoConfig?.sitemap) {
+        // Generate sitemap
+        await generateSitemap(ctx, seoConfig.sitemap || userSeoConfig.sitemap);
+      }
+
+      if (seoConfig?.robots || userSeoConfig?.robots) {
+        // Generate robots.txt
+        await generateRobots(ctx, seoConfig.robots || userSeoConfig.robots, seoConfig.sitemap || userSeoConfig?.sitemap);
+      }
     },
   };
 }

packages/markopress/src/plugins/seo/robots.test.ts

@@ -0,0 +1,94 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { generateRobots } from './robots.js';
+
+// Mock fs
+vi.mock('fs', async () => {
+  const actual = await vi.importActual('fs');
+  return {
+    ...actual,
+    promises: {
+      writeFile: vi.fn(),
+      mkdir: vi.fn().mockResolvedValue(undefined),
+    },
+  };
+});
+
+// Import mocked modules
+import { promises as fs } from 'fs';
+
+describe('generateRobots', () => {
+  const mockBaseContext = {
+    config: { site: { url: 'https://example.com', base: '/docs' } },
+    outDir: '/tmp/dist',
+    routes: {},
+    assets: [],
+    allContent: {},
+  };
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('should generate robots.txt with basic rules', async () => {
+    await generateRobots(mockBaseContext, {
+      userAgent: ['Googlebot', 'Bingbot'],
+      allow: ['/'],
+      disallow: ['/admin'],
+      crawlDelay: 5,
+    });
+
+    expect(fs.writeFile).toHaveBeenCalledWith(
+      '/tmp/dist/public/robots.txt',
+      'User-agent: Googlebot\nAllow: /\nDisallow: /admin\nCrawl-delay: 5\n\nUser-agent: Bingbot\nAllow: /\nDisallow: /admin\nCrawl-delay: 5\n',
+      'utf8'
+    );
+  });
+
+  it('should include sitemap reference when sitemap options are provided', async () => {
+    await generateRobots(mockBaseContext, {
+      disallow: ['/private'],
+    }, {
+      hostname: 'https://example.com',
+    });
+
+    expect(fs.writeFile).toHaveBeenCalledWith(
+      '/tmp/dist/public/robots.txt',
+      expect.stringContaining('Sitemap: https://example.com/docs/sitemap.xml'),
+      'utf8'
+    );
+  });
+
+  it('should use explicit robots sitemap path', async () => {
+    await generateRobots(mockBaseContext, {
+      disallow: ['/private'],
+      sitemap: 'custom-sitemap.xml',
+    }, {
+      hostname: 'https://example.com',
+    });
+
+    expect(fs.writeFile).toHaveBeenCalledWith(
+      '/tmp/dist/public/robots.txt',
+      expect.stringContaining('Sitemap: https://example.com/docs/custom-sitemap.xml'),
+      'utf8'
+    );
+  });
+
+  it('should generate fallback file without site url when no sitemap options are set', async () => {
+    await generateRobots(
+      {
+        ...mockBaseContext,
+        config: { site: {} },
+      },
+      {
+        allow: ['/'],
+      }
+    );
+
+    const writeCall = vi.mocked(fs.writeFile).mock.calls[0];
+    const content = writeCall[1] as string;
+
+    expect(writeCall[0]).toBe('/tmp/dist/public/robots.txt');
+    expect(content).toContain('User-agent: *');
+    expect(content).not.toContain('Sitemap:');
+  });
+});