middleware parses substrings too

Author: ritorhymes

__tests__/middleware.test.ts

@@ -4,16 +4,21 @@ import type { NextRequest } from 'next/server';
 import { middleware } from '../middleware';
 
 const createRequest = (url: string, userAgent: string = '') => {
-  const parsedUrl = new URL(url);
+  const u = new URL(url);
+  const nextUrl = {
+    // minimal NextURL-ish shape the middleware actually uses
+    pathname: u.pathname,
+    search: u.search,
+    searchParams: new URLSearchParams(u.search),
+    hash: u.hash,
+    href: u.href,
+    origin: u.origin,
+    clone: () => new URL(u.href),
+    toString: () => u.toString(),
+  };
+
   return {
-    nextUrl: {
-      ...parsedUrl,
-      clone: () => new URL(parsedUrl.href),
-      pathname: parsedUrl.pathname,
-      search: parsedUrl.search,
-      searchParams: parsedUrl.searchParams,
-      hash: parsedUrl.hash,
-    },
+    nextUrl,
     headers: {
       get: (name: string) => (name.toLowerCase() === 'user-agent' ? userAgent : null),
     },
@@ -25,9 +30,6 @@ const botUAs = [
   'Mozilla/5.0 (compatible; bingbot/2.0)',
   'GPTBot/1.0',
   'Claude-Web/1.0',
-  'ChatGPT-User/1.0',
-  'facebookexternalhit/1.1',
-  'Twitterbot/1.0',
   'googlebot',
   'GOOGLEBOT',
   'GoogleBot',
@@ -39,8 +41,22 @@ const humanUAs = [
   'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0',
 ];
 
-describe('Middleware - Bot Redirects', () => {
-  it('redirects bots from / to /bot', () => {
+const scriptedClients = [
+  'curl/8.5.0',
+  'Wget/1.21.3',
+  'python-requests/2.32.3',
+  'Apache-HttpClient/4.5.14',
+  'okhttp/4.12.0',
+  'GuzzleHttp/7',
+  'libwww-perl/6.68',
+  'Go-http-client/1.1',
+  'Java/11.0.24',
+  'node-fetch/1.0',
+  'axios/1.7.0',
+];
+
+describe('Middleware - Bot/Agent Redirects with AFL heuristics', () => {
+  it('redirects known bot UAs from / to /bot', () => {
     for (const ua of botUAs) {
       const res = middleware(createRequest('http://localhost:3000/', ua));
       expect(res.status).toBe(302);
@@ -51,7 +67,7 @@ describe('Middleware - Bot Redirects', () => {
     }
   });
 
-  it('redirects bots from any page to /bot/* (no bot/bot loops)', () => {
+  it('redirects known bot UAs from any page to /bot/* (no /bot loops)', () => {
     for (const ua of botUAs) {
       const res = middleware(createRequest('http://localhost:3000/gallery', ua));
       expect(res.status).toBe(302);
@@ -62,29 +78,25 @@ describe('Middleware - Bot Redirects', () => {
   });
 
   it('preserves query strings on redirect', () => {
-    const res = middleware(
-      createRequest('http://localhost:3000/about?ref=twitter&x=1', 'Googlebot')
-    );
+    const res = middleware(createRequest('http://localhost:3000/about?ref=twitter&x=1', 'Googlebot'));
     expect(res.status).toBe(302);
     const loc = new URL(res.headers.get('Location') || '');
     expect(loc.pathname).toBe('/bot/about');
     expect(loc.searchParams.get('ref')).toBe('twitter');
     expect(loc.searchParams.get('x')).toBe('1');
   });
 
-  it('does not redirect already /bot/* paths (for bots or humans)', () => {
+  it('does not redirect already /bot/* paths (bots or humans)', () => {
     const botRes = middleware(createRequest('http://localhost:3000/bot/gallery', 'Googlebot'));
     expect(botRes.headers.get('Location')).toBeNull();
     expect(botRes.status).toBe(200);
 
-    const humanRes = middleware(
-      createRequest('http://localhost:3000/bot/gallery', humanUAs[0])
-    );
+    const humanRes = middleware(createRequest('http://localhost:3000/bot/gallery', humanUAs[0]));
     expect(humanRes.headers.get('Location')).toBeNull();
     expect(humanRes.status).toBe(200);
   });
 
-  it('does not redirect API, _next, static, or asset files even for bots', () => {
+  it('bypasses API, _next, static, and asset files even for bots', () => {
     const skipPaths = [
       'http://localhost:3000/api/test',
       'http://localhost:3000/_next/static/chunk.js',
@@ -105,32 +117,55 @@ describe('Middleware - Bot Redirects', () => {
     }
   });
 
-  it('does not redirect human user agents', () => {
+  it('does not redirect normal human browsers', () => {
     for (const ua of humanUAs) {
       const res = middleware(createRequest('http://localhost:3000/gallery', ua));
       expect(res.headers.get('Location')).toBeNull();
       expect(res.status).toBe(200);
     }
   });
 
-  it('handles deep nested paths correctly', () => {
-    const res = middleware(
-      createRequest('http://localhost:3000/deep/nested/path', 'Googlebot')
-    );
+  it('redirects generic crawler terms (substring backstop)', () => {
+    const ua = 'MyAwesomeCrawler/1.0 (+https://example.com)';
+    const res = middleware(createRequest('http://localhost:3000/deep/nested/path', ua));
     expect(res.status).toBe(302);
     const loc = new URL(res.headers.get('Location') || '');
     expect(loc.pathname).toBe('/bot/deep/nested/path');
   });
 
-  it('treats missing/empty user-agent as human (no redirect)', () => {
-  const res1 = middleware(createRequest('http://localhost:3000/gallery'));
-  expect(res1.headers.get('Location')).toBeNull();
-  expect(res1.status).toBe(200);
+  it('redirects scripted HTTP clients to /bot/*', () => {
+    for (const ua of scriptedClients) {
+      const res = middleware(createRequest('http://localhost:3000/gallery', ua));
+      expect(res.status).toBe(302);
+      const loc = new URL(res.headers.get('Location') || '');
+      expect(loc.pathname).toBe('/bot/gallery');
+    }
+  });
+
+  it('treats empty or missing user-agent as scripted client (redirects to /bot/*)', () => {
+    const res1 = middleware(createRequest('http://localhost:3000/gallery'));
+    expect(res1.status).toBe(302);
+    const loc1 = new URL(res1.headers.get('Location') || '');
+    expect(loc1.pathname).toBe('/bot/gallery');
 
-  const res2 = middleware(createRequest('http://localhost:3000/gallery', '   '));
-  expect(res2.headers.get('Location')).toBeNull();
-  expect(res2.status).toBe(200);
-});
+    const res2 = middleware(createRequest('http://localhost:3000/gallery', '   '));
+    expect(res2.status).toBe(302);
+    const loc2 = new URL(res2.headers.get('Location') || '');
+    expect(loc2.pathname).toBe('/bot/gallery');
+  });
 
-  
+  it('override: ?afl=human prevents redirect even for bots', () => {
+    const res = middleware(createRequest('http://localhost:3000/gallery?afl=human', 'Googlebot'));
+    expect(res.headers.get('Location')).toBeNull();
+    expect(res.status).toBe(200);
+  });
+
+  it('override: ?afl=bot forces redirect even for humans and strips the override key', () => {
+    const res = middleware(createRequest('http://localhost:3000/gallery?afl=bot&ref=x', humanUAs[0]));
+    expect(res.status).toBe(302);
+    const loc = new URL(res.headers.get('Location') || '');
+    expect(loc.pathname).toBe('/bot/gallery');
+    expect(loc.searchParams.get('ref')).toBe('x');
+    expect(loc.searchParams.get('afl')).toBeNull();
+  });
 });

middleware.ts

@@ -3,29 +3,85 @@ import { NextResponse } from 'next/server';
 import type { NextRequest } from 'next/server';
 import { isBotUserAgent } from '@lib/bot/userAgents';
 
-export function middleware(request: NextRequest) {
-  const ua = request.headers.get('user-agent') || '';
-  const { pathname } = request.nextUrl;
+// Generic crawler words like "bot", "crawler", etc.
+function isGenericCrawler(uaRaw: string): boolean {
+  const raw = uaRaw ?? '';
+  const norm = raw.trim();
+  if (!norm) return true; // empty/whitespace UA => treat as scripted/automated
+  return /\b(bot|crawler|spider|crawl)\b/i.test(norm);
+}
+
+// Common scripted HTTP clients frequently used by agents/tools
+function isScriptedHttpClient(uaRaw: string): boolean {
+  const raw = uaRaw ?? '';
+  const norm = raw.trim();
+  if (!norm) return true; // empty/whitespace UA
+  const ua = norm.toLowerCase();
+
+  const substrings = [
+    'curl',
+    'wget',
+    'python-requests',
+    'httpclient',        // Apache-HttpClient
+    'okhttp',
+    'guzzlehttp',
+    'libwww-perl',
+    'dart/',
+    'go-http-client',
+    'java/',
+    'node-fetch',
+    'axios/',
+  ];
 
-  // Skip bot pages and non-HTML assets/internals
-  if (
+  return substrings.some(s => ua.includes(s));
+}
+
+function shouldBypass(pathname: string): boolean {
+  return (
     pathname.startsWith('/bot') ||
     pathname.startsWith('/api/') ||
     pathname.startsWith('/_next/') ||
     pathname.startsWith('/static/') ||
     /\.(?:ico|png|jpg|jpeg|webp|svg|gif|css|js|json|xml|txt|map)$/i.test(pathname)
-  ) {
+  );
+}
+
+export function middleware(request: NextRequest) {
+  const uaHeader = request.headers.get('user-agent') || '';
+  const url = request.nextUrl;
+  const { pathname, searchParams } = url;
+
+  if (shouldBypass(pathname)) {
     return NextResponse.next();
   }
 
-  if (!isBotUserAgent(ua)) {
+  // Manual overrides
+  const afl = searchParams.get('afl');
+  if (afl === 'human') {
+    return NextResponse.next();
+  }
+  if (afl === 'bot') {
+    const forced = url.clone();
+    forced.pathname = pathname === '/' ? '/bot' : `/bot${pathname}`;
+    forced.searchParams.delete('afl');
+    const res = NextResponse.redirect(forced, 302);
+    res.headers.set('Vary', 'User-Agent');
+    res.headers.set('Cache-Control', 'no-store');
+    return res;
+  }
+
+  // Detection: strict list + generic crawler + scripted clients
+  const looksAutomated =
+    isBotUserAgent(uaHeader) || isGenericCrawler(uaHeader) || isScriptedHttpClient(uaHeader);
+
+  if (!looksAutomated) {
     return NextResponse.next();
   }
 
-  const url = request.nextUrl.clone();
-  url.pathname = pathname === '/' ? '/bot' : `/bot${pathname}`;
+  const dest = url.clone();
+  dest.pathname = pathname === '/' ? '/bot' : `/bot${pathname}`;
 
-  const res = NextResponse.redirect(url, 302);
+  const res = NextResponse.redirect(dest, 302);
   res.headers.set('Vary', 'User-Agent');
   res.headers.set('Cache-Control', 'no-store');
   return res;