Skip to content

Commit eb30609

Browse files
faisalmanfaisalman
committed
Feat(extensions): Add new bots: BorderxBot, Cloudflare Crawler, FishBot, LinerBot, QualifiedBot, WARDBot
1 parent 1e44951 commit eb30609

4 files changed

Lines changed: 73 additions & 5 deletions

File tree

‎src/enums/ua-parser-enums.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -498,12 +498,14 @@ const Extension = Object.freeze({
498498
BAIDU_SPIDER_RENDER: 'Baiduspider-render',
499499
BAIDU_SPIDER_VIDEO: 'Baiduspider-video',
500500
BLEX_BOT: 'BLEXBot',
501+
BORDERX_BOT: 'BorderxBot',
501502
BOTIFY: 'botify',
502503
BRAVE_BOT: 'Bravebot',
503504
BRIGHTEDGE_CRAWLER: 'BrightEdge Crawler',
504505
BYTEDANCE_BYTESPIDER: 'Bytespider',
505506
BYTEDANCE_TIKTOKSPIDER: 'TikTokSpider',
506507
CLOUDFLARE_AUTORAG: 'Cloudflare-AutoRAG',
508+
CLOUDFLARE_CRAWLER: 'CloudflareBrowserRenderingCrawler',
507509
COMMON_CRAWL_CCBOT: 'CCBot',
508510
COCCOC_BOT_WEB: 'coccocbot-web',
509511
COCCOC_BOT_IMAGE: 'coccocbot-image',
@@ -524,6 +526,7 @@ const Extension = Object.freeze({
524526
ELASTIC_SWIFTYPE_BOT: 'Swiftbot',
525527
EXALEAD_EXABOT: 'Exabot',
526528
FIRECRAWL_AGENT: 'FirecrawlAgent',
529+
FISH_BOT: 'FishBot',
527530
FREESPOKE: 'Freespoke',
528531
GOOGLE_ADSBOT: 'AdsBot-Google',
529532
GOOGLE_ADSBOT_MOBILE: 'Adsbot-Google-Mobile',
@@ -554,6 +557,7 @@ const Extension = Object.freeze({
554557
KAGI_BOT: 'Kagibot',
555558
KANGAROO_BOT: 'Kangaroo Bot',
556559
LINE_SPIDER: 'Linespider',
560+
LINER_BOT: 'LinerBot',
557561
LINKEDIN_BOT: 'LinkedInBot',
558562
MAGPIE_CRAWLER: 'magpie-crawler',
559563
MARGINALIA: 'marginalia',
@@ -577,6 +581,7 @@ const Extension = Object.freeze({
577581
OPENAI_SEARCH_BOT: 'OAI-SearchBot',
578582
PERPLEXITY_BOT: 'PerplexityBot',
579583
QIHOO_360_SPIDER: '360Spider',
584+
QUALIFIED_BOT: 'QualifiedBot',
580585
QWANT_BOT: 'Qwantbot',
581586
QWANT_BOT_NEWS: 'Qwantbot-news',
582587
REPLICATE_BOT: 'Replicate-Bot',
@@ -762,6 +767,7 @@ const Extension = Object.freeze({
762767
VERCEL_FLAGS: 'vercelflags',
763768
VERCEL_TRACING: 'verceltracing',
764769
VIRUSTOTAL: 'virustotal',
770+
WARD_BOT: 'WARDBot',
765771
X_TWITTERBOT: 'Twitterbot',
766772
YANDEX_CALENDAR: 'YandexCalendar',
767773
YANDEX_DIRECT: 'YandexDirect',

‎src/extensions/ua-parser-extensions.js‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ const Crawlers = Object.freeze({
5757
// Amazonbot - https://developer.amazon.com/amazonbot
5858
// Awario - https://awario.com/bots.html
5959
// Bingbot / AdIdxBot - https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0
60+
// BorderxBot - https://www.nubestore.ai/borderxbot.html
6061
// Bravebot - https://search.brave.com/help/brave-search-crawler
6162
// CCBot - https://commoncrawl.org/faq
6263
// contxbot - https://affiliate-program.amazon.com/help/node/topic/GT98G5PPRERNVZ2C
@@ -76,12 +77,13 @@ const Crawlers = Object.freeze({
7677
// Onespot - https://www.onespot.com/identifying-traffic.html
7778
// OpenAI's SearchGPT - https://platform.openai.com/docs/bots
7879
// PerplexityBot - https://perplexity.ai/perplexitybot
80+
// QualifiedBot - https://www.qualified.com/legal/qualified-crawler-user-agent
7981
// SBIntuitionsBot - https://www.sbintuitions.co.jp/bot/
8082
// SeznamBot - http://napoveda.seznam.cz/seznambot-intro
8183
// SurdotlyBot - http://sur.ly/bot.html
8284
// Swiftbot - https://swiftype.com/swiftbot
8385
// YepBot - https://yep.com/yepbot/
84-
/((?:adidx|ahrefs|amazon|(?:amzn|oai)-search|awario(?:smart|rss)?|bing|brave|cc|contx|coveo|criteo|dot|duckduck(?:go-favicons-)?|exa|facebook|gpt|iask|kagi|kangaroo |kimi(?:-search)?|linkedin|mj12|mojeek|onespot-scraper|perplexity|sbintuitions|semrush|seznam|surdotly|swift|yep)bot)\/([\w\.-]+)/i,
86+
/((?:adidx|ahrefs|amazon|(?:amzn|oai)-search|awario(?:smart|rss)?|bing|borderx|brave|cc|contx|coveo|criteo|dot|duckduck(?:go-favicons-)?|exa|facebook|gpt|iask|kagi|kangaroo |kimi(?:-search)?|linkedin|mj12|mojeek|onespot-scraper|perplexity|qualified|sbintuitions|semrush|seznam|surdotly|swift|yep)bot)\/([\w\.-]+)/i,
8587

8688
// Algolia Crawler
8789
/(algolia crawler(?: renderscript)?)\/?([\w\.]*)/i,
@@ -132,10 +134,10 @@ const Crawlers = Object.freeze({
132134
// Yeti (Naver)
133135
/(yeti)\/([\w\.]+)/i,
134136

135-
// aiHitBot / Algolia Crawler / Audisto Crawler / BLEXBot / BrightEdge Crawler / Cloudflare AutoRAG / Diffbot / FirecrawlAgent / HuggingFace-Bot / Linespider / MSNBot / Magpie-Crawler / Omgilibot / OpenAI Image Downloader / PanguBot / Replicate-Bot / RunPod-Bot / Webzio-Extended / Screaming Frog SEO Spider / Startpage / Timpibot / Together-Bot / VelenPublicWebCrawler / xAI-Bot / YisouSpider / YouBot / ZumBot
137+
// aiHitBot / Algolia Crawler / Audisto Crawler / BLEXBot / BrightEdge Crawler / Cloudflare AutoRAG/Crawler / Diffbot / FirecrawlAgent / FishBot / HuggingFace-Bot / LinerBot / Linespider / MSNBot / Magpie-Crawler / Omgilibot / OpenAI Image Downloader / PanguBot / Replicate-Bot / RunPod-Bot / Webzio-Extended / Screaming Frog SEO Spider / Startpage / Timpibot / Together-Bot / VelenPublicWebCrawler / xAI-Bot / YisouSpider / YouBot / ZumBot
136138
// Cotoyogi - https://ds.rois.ac.jp/en_center8/en_crawler/
137139
// Freespoke - https://docs.freespoke.com/search/bot/
138-
/((?:aihit|blex|diff|huggingface-|msn|pangu|replicate-|runpod-|timpi|together-|xai-|you|zum)bot|(?:audisto |brightedge |magpie-|velenpublicweb)crawler|(?:chatglm-|line|screaming frog seo |yisou)spider|cloudflare-autorag|cotoyogi|(?:firecrawl|twin)agent|freespoke|omgili(?:bot)?|openai image downloader|startpageprivateimageproxy|webzio-extended)\/?([\w\.]*)/i
140+
/((?:aihit|blex|diff|fish|huggingface-|liner|msn|pangu|replicate-|runpod-|timpi|together-|xai-|you|zum)bot|(?:audisto |brightedge |magpie-|velenpublicweb)crawler|(?:chatglm-|line|screaming frog seo |yisou)spider|cloudflare(?:-autorag|b\w{21}r)|cotoyogi|(?:firecrawl|twin)agent|freespoke|omgili(?:bot)?|openai image downloader|startpagep\w{14}xy|webzio-extended)\/?([\w\.]*)/i
139141
],
140142
[NAME, VERSION, [TYPE, CRAWLER]],
141143

@@ -309,7 +311,7 @@ const Emails = Object.freeze({
309311
const Fetchers = Object.freeze({
310312
browser : [
311313
[
312-
// Asana / Bitlybot / Better Uptime / BingPreview / Blueno / Cohere-AI / HubSpot Page Fetcher / kakaotalk-scrap / Mastodon / MicrosoftPreview / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot / WhatsApp
314+
// Asana / Bitlybot / Better Uptime / BingPreview / Blueno / Cohere-AI / HubSpot Page Fetcher / kakaotalk-scrap / Mastodon / MicrosoftPreview / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot / WARDBot / WhatsApp
313315
// AhrefsSiteAudit - https://ahrefs.com/robot/site-audit
314316
// Buffer Link Preview Bot - https://scraper.buffer.com/about/bots/link-preview-bot
315317
// ChatGPT-User - https://platform.openai.com/docs/plugins/bot
@@ -321,7 +323,7 @@ const Fetchers = Object.freeze({
321323
// Perplexity-User - https://docs.perplexity.ai/guides/bots
322324
// MistralAI-User - https://docs.mistral.ai/robots/
323325
// Yandex Bots - https://yandex.com/bots
324-
/(asana|ahrefssiteaudit|(?:bing|microsoft)preview|blueno|(?:amzn|chatgpt|claude|kimi|mistralai|perplexity)-user|cohere-ai|flipboardproxy|hubspot page fetcher|mastodon|(?:bitly|bufferlinkpreview|discord|duckassist|linkedin|pinterest|reddit|roger|siteaudit|twitter|uptime(?:ro)?|zoom)bot|google-site-verification|iframely|kakaotalk-scrap|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|fordomain|pagechecker|searchshop)|yadirectfetcher|whatsapp)\/([\w\.]+)/i,
326+
/(asana|ahrefssiteaudit|(?:bing|microsoft)preview|blueno|(?:amzn|chatgpt|claude|kimi|mistralai|perplexity)-user|cohere-ai|flipboardproxy|hubspot page fetcher|mastodon|(?:bitly|bufferlinkpreview|discord|duckassist|linkedin|pinterest|reddit|roger|siteaudit|twitter|uptime(?:ro)?|ward|zoom)bot|google-site-verification|iframely|kakaotalk-scrap|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|fordomain|pagechecker|searchshop)|yadirectfetcher|whatsapp)\/([\w\.]+)/i,
325327

326328
// Bluesky
327329
/(bluesky) cardyb\/([\w\.]+)/i,

‎test/data/ua/extension/crawler.json‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,16 @@
359359
"type" : "crawler"
360360
}
361361
},
362+
{
363+
"desc" : "BorderxBot",
364+
"ua" : "Mozilla/5.0 (compatible; BorderxBot/1.0; +https://www.nubestore.ai/borderxbot.html)",
365+
"expect" :
366+
{
367+
"name" : "BorderxBot",
368+
"version" : "1.0",
369+
"type" : "crawler"
370+
}
371+
},
362372
{
363373
"desc" : "botify",
364374
"ua" : "Desktop: Mozilla/5.0 (compatible; botify; http://botify.com)",
@@ -459,6 +469,16 @@
459469
"type" : "crawler"
460470
}
461471
},
472+
{
473+
"desc" : "Cloudflare Crawler",
474+
"ua" : "CloudflareBrowserRenderingCrawler/1.0",
475+
"expect" :
476+
{
477+
"name" : "CloudflareBrowserRenderingCrawler",
478+
"version" : "1.0",
479+
"type" : "crawler"
480+
}
481+
},
462482
{
463483
"desc" : "Coc Coc Bot (web)",
464484
"ua" : "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)",
@@ -679,6 +699,16 @@
679699
"type" : "crawler"
680700
}
681701
},
702+
{
703+
"desc" : "FishBot",
704+
"ua" : "FishBot",
705+
"expect" :
706+
{
707+
"name" : "FishBot",
708+
"version" : "undefined",
709+
"type" : "crawler"
710+
}
711+
},
682712
{
683713
"desc" : "Freespoke",
684714
"ua" : "Mozilla/5.0 (compatible; Freespoke/2.0; +https://docs.freespoke.com/search/bot)",
@@ -929,6 +959,16 @@
929959
"type" : "crawler"
930960
}
931961
},
962+
{
963+
"desc" : "LinerBot",
964+
"ua" : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; LinerBot/1.0; +https://docs.getliner.com/docs/linerbot)",
965+
"expect" :
966+
{
967+
"name" : "LinerBot",
968+
"version" : "1.0",
969+
"type" : "crawler"
970+
}
971+
},
932972
{
933973
"desc" : "Linespider",
934974
"ua" : "Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)",
@@ -1130,6 +1170,16 @@
11301170
"type" : "crawler"
11311171
}
11321172
},
1173+
{
1174+
"desc" : "QualifiedBot",
1175+
"ua" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.6668.29 QualifiedBot/1.0 +https://www.qualified.com/legal/qualified-crawler-user-agent Safari/537.36",
1176+
"expect" :
1177+
{
1178+
"name" : "QualifiedBot",
1179+
"version" : "1.0",
1180+
"type" : "crawler"
1181+
}
1182+
},
11331183
{
11341184
"desc" : "Qwantbot",
11351185
"ua" : "Mozilla/5.0 (compatible; Qwantbot/1.0_12345; +https://help.qwant.com/bot/)",

‎test/data/ua/extension/fetcher.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -559,6 +559,16 @@
559559
"type" : "fetcher"
560560
}
561561
},
562+
{
563+
"desc" : "WARDBot",
564+
"ua" : "Mozilla/5.0 (compatible; WARDBot/1.0; http://ward.ai/robot)",
565+
"expect" :
566+
{
567+
"name" : "WARDBot",
568+
"version" : "1.0",
569+
"type" : "fetcher"
570+
}
571+
},
562572
{
563573
"desc" : "YaDirectFetcher",
564574
"ua" : "Mozilla/5.0 (compatible; YaDirectFetcher/1.0; Dyatel; +http://yandex.com/bots)",

0 commit comments

Comments
 (0)