diff --git a/index.js b/index.js index 56e98af..c733083 100644 --- a/index.js +++ b/index.js @@ -42,11 +42,24 @@ var prerender = module.exports = function(req, res, next) { prerender.crawlerUserAgents = [ 'googlebot', + 'adsbot-google', + 'apis-google', + 'mediapartners-google', + 'google-safety', + 'feedfetcher-google', + 'googleproducer', + 'google-site-verification' 'Google-InspectionTool', 'Yahoo! Slurp', 'bingbot', 'yandex', + 'yandexbot', 'baiduspider', + 'naver', + 'seznambot', + 'sznprohlizec', + 'qwantbot', + 'ecosia', 'facebookexternalhit', 'twitterbot', 'rogerbot', @@ -85,6 +98,10 @@ prerender.crawlerUserAgents = [ 'ChatGPT-User', 'GPTBot', 'ChatGPT-User/1.0', + 'oai-searchbot', + 'claudebot', + 'claude-web', + 'amazonbot', 'Anthropic/Claude', 'anthropic-ai', 'Anthropic/Claude', @@ -93,9 +110,10 @@ prerender.crawlerUserAgents = [ 'Microsoft/Bing AI', 'BingBot/AI', 'BingPreview', - 'Perplexity', 'Perplexitybot', 'PerplexityBot/1.0', + 'perplexity-user', + 'youbot', 'Cohere', 'Cohere-ai', 'cohere-crawler', @@ -106,11 +124,14 @@ prerender.crawlerUserAgents = [ 'Baiduspider-AI', 'YourBot', 'You-bot', + 'mistralai-user', 'NeevaBot', 'Hugging-Face-AI', 'HuggingFaceBot', 'LinkedInBot/AI', 'facebookexternalhit/AI', + 'meta-externalagent', + 'facebookcatalog', 'FacebookBot', 'Twitterbot', 'redditbot', @@ -119,7 +140,21 @@ prerender.crawlerUserAgents = [ 'research-crawler', 'Diffbot', 'ScaleAI', - 'scale-crawler' + 'scale-crawler', + 'rogerbot', + 'semrushbot', + 'ahrefsbot', + 'chrome-lighthouse', + 'screaming-frog', + 'oncrawlbot', + 'botifybot', + 'deepcrawl', + 'lumar', + 'dotbot', + 'duckduckbot', + 'duckassistbot', + 'instagram', + 'tiktokspider', ];