diff --git a/lib/prerender_rails.rb b/lib/prerender_rails.rb index af264c9..c88234f 100644 --- a/lib/prerender_rails.rb +++ b/lib/prerender_rails.rb @@ -19,38 +19,82 @@ def initialize(app, options={}) @crawler_user_agents = [ 'googlebot', + 'adsbot-google', + 'apis-google', + 'mediapartners-google', + 'google-safety', + 'feedfetcher-google', + 'googleproducer', + 'google-site-verification', 'Google-InspectionTool', - 'yahoo', + 'developers.google.com/+/web/snippet', + 'www.google.com/webmasters/tools/richsnippets', + 'Google Page Speed', + 'google-extended', 'bingbot', + 'yandexbot', + 'yabrowser', + 'yahoo', 'baiduspider', + 'naver', + 'seznambot', + 'sznprohlizec', + 'qwantbot', + 'Qwantify', + 'ecosia', + 'duckduckbot', + 'duckassistbot', + 'applebot', 'facebookexternalhit', + 'facebookcatalog', + 'facebookbot', + 'meta-externalagent', + 'instagram', 'twitterbot', - 'rogerbot', 'linkedinbot', - 'embedly', - 'bufferbot', - 'quora link preview', - 'showyoubot', - 'outbrain', - 'pinterest/0.', - 'developers.google.com/+/web/snippet', - 'www.google.com/webmasters/tools/richsnippets', + 'whatsapp', 'slackbot', - 'vkShare', - 'W3C_Validator', + 'pinterest', + 'pinterest/0.', + 'tiktok', + 'tiktokspider', + 'bytespider', + 'discordbot', 'redditbot', - 'Applebot', - 'WhatsApp', + 'skypeuripreview', 'flipboard', 'tumblr', 'bitlybot', - 'SkypeUriPreview', 'nuzzel', - 'Discordbot', - 'Google Page Speed', - 'Qwantify', - 'Chrome-Lighthouse', - 'TelegramBot' + 'outbrain', + 'showyoubot', + 'vkshare', + 'embedly', + 'bufferbot', + 'w3c_validator', + 'chrome-lighthouse', + 'screaming-frog', + 'oncrawlbot', + 'botifybot', + 'deepcrawl', + 'lumar', + 'semrushbot', + 'ahrefsbot', + 'rogerbot', + 'dotbot', + 'gptbot', + 'chatgpt', + 'oai-searchbot', + 'chatgpt-user', + 'claudebot', + 'anthropic-ai', + 'claude-web', + 'perplexitybot', + 'perplexity-user', + 'youbot', + 'amazonbot', + 'ccbot', + 'mistralai-user', ] @extensions_to_ignore = [