{ "title": "Which top websites block AI crawlers", "updated": "2026-10-24T02:21:02.576Z", "full_passes": 13, "sites_measured": 225, "sites_blocking_any": 91, "by_bot": [ { "bot": "GPTBot", "who": "OpenAI (training)", "blocked": 68, "named": 69 }, { "bot": "ChatGPT-User", "who": "ChatGPT browsing", "blocked": 52, "named": 53 }, { "bot": "OAI-SearchBot", "who": "ChatGPT search", "blocked": 40, "named": 43 }, { "bot": "ClaudeBot", "who": "Anthropic (training)", "blocked": 71, "named": 68 }, { "bot": "Claude-User", "who": "Claude browsing", "blocked": 46, "named": 39 }, { "bot": "Claude-SearchBot", "who": "Claude search", "blocked": 44, "named": 39 }, { "bot": "PerplexityBot", "who": "Perplexity", "blocked": 56, "named": 63 }, { "bot": "Google-Extended", "who": "Gemini (training)", "blocked": 59, "named": 65 }, { "bot": "Applebot-Extended", "who": "Apple Intelligence", "blocked": 51, "named": 41 }, { "bot": "CCBot", "who": "Common Crawl", "blocked": 77, "named": 66 }, { "bot": "Meta-ExternalAgent", "who": "Meta AI", "blocked": 60, "named": 51 }, { "bot": "Bytespider", "who": "ByteDance", "blocked": 76, "named": 61 }, { "bot": "Amazonbot", "who": "Amazon", "blocked": 45, "named": 37 }, { "bot": "cohere-ai", "who": "Cohere", "blocked": 61, "named": 49 } ], "sites": [ { "domain": "asahi.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T02:21:01.724Z" }, { "domain": "baidu.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [], "checked": "2026-10-24T01:41:15.233Z" }, { "domain": "dailymail.co.uk", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T01:21:02.937Z" }, { "domain": "daum.net", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [], "checked": "2026-10-24T01:31:04.814Z" }, { "domain": "facebook.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "Amazonbot" ], "checked": "2026-10-24T01:11:03.536Z" }, { "domain": "hihonorcloud.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [], "checked": "2026-10-24T02:00:59.867Z" }, { "domain": "huffpost.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T02:21:02.443Z" }, { "domain": "instagram.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "Amazonbot" ], "checked": "2026-10-24T01:11:04.019Z" }, { "domain": "kakao.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [], "checked": "2026-10-24T01:21:10.702Z" }, { "domain": "mail.ru", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [], "checked": "2026-10-24T02:01:03.982Z" }, { "domain": "mediacategory.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [], "checked": "2026-10-24T02:01:10.280Z" }, { "domain": "msn.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T01:01:11.116Z" }, { "domain": "nbcnews.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T02:21:00.221Z" }, { "domain": "pinterest.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [], "checked": "2026-10-24T01:01:11.011Z" }, { "domain": "reddit.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [], "checked": "2026-10-24T02:21:02.413Z" }, { "domain": "soundcloud.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T01:20:59.510Z" }, { "domain": "zepto.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [], "checked": "2026-10-24T01:51:01.652Z" }, { "domain": "cnbc.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T01:01:10.788Z" }, { "domain": "imdb.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "Amazonbot" ], "checked": "2026-10-24T01:00:59.513Z" }, { "domain": "theglobeandmail.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T01:21:05.348Z" }, { "domain": "amazon.ca", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-23T22:31:12.602Z" }, { "domain": "amazon.co.jp", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:31:03.446Z" }, { "domain": "amazon.co.uk", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:41:03.378Z" }, { "domain": "amazon.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:11:05.578Z" }, { "domain": "amazon.com.au", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T02:11:11.711Z" }, { "domain": "amazon.com.br", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-23T23:01:15.286Z" }, { "domain": "amazon.com.mx", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:41:15.248Z" }, { "domain": "amazon.de", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:51:14.613Z" }, { "domain": "amazon.es", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:31:06.744Z" }, { "domain": "amazon.fr", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:51:01.652Z" }, { "domain": "amazon.in", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:31:01.672Z" }, { "domain": "amazon.it", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:31:04.021Z" }, { "domain": "bbc.co.uk", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T02:20:58.833Z" }, { "domain": "bbc.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T02:20:58.830Z" }, { "domain": "cnn.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Applebot-Extended", "CCBot", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Applebot-Extended", "CCBot", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T02:20:58.874Z" }, { "domain": "tiktok.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider" ], "checked": "2026-10-24T01:11:03.970Z" }, { "domain": "twitter.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Applebot-Extended", "CCBot", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "Google-Extended", "Meta-ExternalAgent" ], "checked": "2026-10-24T01:31:06.744Z" }, { "domain": "x.com", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Applebot-Extended", "CCBot", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "Google-Extended", "Meta-ExternalAgent" ], "checked": "2026-10-24T01:11:02.216Z" }, { "domain": "nature.com", "blocks": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T01:01:08.847Z" }, { "domain": "newyorker.com", "blocks": [ "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T02:21:02.576Z" }, { "domain": "spiegel.de", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T02:21:01.867Z" }, { "domain": "theverge.com", "blocks": [ "ChatGPT-User", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T02:20:59.766Z" }, { "domain": "vox.com", "blocks": [ "ChatGPT-User", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T02:21:02.225Z" }, { "domain": "wired.com", "blocks": [ "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T02:20:59.937Z" }, { "domain": "buzzfeed.com", "blocks": [ "GPTBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T02:21:02.407Z" }, { "domain": "lefigaro.fr", "blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "Google-Extended", "CCBot", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "Google-Extended", "CCBot", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T01:21:02.382Z" }, { "domain": "abc.net.au", "blocks": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:21:05.000Z" }, { "domain": "corriere.it", "blocks": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot" ], "checked": "2026-10-24T02:21:01.692Z" }, { "domain": "forbes.com", "blocks": [ "GPTBot", "ClaudeBot", "PerplexityBot", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-23T17:41:00.722Z" }, { "domain": "genius.com", "blocks": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:20:59.626Z" }, { "domain": "lemonde.fr", "blocks": [ "ClaudeBot", "Claude-User", "Claude-SearchBot", "Google-Extended", "Applebot-Extended", "CCBot", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "ClaudeBot", "Claude-User", "Claude-SearchBot", "Google-Extended", "Applebot-Extended", "CCBot", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T02:21:01.359Z" }, { "domain": "newsweek.com", "blocks": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Applebot-Extended", "CCBot", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Applebot-Extended", "CCBot", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T01:21:02.228Z" }, { "domain": "theguardian.com", "blocks": [ "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot" ], "names_in_robots": [ "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot" ], "checked": "2026-10-24T02:20:58.926Z" }, { "domain": "zigzag.kr", "blocks": [ "GPTBot", "ClaudeBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent" ], "checked": "2026-10-24T02:01:01.120Z" }, { "domain": "11st.co.kr", "blocks": [ "GPTBot", "ClaudeBot", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "ChatGPT-User", "OAI-SearchBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended" ], "checked": "2026-10-24T01:31:00.305Z" }, { "domain": "techcrunch.com", "blocks": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "Applebot-Extended", "CCBot", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "Google-Extended", "Applebot-Extended", "CCBot", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T02:20:59.498Z" }, { "domain": "yahoo.com", "blocks": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Google-Extended", "CCBot", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:01:11.021Z" }, { "domain": "amazonvideo.com", "blocks": [ "ChatGPT-User", "Claude-User", "PerplexityBot", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "names_in_robots": [ "ChatGPT-User", "Claude-User", "PerplexityBot", "CCBot", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:31:07.292Z" }, { "domain": "elpais.com", "blocks": [ "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "CCBot", "Bytespider", "Amazonbot" ], "names_in_robots": [ "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "CCBot", "Bytespider", "Amazonbot" ], "checked": "2026-10-24T02:21:00.739Z" }, { "domain": "healthline.com", "blocks": [ "GPTBot", "ClaudeBot", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot" ], "names_in_robots": [ "GPTBot", "ClaudeBot", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot" ], "checked": "2026-10-24T01:01:10.241Z" }, { "domain": "repubblica.it", "blocks": [ "GPTBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T01:21:03.745Z" }, { "domain": "t-online.de", "blocks": [ "GPTBot", "ClaudeBot", "PerplexityBot", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider" ], "names_in_robots": [ "GPTBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider" ], "checked": "2026-10-24T01:51:04.571Z" }, { "domain": "aljazeera.com", "blocks": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:21:10.710Z" }, { "domain": "aol.com", "blocks": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Google-Extended", "CCBot", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:01:11.524Z" }, { "domain": "bild.de", "blocks": [ "PerplexityBot", "Applebot-Extended", "CCBot", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "PerplexityBot", "Applebot-Extended", "CCBot", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T01:21:03.150Z" }, { "domain": "canva.com", "blocks": [ "GPTBot", "ClaudeBot", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider" ], "names_in_robots": [ "GPTBot", "ClaudeBot", "PerplexityBot", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider" ], "checked": "2026-10-23T23:40:59.102Z" }, { "domain": "figma.com", "blocks": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Bytespider", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "CCBot", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:11:01.294Z" }, { "domain": "netflix.com", "blocks": [ "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended" ], "checked": "2026-10-24T01:31:00.814Z" }, { "domain": "tumblr.com", "blocks": [ "ClaudeBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Amazonbot" ], "names_in_robots": [ "ClaudeBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Amazonbot" ], "checked": "2026-10-24T01:01:11.141Z" }, { "domain": "whatsapp.com", "blocks": [ "GPTBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "Amazonbot" ], "names_in_robots": [ "GPTBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "Amazonbot" ], "checked": "2026-10-24T01:31:01.219Z" }, { "domain": "espn.com", "blocks": [ "GPTBot", "ChatGPT-User", "Google-Extended", "CCBot", "Bytespider" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "Google-Extended", "CCBot", "Bytespider" ], "checked": "2026-10-24T01:01:10.185Z" }, { "domain": "vimeo.com", "blocks": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Bytespider" ], "names_in_robots": [ "GPTBot", "ClaudeBot", "Google-Extended", "CCBot", "Bytespider" ], "checked": "2026-10-24T01:11:08.046Z" }, { "domain": "airbnb.com", "blocks": [ "GPTBot", "ClaudeBot", "Applebot-Extended" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "PerplexityBot", "Applebot-Extended", "Meta-ExternalAgent", "cohere-ai" ], "checked": "2026-10-24T01:11:03.788Z" }, { "domain": "businessinsider.com", "blocks": [ "ClaudeBot", "CCBot", "Bytespider" ], "names_in_robots": [ "ClaudeBot", "CCBot", "Bytespider" ], "checked": "2026-10-24T02:20:59.780Z" }, { "domain": "dictionary.com", "blocks": [ "GPTBot", "Applebot-Extended", "CCBot" ], "names_in_robots": [ "GPTBot", "Applebot-Extended", "CCBot" ], "checked": "2026-10-24T01:01:09.414Z" }, { "domain": "globo.com", "blocks": [ "GPTBot", "Google-Extended", "CCBot" ], "names_in_robots": [ "GPTBot", "Google-Extended", "CCBot" ], "checked": "2026-10-24T01:21:09.497Z" }, { "domain": "jio.com", "blocks": [ "CCBot", "Meta-ExternalAgent", "Bytespider" ], "names_in_robots": [ "GPTBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider" ], "checked": "2026-10-24T01:51:18.462Z" }, { "domain": "latimes.com", "blocks": [ "GPTBot", "ClaudeBot", "Bytespider" ], "names_in_robots": [ "GPTBot", "ClaudeBot", "Bytespider" ], "checked": "2026-10-24T02:21:00.022Z" }, { "domain": "weather.com", "blocks": [ "CCBot", "Bytespider", "Amazonbot" ], "names_in_robots": [ "ChatGPT-User", "CCBot", "Bytespider", "Amazonbot" ], "checked": "2026-10-24T01:01:09.320Z" }, { "domain": "webmd.com", "blocks": [ "GPTBot", "ClaudeBot", "CCBot" ], "names_in_robots": [ "GPTBot", "ClaudeBot", "CCBot" ], "checked": "2026-10-24T01:01:10.227Z" }, { "domain": "goodreads.com", "blocks": [ "GPTBot", "CCBot" ], "names_in_robots": [ "GPTBot", "CCBot" ], "checked": "2026-10-24T01:21:01.215Z" }, { "domain": "notion.com", "blocks": [ "Meta-ExternalAgent", "Amazonbot" ], "names_in_robots": [ "Meta-ExternalAgent", "Amazonbot" ], "checked": "2026-10-24T02:11:01.095Z" }, { "domain": "yahoo.co.jp", "blocks": [ "GPTBot", "Google-Extended" ], "names_in_robots": [ "GPTBot", "Google-Extended" ], "checked": "2026-10-24T01:21:10.365Z" }, { "domain": "amplitude.com", "blocks": [ "Bytespider" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "Meta-ExternalAgent", "Bytespider", "Amazonbot", "cohere-ai" ], "checked": "2026-10-24T01:41:14.169Z" }, { "domain": "coursera.org", "blocks": [ "Meta-ExternalAgent" ], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "PerplexityBot", "Google-Extended", "CCBot", "Meta-ExternalAgent" ], "checked": "2026-10-24T01:21:00.407Z" }, { "domain": "github.com", "blocks": [ "Bytespider" ], "names_in_robots": [ "GPTBot", "OAI-SearchBot", "ClaudeBot", "PerplexityBot", "Bytespider" ], "checked": "2026-10-24T01:01:09.005Z" }, { "domain": "hindustantimes.com", "blocks": [ "Meta-ExternalAgent" ], "names_in_robots": [ "Google-Extended", "Meta-ExternalAgent" ], "checked": "2026-10-24T01:21:05.317Z" }, { "domain": "indiatimes.com", "blocks": [ "Meta-ExternalAgent" ], "names_in_robots": [ "Meta-ExternalAgent" ], "checked": "2026-10-24T01:21:05.490Z" }, { "domain": "kwai.com", "blocks": [ "Bytespider" ], "names_in_robots": [ "Bytespider" ], "checked": "2026-10-24T01:51:03.900Z" }, { "domain": "notion.so", "blocks": [ "Amazonbot" ], "names_in_robots": [ "Amazonbot" ], "checked": "2026-10-24T01:11:00.378Z" }, { "domain": "straitstimes.com", "blocks": [ "Google-Extended" ], "names_in_robots": [ "Google-Extended" ], "checked": "2026-10-24T01:21:05.494Z" }, { "domain": "163.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:01:02.119Z" }, { "domain": "360yield.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:51:04.407Z" }, { "domain": "a-mo.net", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:01:08.315Z" }, { "domain": "adobe.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:11:08.523Z" }, { "domain": "adtrafficquality.google", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:01:08.432Z" }, { "domain": "ahnlab.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T22:31:12.464Z" }, { "domain": "ampproject.org", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T00:41:06.438Z" }, { "domain": "android.com", "blocks": [], "names_in_robots": [ "GPTBot", "OAI-SearchBot", "ClaudeBot", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot", "cohere-ai" ], "checked": "2026-10-24T01:51:00.833Z" }, { "domain": "anthropic.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:11:04.981Z" }, { "domain": "appier.net", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:51:01.652Z" }, { "domain": "apple.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:11:04.622Z" }, { "domain": "arxiv.org", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:01:07.159Z" }, { "domain": "auction.co.kr", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:21:10.612Z" }, { "domain": "bing.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:31:01.094Z" }, { "domain": "blogspot.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:41:03.326Z" }, { "domain": "booking.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:11:02.990Z" }, { "domain": "brave.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:41:04.193Z" }, { "domain": "braze.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:11:13.431Z" }, { "domain": "cdc.gov", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:01:07.851Z" }, { "domain": "charter.net", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:01:05.100Z" }, { "domain": "cisco.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:11:10.165Z" }, { "domain": "clarin.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:21:04.899Z" }, { "domain": "claude.ai", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T12:31:04.307Z" }, { "domain": "cloudflare.com", "blocks": [], "names_in_robots": [ "GPTBot", "ChatGPT-User", "PerplexityBot", "Google-Extended", "CCBot", "cohere-ai" ], "checked": "2026-10-24T01:11:04.923Z" }, { "domain": "coupang.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T21:01:01.440Z" }, { "domain": "cursor.sh", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:51:06.224Z" }, { "domain": "dell.com", "blocks": [], "names_in_robots": [ "Bytespider" ], "checked": "2026-10-24T01:11:09.436Z" }, { "domain": "discord.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:10:59.725Z" }, { "domain": "domaindetails.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:01:09.441Z" }, { "domain": "dotomi.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:41:13.855Z" }, { "domain": "doubao.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T00:41:09.006Z" }, { "domain": "dropbox.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:11:01.299Z" }, { "domain": "duolingo.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:11:03.520Z" }, { "domain": "fast.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:51:12.704Z" }, { "domain": "flashtalking.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:01:08.383Z" }, { "domain": "flipkart.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:31:04.745Z" }, { "domain": "foxnews.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:20:59.928Z" }, { "domain": "google.ca", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T19:41:59.945Z" }, { "domain": "google.co.in", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T22:31:00.998Z" }, { "domain": "google.co.jp", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T22:31:03.988Z" }, { "domain": "google.co.kr", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T22:31:01.801Z" }, { "domain": "google.co.uk", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T19:42:03.783Z" }, { "domain": "google.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T23:41:01.556Z" }, { "domain": "google.com.au", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T22:31:12.391Z" }, { "domain": "google.com.br", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T22:31:00.940Z" }, { "domain": "google.com.mx", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T22:31:12.440Z" }, { "domain": "google.de", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T22:50:59.005Z" }, { "domain": "google.es", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T19:41:27.936Z" }, { "domain": "google.fr", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T19:42:00.009Z" }, { "domain": "google.it", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T22:31:05.898Z" }, { "domain": "google.nl", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T00:31:16.887Z" }, { "domain": "google.ru", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T23:14:04.416Z" }, { "domain": "grammarly.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:41:01.341Z" }, { "domain": "harvard.edu", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:21:00.272Z" }, { "domain": "hcaptcha.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:11:01.116Z" }, { "domain": "hotstar.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:31:04.015Z" }, { "domain": "hp.com", "blocks": [], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended" ], "checked": "2026-10-24T01:11:10.756Z" }, { "domain": "ibm.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:11:05.319Z" }, { "domain": "icloud.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:21:10.637Z" }, { "domain": "independent.co.uk", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:21:02.810Z" }, { "domain": "intel.com", "blocks": [], "names_in_robots": [ "GPTBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Google-Extended" ], "checked": "2026-10-24T01:11:09.500Z" }, { "domain": "intercom.io", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:51:06.063Z" }, { "domain": "ip-api.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:31:05.162Z" }, { "domain": "iubenda.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:51:15.589Z" }, { "domain": "japantimes.co.jp", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:21:05.652Z" }, { "domain": "kaspersky.com", "blocks": [], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "PerplexityBot", "Google-Extended", "Applebot-Extended", "CCBot" ], "checked": "2026-10-24T02:11:05.650Z" }, { "domain": "khanacademy.org", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:21:00.162Z" }, { "domain": "launchdarkly.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:31:05.152Z" }, { "domain": "line.me", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:51:14.945Z" }, { "domain": "live.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:31:01.398Z" }, { "domain": "microsoft.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:11:05.496Z" }, { "domain": "mit.edu", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:21:00.397Z" }, { "domain": "moengage.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:51:06.027Z" }, { "domain": "mozilla.org", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:41:03.105Z" }, { "domain": "nasa.gov", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:01:07.733Z" }, { "domain": "nationalgeographic.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:21:01.431Z" }, { "domain": "naver.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:21:10.800Z" }, { "domain": "naver.net", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:11:12.505Z" }, { "domain": "navercorp.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:41:01.845Z" }, { "domain": "nerdwallet.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:01:11.208Z" }, { "domain": "nexon.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:01:00.645Z" }, { "domain": "nike.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:01:08.579Z" }, { "domain": "nubank.com.br", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:41:04.125Z" }, { "domain": "nvidia.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:11:05.356Z" }, { "domain": "office.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:21:10.712Z" }, { "domain": "onelink.me", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:51:03.660Z" }, { "domain": "opera.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:31:03.875Z" }, { "domain": "outlook.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:41:06.109Z" }, { "domain": "paytm.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:51:05.871Z" }, { "domain": "perplexity.ai", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:11:04.923Z" }, { "domain": "phonepe.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:11:02.001Z" }, { "domain": "publicnode.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:01:04.878Z" }, { "domain": "rakuten.co.jp", "blocks": [], "names_in_robots": [], "checked": "2026-10-23T18:29:24.756Z" }, { "domain": "roblox.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:51:02.063Z" }, { "domain": "salesforce.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:11:09.118Z" }, { "domain": "samsung.com", "blocks": [], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "PerplexityBot", "Google-Extended" ], "checked": "2026-10-24T01:11:09.466Z" }, { "domain": "scmp.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:21:04.981Z" }, { "domain": "shopee.com.br", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:31:02.067Z" }, { "domain": "shopify.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:10:59.754Z" }, { "domain": "slack.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:10:59.728Z" }, { "domain": "smilewanted.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T00:41:05.588Z" }, { "domain": "sonyliv.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T00:41:05.833Z" }, { "domain": "spotify.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:11:01.334Z" }, { "domain": "springer.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:01:09.151Z" }, { "domain": "stackadapt.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:11:02.637Z" }, { "domain": "stanford.edu", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:21:00.810Z" }, { "domain": "stripe.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:51:03.275Z" }, { "domain": "target.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:00:59.950Z" }, { "domain": "teads.tv", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:01:04.942Z" }, { "domain": "telegram.org", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:31:16.852Z" }, { "domain": "time.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:21:01.700Z" }, { "domain": "toss.im", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:31:05.904Z" }, { "domain": "tossinvest.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:01:09.366Z" }, { "domain": "truecaller.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:01:09.235Z" }, { "domain": "twitch.tv", "blocks": [], "names_in_robots": [ "Amazonbot" ], "checked": "2026-10-24T01:11:01.200Z" }, { "domain": "uber.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:11:03.484Z" }, { "domain": "ubuntu.com", "blocks": [], "names_in_robots": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "Meta-ExternalAgent", "Bytespider", "cohere-ai" ], "checked": "2026-10-24T01:41:00.594Z" }, { "domain": "v-videoapp.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:51:04.590Z" }, { "domain": "visualstudio.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:41:05.279Z" }, { "domain": "vivoglobal.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:01:00.593Z" }, { "domain": "vk.ru", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T00:41:05.138Z" }, { "domain": "walmart.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:00:59.669Z" }, { "domain": "weplayapp.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:51:14.152Z" }, { "domain": "wikipedia.org", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:00:59.690Z" }, { "domain": "wildberries.ru", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:11:04.796Z" }, { "domain": "wordpress.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:01:10.745Z" }, { "domain": "wps.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:41:05.266Z" }, { "domain": "xero.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:11:02.007Z" }, { "domain": "xhamster46.desi", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:41:14.079Z" }, { "domain": "yandex.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:11:16.433Z" }, { "domain": "yandex.ru", "blocks": [], "names_in_robots": [ "OAI-SearchBot" ], "checked": "2026-10-24T01:41:03.858Z" }, { "domain": "yastatic.net", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T02:01:07.721Z" }, { "domain": "youtube.com", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:11:03.809Z" }, { "domain": "zoom.us", "blocks": [], "names_in_robots": [], "checked": "2026-10-24T01:10:59.818Z" } ], "changes": [ { "at": "2026-10-24T01:51:01.652Z", "domain": "zepto.com", "now_blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "Meta-ExternalAgent", "Amazonbot" ], "now_allows": [] }, { "at": "2026-10-23T22:50:59.992Z", "domain": "zepto.com", "now_blocks": [], "now_allows": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "Meta-ExternalAgent", "Amazonbot" ] }, { "at": "2026-10-23T19:51:15.863Z", "domain": "zepto.com", "now_blocks": [ "GPTBot", "ChatGPT-User", "OAI-SearchBot", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Google-Extended", "Applebot-Extended", "Meta-ExternalAgent", "Amazonbot" ], "now_allows": [] }, { "at": "2026-10-23T12:01:05.441Z", "domain": "hindustantimes.com", "now_blocks": [ "Meta-ExternalAgent" ], "now_allows": [] }, { "at": "2026-10-23T10:31:59.283Z", "domain": "hindustantimes.com", "now_blocks": [], "now_allows": [ "Meta-ExternalAgent" ] }, { "at": "2026-10-23T09:01:09.235Z", "domain": "hindustantimes.com", "now_blocks": [ "Meta-ExternalAgent" ], "now_allows": [] } ], "method": "Each site's public robots.txt, read with RFC 9309 group rules: a bot is blocked when its own group (or, if it has none, the * group) disallows / without allowing it. Homepage must answer as a website.", "license": "CC BY 4.0, cite https://check.agentexchange.work/ai-crawler-blocklist" }