{ "name": "Suede Labs AI Crawler Access Index", "version": "2026-08-19", "measured_utc": "2026-08-19", "sample_frame": "Tranco top 1,000 domains (tranco-list.eu top-1m, rank 1-1000)", "method": "For each domain, GET https:///robots.txt and https:///llms.txt (falling back to the www host), 8s timeout. robots.txt parsed with consecutive User-agent lines sharing the following rule block; a named agent group overrides the wildcard group. An agent counts as blocked when its governing group contains Disallow: / without an overriding Allow: /. Every llms.txt hit was re-tested against a control path that cannot exist; hits from servers that return the same 200 body for the control were discarded as catch-all false positives.", "agents_tested": [ "gptbot", "oai-searchbot", "chatgpt-user", "claudebot", "anthropic-ai", "claude-web", "perplexitybot", "ccbot", "google-extended", "bytespider", "applebot-extended", "meta-externalagent" ], "totals": { "sampled": 1000, "served_robots_txt": 597, "no_robots_txt_or_unreachable": 403, "llms_txt_confirmed": 103, "llms_txt_false_positives_discarded": 3, "blocks_any_of_five_major": 160, "blocks_all_five_major": 75, "names_at_least_one_ai_agent": 175 }, "blocked_at_root_by_agent": { "gptbot": 128, "oai-searchbot": 76, "chatgpt-user": 88, "claudebot": 129, "anthropic-ai": 103, "claude-web": 102, "perplexitybot": 104, "ccbot": 146, "google-extended": 119, "bytespider": 139, "applebot-extended": 111, "meta-externalagent": 119 }, "notes": "Percentages for per-agent blocking use the 597 domains that served a parseable robots.txt as the denominator, not all 1,000. The remaining 403 are largely CDN, DNS and API endpoints that serve no robots.txt. llms.txt percentages are reported against all 1,000 sampled.", "license": "CC BY 4.0. Cite as: Suede Labs AI Crawler Access Index, 19 August 2026, https://suedeai.ai/blog/ai-crawler-access-index", "results": [ { "domain": "google.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cloudflare.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "anthropic-ai", "ccbot", "chatgpt-user", "claude-web", "google-extended", "gptbot", "perplexitybot" ] }, { "domain": "gstatic.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "facebook.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "bytespider", "ccbot", "chatgpt-user", "claude-web", "meta-externalagent", "oai-searchbot" ], "names_ai_agents": [ "applebot-extended", "claudebot", "google-extended", "gptbot", "perplexitybot" ] }, { "domain": "gtld-servers.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "microsoft.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "googleapis.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amazonaws.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "youtube.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "apple.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "instagram.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "applebot-extended", "claudebot", "google-extended", "gptbot", "perplexitybot" ] }, { "domain": "mail.ru", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "akamai.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ezviz7.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "fbcdn.net", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "applebot-extended", "claudebot", "google-extended", "gptbot", "perplexitybot" ] }, { "domain": "twitter.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "gptbot", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "google-extended", "meta-externalagent" ] }, { "domain": "dzen.ru", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "linkedin.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "domaincontrol.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "office.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "googlevideo.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "hicloudcam.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "googletagmanager.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "live.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "akamaiedge.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amazon.com", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "akadns.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "azure.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "bing.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "github.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "wikipedia.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "whatsapp.net", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "claudebot", "google-extended", "gptbot", "perplexitybot" ], "names_ai_agents": [ "applebot-extended", "claudebot", "google-extended", "gptbot", "perplexitybot" ] }, { "domain": "fastly.net", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "apple-dns.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "googleusercontent.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "appsflyersdk.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "doubleclick.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "aaplimg.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "microsoftonline.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "netflix.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "meta-externalagent" ], "names_ai_agents": [ "chatgpt-user", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "office.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "trafficmanager.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "sharepoint.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "gandi.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "digicert.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "chatgpt-user", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "wordpress.org", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "perplexitybot" ] }, { "domain": "youtu.be", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "skype.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "goo.gl", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cloud.microsoft", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cloudfront.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "x.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "gptbot", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "google-extended", "meta-externalagent" ] }, { "domain": "googledomains.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "tiktok.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "pinterest.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "whatsapp.com", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "claudebot", "google-extended", "gptbot", "perplexitybot" ], "names_ai_agents": [ "applebot-extended", "claudebot", "google-extended", "gptbot", "perplexitybot" ] }, { "domain": "icloud.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "roblox.com", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ] }, { "domain": "yahoo.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "windowsupdate.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cloudflare.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "msn.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "gvt1.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "googlesyndication.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "spotify.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "windows.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ntp.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "adobe.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "wa.me", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "myfritz.net", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "cdninstagram.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "pv-cdn.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "chatgpt.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "bytespider", "ccbot", "claude-web", "google-extended", "perplexitybot" ] }, { "domain": "ripn.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nginx.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "akam.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "tiktokcdn.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "gwfb.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "zoom.us", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "vimeo.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "gptbot" ], "names_ai_agents": [ "anthropic-ai", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "gptbot" ] }, { "domain": "tiktokv.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "edgekey.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "baidu.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "gvt2.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "windows.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "yandex.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cloudflare-dns.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "root-servers.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ytimg.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "qq.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "opera.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "mozilla.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "samsung.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "chatgpt-user", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "workers.dev", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "openai.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "blogspot.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nic.ru", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nginx.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wordpress.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "workbox.dk", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "edgesuite.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "yandex.ru", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "outlook.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "capgemini.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "sentry.io", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "reddit.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "ui.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [ "applebot-extended", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "t.me", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "office365.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "a2z.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "b-cdn.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bit.ly", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "okcdn.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "europa.eu", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "google-analytics.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "registrar-servers.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "vk.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "github.io", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "discord.gg", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "criteo.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amazonvideo.com", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "chatgpt-user", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "meta-externalagent", "perplexitybot" ] }, { "domain": "vkuserphoto.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "snapchat.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "aws.dev", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "apache.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "afternic.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dnsowl.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "yccdn.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "app-measurement.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cdn77.org", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "app-analytics-services.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "unity3d.com", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider" ], "names_ai_agents": [ "bytespider" ] }, { "domain": "amazon-adsystem.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nih.gov", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "f5.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "prodregistryv2.org", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amazon.dev", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "trbcdn.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dns.google", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "msftncsi.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dropbox.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "miit.gov.cn", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "iiko.it", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "kaspersky.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "applebot-extended", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "forms.gle", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "reg.ru", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dns-parking.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "vedcdnlb.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "netflix.net", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "meta-externalagent" ], "names_ai_agents": [ "chatgpt-user", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "archive.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "godaddy.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "intuit.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "xiaomi.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "gravatar.com", "robots_status": 200, "llms_txt": true, "blocked": [ "bytespider", "ccbot", "google-extended", "gptbot" ], "names_ai_agents": [ "bytespider", "ccbot", "google-extended", "gptbot" ] }, { "domain": "static.microsoft", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "azurefd.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "steamserver.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nytimes.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "epicgames.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "tumblr.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "ccbot", "claudebot", "google-extended", "meta-externalagent" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "ccbot", "claudebot", "google-extended", "meta-externalagent" ] }, { "domain": "one.one", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "paypal.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "discord.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "msftconnecttest.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "applovin.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "3gppnetwork.org", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "aliyuncs.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "adtrafficquality.google", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ax-msedge.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "tm-azurefd.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "shopify.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "googleadservices.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "soundcloud.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "vk.ru", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ggpht.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mts.ru", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "flickr.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "chatgpt-user", "oai-searchbot", "perplexitybot" ] }, { "domain": "macromedia.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "adobe.io", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "jomodns.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nflxso.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "example.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "azurewebsites.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "webex.com", "robots_status": null, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "w3.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "medium.com", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "bytespider", "claudebot", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "applebot-extended", "bytespider", "claudebot", "gptbot", "meta-externalagent" ] }, { "domain": "cdn-apple.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "playstation.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "hichina.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cnn.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "theguardian.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claudebot", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claudebot", "meta-externalagent", "perplexitybot" ] }, { "domain": "rubiconproject.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "namebrightdns.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "vungle.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bytefcdn-oversea.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "creativecommons.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "oracle.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "spo-msedge.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "sfx.ms", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "lsrelayaccess.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "shifen.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "lencr.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "taboola.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "edgcdn.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "forbes.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "doi.org", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bbc.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "ttdns2.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "pki.goog", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "miui.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ebay.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claudebot", "gptbot", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claudebot", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "cpanel.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "youtube-nocookie.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "sciencedirect.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ozon.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "t-online.de", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent", "perplexitybot" ] }, { "domain": "researchgate.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mit.edu", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "gmail.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "twitch.tv", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "comcast.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "adsafeprotected.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "fastly-edge.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cdngslb.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nist.gov", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bbc.co.uk", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "t.co", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "ack.de", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "akamaized.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "sourceforge.net", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "ubuntu.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [ "anthropic-ai", "bytespider", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "perplexitybot" ] }, { "domain": "userapi.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "pages.dev", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "azureedge.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nr-data.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "etsy.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "telecid.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "stripe.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "ksyuncdn.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "canva.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "gptbot", "meta-externalagent", "perplexitybot" ] }, { "domain": "android.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "ccbot", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "pubmatic.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "oxylabs.io", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "telekom.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "telegram.org", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cisco.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "ccbot", "claudebot", "google-extended", "gptbot" ] }, { "domain": "who.int", "robots_status": 200, "llms_txt": false, "blocked": [ "ccbot" ], "names_ai_agents": [ "ccbot" ] }, { "domain": "hubspot.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "imdb.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "adsrvr.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "rbxcdn.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wsdvs.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "booking.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wixsite.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wikimedia.org", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "adnxs.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "linktr.ee", "robots_status": 200, "llms_txt": true, "blocked": [ "anthropic-ai", "bytespider", "meta-externalagent" ], "names_ai_agents": [ "applebot-extended", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "omtrdc.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "scorecardresearch.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "launchpad.net", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "wac-msedge.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "meraki.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "ccbot" ] }, { "domain": "roku.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "name-services.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "vecdnlb.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ibm.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "hcaptcha.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "alibabadns.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "facebook.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "geobasket.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wildberries.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "springer.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "msedge.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "reuters.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "chatgpt-user", "oai-searchbot" ] }, { "domain": "alidns.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "tinyurl.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ttvnw.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "duckdns.org", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "weather.com", "robots_status": 200, "llms_txt": true, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "perplexitybot" ] }, { "domain": "forter.com", "robots_status": 200, "llms_txt": true, "blocked": [ "bytespider", "ccbot", "meta-externalagent" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "arubanetworks.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "virginm.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nature.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "perplexitybot" ] }, { "domain": "mozilla.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "weibo.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "bytespider", "chatgpt-user", "meta-externalagent", "oai-searchbot" ] }, { "domain": "hp.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [ "chatgpt-user", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "tiktokcdn-us.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "spov-msedge.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "byteoversea.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "imcmdb.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mangosip.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "www.gov.uk", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [ "meta-externalagent" ] }, { "domain": "debian.org", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "harvard.edu", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "clarity.ms", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "inmobi.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "avast.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "chatgpt-user", "claudebot", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "steampowered.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "jsdelivr.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "alibaba.com", "robots_status": 200, "llms_txt": false, "blocked": [ "google-extended", "gptbot" ], "names_ai_agents": [ "google-extended", "gptbot" ] }, { "domain": "media-amazon.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "autodesk.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "php.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "samsungcloud.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mzstatic.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ok.ru", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wiley.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amplitude.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "appsflyer.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "googleblog.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "shalltry.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "myshopify.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mtgglobals.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "salesforce.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "dnsmadeeasy.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "hostgator.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "heytapmobile.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amazon.co.uk", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "samsungqbe.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "aliyuncsslbintl.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "gnu.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "launchdarkly.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "datadoghq.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "aliyun.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "libp2p.direct", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "douyincdn.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "crpt.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "smartadserver.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "un.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "drom.ru", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "doubleverify.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "crashlytics.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "media.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ea.com", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "meta-externalagent" ], "names_ai_agents": [ "bytespider", "meta-externalagent" ] }, { "domain": "telegram.me", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bilibili.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "cdc.gov", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "weebly.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "checkpoint.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "name.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "inner-active.mobi", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "issuu.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "naver.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "mi.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "trustpilot.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "meta-externalagent" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "onelink.me", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "2mdn.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "akamaihd.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "tbcache.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wsj.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "chatgpt-user", "gptbot", "oai-searchbot" ] }, { "domain": "ioref.io", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "slack.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "allawnos.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "washingtonpost.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "meta-externalagent", "perplexitybot" ] }, { "domain": "opendns.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dotaplabs.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "openx.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "quickconnect.to", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "tld-servers.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bloomberg.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "tradingview.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [ "applebot-extended", "bytespider", "claudebot", "google-extended", "meta-externalagent", "perplexitybot" ] }, { "domain": "worldnic.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "google.cn", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "zendesk.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "stanford.edu", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "trueconf.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nominetdns.uk", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "walmart.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "calendly.com", "robots_status": 200, "llms_txt": true, "blocked": [ "ccbot" ], "names_ai_agents": [ "ccbot" ] }, { "domain": "nvidia.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "dropcatch.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "enacdn.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "samsungcloudsolution.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "plesk.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "ip-api.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cookiedatabase.org", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "selectel.ru", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "xboxlive.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ezvizlife.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amazon.de", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "indeed.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "cdnbuild.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "rocket-cdn.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amazontrust.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "live-video.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bluehost.com", "robots_status": null, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "tp-link.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "gitlab.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nasa.gov", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "kwai-pro.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "demdex.net", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "bsky.app", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "capcutapi.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "yandex.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ibyteimg.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "163.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "tiktokv.us", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "unsplash.com", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "claudebot" ], "names_ai_agents": [ "bytespider", "claudebot" ] }, { "domain": "aliexpress.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "vivoglobal.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "volcfcdndvs.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "kaspersky-labs.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "netangels.ru", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "heytapdl.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "awswaf.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dailymotion.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "line.me", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "tiktokv.eu", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "shiabank.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "casalemedia.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "globo.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "temu.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [ "bytespider", "chatgpt-user", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "huawei.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "tiktokcdn-eu.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amazonalexa.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "beian.gov.cn", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dnspod.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "pangle.io", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "brave.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "avcdn.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "taobao.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "steamcommunity.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "expireddomains.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nic.direct", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nflxvideo.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "kwaipros.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mynetname.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "businessinsider.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "bytespider", "ccbot", "claudebot" ], "names_ai_agents": [ "anthropic-ai", "bytespider", "ccbot", "claudebot" ] }, { "domain": "avsxappcaptiveportal.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "yandexcloud.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "online-metrix.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "vkontakte.ru", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "vkuser.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "gandi-ns.fr", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wp.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "espn.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "bytespider", "ccbot", "chatgpt-user", "google-extended", "gptbot" ], "names_ai_agents": [ "anthropic-ai", "bytespider", "ccbot", "chatgpt-user", "google-extended", "gptbot" ] }, { "domain": "sohu.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "bytespider" ] }, { "domain": "wbx2.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dell.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "bytespider" ] }, { "domain": "claude.ai", "robots_status": 200, "llms_txt": false, "blocked": [ "chatgpt-user", "google-extended", "gptbot", "oai-searchbot" ], "names_ai_agents": [ "chatgpt-user", "google-extended", "gptbot", "oai-searchbot" ] }, { "domain": "substack.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "intel.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "pixabay.com", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot" ], "names_ai_agents": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot" ] }, { "domain": "g.page", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ipinfo.io", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "yahoo.co.jp", "robots_status": 200, "llms_txt": false, "blocked": [ "google-extended", "gptbot" ], "names_ai_agents": [ "google-extended", "gptbot" ] }, { "domain": "netease.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ivi.ru", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "goodreads.com", "robots_status": 200, "llms_txt": false, "blocked": [ "ccbot", "gptbot" ], "names_ai_agents": [ "ccbot", "gptbot" ] }, { "domain": "go.com", "robots_status": 200, "llms_txt": false, "blocked": [ "google-extended", "gptbot" ], "names_ai_agents": [ "google-extended", "gptbot" ] }, { "domain": "sina.com.cn", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "speedtest.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cnbc.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "list-manage.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cloudns.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "spaceweb.pro", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "statista.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "perplexitybot" ] }, { "domain": "skyhigh.cloud", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mikrotik.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "att.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "dual-s-msedge.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "awsglobalaccelerator.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ipify.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "featureassets.org", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ietf.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "gamepass.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "eye4.cn", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nflximg.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "squarespacedns.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "kwai.net", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "firetvcaptiveportal.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "xcal.tv", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "addtoany.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "arxiv.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "edgecdn.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "paloaltonetworks.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "chatgpt-user", "oai-searchbot", "perplexitybot" ] }, { "domain": "adriver.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "sc-gw.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "quora.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "applebot-extended", "bytespider", "chatgpt-user", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "adobe.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "duolingo.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "npr.org", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "docker.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "timeweb.ru", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "myqcloud.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nease.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mysql.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ui-dns.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "chinamobile.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "eventbrite.com", "robots_status": 200, "llms_txt": false, "blocked": [ "ccbot" ], "names_ai_agents": [ "ccbot" ] }, { "domain": "stackoverflow.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amazon.co.jp", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "scribd.com", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "claude-web", "claudebot" ], "names_ai_agents": [ "bytespider", "ccbot", "claude-web", "claudebot" ] }, { "domain": "exp-tas.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ampproject.org", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ngenix.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "beyondwickedmapping.org", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "aol.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "perplexitybot" ] }, { "domain": "behance.net", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent" ] }, { "domain": "shein.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "tencent-cloud.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "gosuslugi.ru", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "foxnews.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "gcdn.co", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ipv4only.arpa", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amzn.to", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mediatek.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "markmonitor.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "indiatimes.com", "robots_status": 200, "llms_txt": false, "blocked": [ "meta-externalagent" ], "names_ai_agents": [ "meta-externalagent" ] }, { "domain": "fandom.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "namecheap.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "newrelic.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "google.de", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mcafee.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "sberbank.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "blogger.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cookielaw.org", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ] }, { "domain": "byteglb.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ft.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "meta-externalagent", "perplexitybot" ] }, { "domain": "google.com.hk", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "atlassian.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "myhuaweicloud.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bdydns.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "adobedtm.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "run.app", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "slideshare.net", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "claude-web", "claudebot" ], "names_ai_agents": [ "bytespider", "ccbot", "claude-web", "claudebot" ] }, { "domain": "discord.media", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "supertms.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "firefox.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "visualstudio.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dynatrace.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "wired.com", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "meta-externalagent", "perplexitybot" ] }, { "domain": "duckduckgo.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "impervadns.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "t-mobile.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "svc.ms", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "2gis.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "jquery.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "oup.com", "robots_status": 202, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "vidaahub.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "okta.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "trendmicro.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "capcut.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "applebot-extended", "ccbot", "chatgpt-user", "claudebot", "google-extended", "gptbot", "perplexitybot" ] }, { "domain": "elasticbeanstalk.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "patreon.com", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent", "perplexitybot" ] }, { "domain": "cloudflareinsights.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "hicloud.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "time.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "live.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "usatoday.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "tandfonline.com", "robots_status": 200, "llms_txt": false, "blocked": [ "gptbot" ], "names_ai_agents": [ "gptbot" ] }, { "domain": "e2ro.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cornell.edu", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nstld.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "fast.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "palmplaystore.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "supercell.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "tplinkcloud.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "noaa.gov", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "deviantart.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "tripadvisor.com", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "squarespace.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ] }, { "domain": "360yield.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "shopee.com.br", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "vercel-dns-016.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "uol.com.br", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "meta-externalagent", "perplexitybot" ] }, { "domain": "share.google", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "eset.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "berkeley.edu", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "aboutads.info", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "rakuten.co.jp", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cdn20.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "teamviewer.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "on.aws", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "azure-dns.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "grammarly.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "360.cn", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "telegraph.co.uk", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "3lift.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "qlivecdn.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "target.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "rlcdn.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "oraclecloud.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ca.gov", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ieee.org", "robots_status": 202, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dreamhost.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "hotjar.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "licdn.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "yahoodns.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "my.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "easebar.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "icloud-content.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "imgsmail.ru", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "avito.ru", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "adobedc.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "xiaomi.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amazon.co.za", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "samsungapps.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wix.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "mdpi.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "playstation.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "t-msedge.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "uk.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "webempresa.eu", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "threads.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "applebot-extended", "claudebot", "google-extended", "gptbot", "perplexitybot" ] }, { "domain": "digitalocean.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "surveymonkey.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "amazon.fr", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "msidentity.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "alrosa.ru", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "bytespider", "ccbot", "chatgpt-user", "gptbot" ], "names_ai_agents": [ "anthropic-ai", "bytespider", "ccbot", "chatgpt-user", "gptbot" ] }, { "domain": "accuweather.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "loc.gov", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "klaviyo.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "chatgpt-user", "claudebot", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "jotform.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "anthropic.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "unesco.org", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "shopifysvc.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "pushy.io", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "onetrust.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "ikea.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "alicdn.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "sagepub.com", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "meta-externalagent" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claudebot", "google-extended", "gptbot", "meta-externalagent" ] }, { "domain": "force.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dailymail.co.uk", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "sharethrough.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "claudebot", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "nintendo.com", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent" ] }, { "domain": "techcrunch.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot" ] }, { "domain": "bidswitch.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "atlassian.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "vedsalb.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "free.fr", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "googlezip.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "britannica.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "tencent.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amazon.ca", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "cdnvideo.ru", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bx-msedge.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dyndns.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "withgoogle.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "googletagservices.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "branch.io", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "ccbot", "claudebot", "google-extended", "gptbot", "perplexitybot" ] }, { "domain": "braze.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "yelp.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "redhat.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "ovh.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "elpais.com", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "claudebot", "perplexitybot" ], "names_ai_agents": [ "bytespider", "ccbot", "claudebot", "perplexitybot" ] }, { "domain": "ups.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "fontawesome.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "service.gov.uk", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cambridge.org", "robots_status": 200, "llms_txt": false, "blocked": [ "chatgpt-user" ], "names_ai_agents": [ "chatgpt-user" ] }, { "domain": "viber.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "stackadapt.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "nike.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cbsnews.com", "robots_status": 200, "llms_txt": false, "blocked": [ "gptbot" ], "names_ai_agents": [ "gptbot" ] }, { "domain": "imgur.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "pvp.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amazon.in", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "meta.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mobolize.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "intercom.io", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "hostgator.com.br", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "claudebot", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "cnet.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "ubi.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "outbrain.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "anydesk.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "twimg.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bytedns1.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "fb.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "applebot-extended", "claudebot", "google-extended", "gptbot", "perplexitybot" ] }, { "domain": "independent.co.uk", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wattpad.com", "robots_status": 200, "llms_txt": false, "blocked": [ "gptbot" ], "names_ai_agents": [ "gptbot" ] }, { "domain": "lemonde.fr", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended" ] }, { "domain": "dotomi.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "optimizely.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "g.co", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "xhamster.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mhverifier.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "pinimg.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "typekit.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wb.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "hihonorcloud.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "shop.app", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "bitrix24.ru", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "chatgpt-user", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "fifa.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dailymail.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "mailchimp.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "consultant.ru", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nel.goog", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "quantserve.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "vercel.app", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mega.co.nz", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "id5-sync.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "primevideo.com", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "chatgpt-user", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "meta-externalagent", "perplexitybot" ] }, { "domain": "ks-cdn.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "globalsign.com", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "dbankcloud.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "kick.com", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ] }, { "domain": "ya.ru", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "resolver.arpa", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "1rx.io", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "iso.org", "robots_status": 200, "llms_txt": false, "blocked": [ "ccbot", "chatgpt-user", "google-extended", "gptbot" ], "names_ai_agents": [ "ccbot", "chatgpt-user", "google-extended", "gptbot" ] }, { "domain": "telephony.goog", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "uber.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amazon.it", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "crwdcntrl.net", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "cmediahub.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bandcamp.com", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ] }, { "domain": "google.co.uk", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "itch.io", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "eeroup.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "creativecdn.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "seznam.cz", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wswebcdn.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "agkn.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "githubusercontent.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wps.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "ozone.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cursor.sh", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "ted.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "samsungacr.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "stripchat.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amazon.es", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "aiv-delivery.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mailinabox.email", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "vercel-dns-3.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "openstreetmap.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "footprintdns.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "allaboutcookies.org", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "att.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ring.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "chatgpt-user", "claudebot", "google-extended", "gptbot", "perplexitybot" ] }, { "domain": "mediafire.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "shopee.co.id", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "asus.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "linode.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "python.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "lijit.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "box.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "criteo.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "browser-intake-datadoghq.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bidr.io", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wpguardian.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mercadolivre.com.br", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "homedepot.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "magnit.ru", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wpguardian.io", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "perplexity.ai", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "sedo.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "twc.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mckinsey.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "hstgr.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "zillow.com", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "meta-externalagent" ], "names_ai_agents": [ "applebot-extended", "meta-externalagent" ] }, { "domain": "sophos.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "anthropic-ai", "chatgpt-user", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "marriott.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "blackhub.team", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "mlb.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "imrworldwide.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "queniuaa.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "curseforge.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "airbnb.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [ "anthropic-ai", "chatgpt-user", "claudebot", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "fda.gov", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "google.com.br", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "appcenter.ms", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bild.de", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "bytespider", "ccbot", "perplexitybot" ], "names_ai_agents": [ "applebot-extended", "bytespider", "ccbot", "perplexitybot" ] }, { "domain": "nikkei.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "steamstatic.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bamgrid.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "rt.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "networkadvertising.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ddnss.de", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bestbuy.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bugsnag.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "allegro.pl", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "safebrowsing.apple", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "adform.net", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "vercel-dns-017.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "abovedomains.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "adjust.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "singular.net", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "a-msedge.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "lenovo.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "comcast.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "nmrodam.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "gumgum.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "anthropic-ai", "chatgpt-user", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "orderbox-dns.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "apnews.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "ccbot", "claude-web", "claudebot", "gptbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "ccbot", "claude-web", "claudebot", "gptbot", "perplexitybot" ] }, { "domain": "fiverr.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "cdnhwc1.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "parklogic.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wp.pl", "robots_status": 200, "llms_txt": false, "blocked": [ "ccbot", "gptbot" ], "names_ai_agents": [ "ccbot", "chatgpt-user", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "ovscdns.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "translate.goog", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "liadm.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "teads.tv", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "prnewswire.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "epam.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ameblo.jp", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "alipaydns.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "welt.de", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "bytespider", "ccbot", "perplexitybot" ], "names_ai_agents": [ "applebot-extended", "bytespider", "ccbot", "perplexitybot" ] }, { "domain": "washington.edu", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "livejournal.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "people.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "academia.edu", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "applebot-extended", "chatgpt-user", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "rakuten.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "akamaitech.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "disneyplus.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "claude-web" ], "names_ai_agents": [ "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "bunnyinfra.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "it.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "verisign.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "a-mo.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "grammarly.io", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "moloco.com", "robots_status": 200, "llms_txt": true, "blocked": [ "bytespider" ], "names_ai_agents": [ "bytespider", "ccbot" ] }, { "domain": "amazon.com.au", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "mixpanel.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "theverge.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "perplexitybot" ] }, { "domain": "daum.net", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "shopeemobile.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "xbox.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "whecloud.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "erome.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "hbr.org", "robots_status": 200, "llms_txt": false, "blocked": [ "ccbot", "google-extended", "gptbot" ], "names_ai_agents": [ "ccbot", "google-extended", "gptbot" ] }, { "domain": "yieldmo.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "share-dns.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nbcnews.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "liftoff.io", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "zoho.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "33across.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "unpkg.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "blackberry.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dyingbirds.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "sc-cdn.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mybluehost.me", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "turn.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "discordapp.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "360safe.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "amazon.com.br", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "focus.de", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "recaptcha.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "appspot.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "lefigaro.fr", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "oai-searchbot" ], "names_ai_agents": [ "anthropic-ai", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "oai-searchbot" ] }, { "domain": "zoom.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "msftauth.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dbankcloud.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "usda.gov", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "myspace.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "me.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "no-ip.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "thenai.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "hilton.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "lgtvcommon.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "deepl.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ebay.de", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claudebot", "gptbot", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claudebot", "gptbot", "meta-externalagent", "perplexitybot" ] }, { "domain": "vivo.com.cn", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "mayoclinic.org", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "discogs.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "xvideos.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "agoda.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "pexels.com", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "perplexitybot" ], "names_ai_agents": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "perplexitybot" ] }, { "domain": "cdn-vk.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "onlyfans.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "princeton.edu", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "onetag-sys.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "frontiersin.org", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "disqus.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ebay.co.uk", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claudebot", "gptbot", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claudebot", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "yellowblue.io", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ssl-images-amazon.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "adgrx.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "arcgis.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "pornhub.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "figma.com", "robots_status": 200, "llms_txt": false, "blocked": [ "ccbot", "chatgpt-user", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "ccbot", "chatgpt-user", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "lg.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "chatgpt-user", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "fbsbx.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "webhostbox.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "azure-devices.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "chess.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "goskope.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "pikabu.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "qualtrics.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "cloudsink.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "synology.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "latimes.com", "robots_status": 200, "llms_txt": false, "blocked": [ "bytespider", "claudebot", "gptbot" ], "names_ai_agents": [ "bytespider", "claudebot", "gptbot" ] }, { "domain": "worldbank.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "spotifycdn.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ln-msedge.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "biblegateway.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "b-msedge.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wyzecam.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "www.gov.br", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "bankofamerica.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "zpath.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "fwmrm.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "herokuapp.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "usgovcloudapi.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ad-tech.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "t-s1-msedge.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "jetbrains.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "readthedocs.io", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "kleinanzeigen.de", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "scdn.co", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "sentinelone.net", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "tiktokw.us", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "eu-1-id5-sync.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "herokudns.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "go-mpulse.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "lwsdns.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "kunluncan.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "youku.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "samsungosp.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "miwifi.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dv.tech", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "deloitte.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "investopedia.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "lowes.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "state.gov", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "coupang.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "epa.gov", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "sf-cloud.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "eporner.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "life360.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [ "chatgpt-user", "claudebot", "google-extended", "gptbot", "perplexitybot" ] }, { "domain": "paramountplus.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ys7.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "seedtag.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "flashtalking.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "change.org", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "samsungiotcloud.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "porkbun.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "keenetic.io", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "smartthings.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "genius.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claudebot", "google-extended", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claudebot", "google-extended", "gptbot", "meta-externalagent" ] }, { "domain": "mercadolibre.com.ar", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "webmd.com", "robots_status": 200, "llms_txt": false, "blocked": [ "ccbot", "chatgpt-user", "claudebot", "gptbot" ], "names_ai_agents": [ "ccbot", "chatgpt-user", "claudebot", "gptbot" ] }, { "domain": "mailchi.mp", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "agora.io", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "docker.io", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "playrix.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "weforum.org", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ubnt.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [ "applebot-extended", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "chaturbate.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "character.ai", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nintendo.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "smaato.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "healthline.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claudebot", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claudebot", "gptbot", "meta-externalagent" ] }, { "domain": "internetwarriors.net", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "repubblica.it", "robots_status": 200, "llms_txt": true, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "google-extended", "gptbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "google-extended", "gptbot", "perplexitybot" ] }, { "domain": "fidelity.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "hm.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nationalgeographic.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "netlify.app", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "xnxx.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "jd.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "umich.edu", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "otto.de", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [ "chatgpt-user", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "note.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "leiniao.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "akahost.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "odoo.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "corriere.it", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ] }, { "domain": "wbbasket.ru", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "tsyndicate.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nexusmods.com", "robots_status": 200, "llms_txt": false, "blocked": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "applebot-extended", "bytespider", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ] }, { "domain": "tplinknbu.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "ancestry.com", "robots_status": 200, "llms_txt": false, "blocked": [ "google-extended" ], "names_ai_agents": [ "google-extended" ] }, { "domain": "tailscale.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "segment.io", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "eu.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "stbid.ru", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "xhamster46.desi", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "kwai.com", "robots_status": 200, "llms_txt": true, "blocked": [ "bytespider" ], "names_ai_agents": [ "bytespider" ] }, { "domain": "presage.io", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "wcdnga.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "4dex.io", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "opera-api.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "caixa.gov.br", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "volcgslb.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "theatlantic.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "claude-web", "claudebot", "google-extended", "meta-externalagent", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ] }, { "domain": "digitaloceanspaces.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "futbin.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "onet.pl", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "freepik.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "btloader.com", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "bytespider", "ccbot", "chatgpt-user", "claude-web", "claudebot", "google-extended", "gptbot", "meta-externalagent", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [] }, { "domain": "elmundo.es", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "ccbot", "chatgpt-user", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ], "names_ai_agents": [ "anthropic-ai", "ccbot", "chatgpt-user", "google-extended", "gptbot", "oai-searchbot", "perplexitybot" ] }, { "domain": "transsion-os.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "pixiv.net", "robots_status": 200, "llms_txt": false, "blocked": [ "anthropic-ai", "applebot-extended", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ], "names_ai_agents": [ "anthropic-ai", "applebot-extended", "ccbot", "claudebot", "google-extended", "gptbot", "meta-externalagent" ] }, { "domain": "themeforest.net", "robots_status": 200, "llms_txt": false, "blocked": [ "ccbot" ], "names_ai_agents": [ "ccbot" ] }, { "domain": "namecheaphosting.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "dnse0.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "faphouse.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "whitehouse.gov", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "sofascore.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "singtelchina.cn", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "kontur.ru", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "warnerbros.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "nextlgsdp.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "xhcdn.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "zenecn.net", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "binance.com", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "moe.video", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "pendo.io", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "costco.com", "robots_status": null, "llms_txt": false, "blocked": [], "names_ai_agents": [] }, { "domain": "onesignal.com", "robots_status": 200, "llms_txt": true, "blocked": [], "names_ai_agents": [] }, { "domain": "ox.ac.uk", "robots_status": 200, "llms_txt": false, "blocked": [], "names_ai_agents": [] } ] }