{ "method_version": 2, "started_at": "2026-09-20T06:23:41.644402+00:00", "finished_at": "2026-09-20T06:27:48.585147+00:00", "method_sha256": "68fcd577149efecd3f4b478db2ac850c9391718c886b151901ce98306fda6419", "sample": { "list_id": "PY96J", "permalink": "https://tranco-list.eu/list/PY96J", "download_url": "https://tranco-list.eu/download/PY96J/1000000", "archive_sha256": "560b701c3fdddb37ee84aa85cb13bc3f0a2d40a7b85d09eb8ed138f1c3d2eb13", "size": 1000 }, "user_agent": "SuedeCrawlerIndex/2.0 (+https://suedeai.ai/blog/ai-crawler-access-index)", "agents": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "anthropic-ai", "Claude-Web", "PerplexityBot", "CCBot", "Google-Extended", "Bytespider", "Applebot-Extended", "meta-externalagent" ], "core_agents": [ "CCBot", "ClaudeBot", "GPTBot", "Google-Extended", "PerplexityBot" ], "scope": "Declared robots.txt policy for URI path '/' only. llms.txt candidates pass a Markdown-shape and differing-control-response check. Neither metric proves indexing, citation, actual crawler access or vendor use.", "summary": { "sample_size": 1000, "robots_readable": 497, "robots_unclassified": 503, "root_disallowed": { "GPTBot": 103, "OAI-SearchBot": 58, "ChatGPT-User": 68, "ClaudeBot": 106, "anthropic-ai": 83, "Claude-Web": 83, "PerplexityBot": 84, "CCBot": 122, "Google-Extended": 94, "Bytespider": 124, "Applebot-Extended": 89, "meta-externalagent": 99 }, "any_core_root_disallowed": 138, "all_core_root_disallowed": 55, "llms_states": { "candidate": 78, "not_detected": 460, "unavailable": 453, "unknown_control": 9 } }, "results": [ { "rank": 1, "domain": "google.com", "robots": { "requested_url": "https://google.com/robots.txt", "observed_at": "2026-09-20T06:23:44.575817+00:00", "status": 200, "final_url": "https://www.google.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1aba570a1b32deb66941e0c29d4f37be71ee58aa1a624aa20754b3639538cf2c", "bytes_read": 6624 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://google.com/llms.txt", "observed_at": "2026-09-20T06:23:44.972904+00:00", "status": 404, "final_url": "https://google.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 2, "domain": "cloudflare.com", "robots": { "requested_url": "https://cloudflare.com/robots.txt", "observed_at": "2026-09-20T06:23:44.576275+00:00", "status": 200, "final_url": "https://www.cloudflare.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "c6e360cc6e931ea35c33ae917ff9e2194eb55f79f68e949ec54bcd4533324d98", "bytes_read": 1099 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://cloudflare.com/llms.txt", "observed_at": "2026-09-20T06:23:44.996931+00:00", "status": 200, "final_url": "https://www.cloudflare.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1f71c491c1bd85d2f9f63fc2e718ca0f2cb86993d299c9d26179a90e87758909", "bytes_read": 17176 }, "llms_state": "candidate", "control": { "requested_url": "https://cloudflare.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:45.356850+00:00", "status": 404, "final_url": "https://www.cloudflare.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "69967e8148f9451562c4ef70a9ba15c29cd14b13376b2595a71da547f9a03802", "bytes_read": 12 } }, { "rank": 3, "domain": "facebook.com", "robots": { "requested_url": "https://facebook.com/robots.txt", "observed_at": "2026-09-20T06:23:44.576676+00:00", "status": 200, "final_url": "https://facebook.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "6dc6e048d337f9f673706cfddc036c9b21d7a423fb5f515ef8f630522fa303e9", "bytes_read": 35223 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://facebook.com/llms.txt", "observed_at": "2026-09-20T06:23:44.900117+00:00", "status": 200, "final_url": "https://www.facebook.com/llms.txt", "content_type": "text/html; charset=\"utf-8\"", "truncated": false, "body_sha256": "7c5f2d9211f1c6d22c87c478ac94998a543ecf2f9d00cbc269134435d7f99694", "bytes_read": 321751 }, "llms_state": "not_detected", "control": null }, { "rank": 4, "domain": "gstatic.com", "robots": { "requested_url": "https://gstatic.com/robots.txt", "observed_at": "2026-09-20T06:23:44.577053+00:00", "status": 200, "final_url": "https://www.gstatic.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "20484aab9b6c0323b3e986bbf040af084dc39094a01a794d5daee09826ff0111", "bytes_read": 5078 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://gstatic.com/llms.txt", "observed_at": "2026-09-20T06:23:44.939905+00:00", "status": 404, "final_url": "https://www.gstatic.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 5, "domain": "googleapis.com", "robots": { "requested_url": "https://googleapis.com/robots.txt", "observed_at": "2026-09-20T06:23:44.577453+00:00", "status": 404, "final_url": "https://googleapis.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2", "bytes_read": 1571 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://googleapis.com/llms.txt", "observed_at": "2026-09-20T06:23:44.868198+00:00", "status": 404, "final_url": "https://googleapis.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 6, "domain": "microsoft.com", "robots": { "requested_url": "https://microsoft.com/robots.txt", "observed_at": "2026-09-20T06:23:44.577875+00:00", "status": 200, "final_url": "https://www.microsoft.com/robots.txt", "content_type": "text/plain;charset=iso-8859-1", "truncated": false, "body_sha256": "8d6eb902f7fb573f0430107afaae01cbe9a0a38923116065b322b6c8133b878b", "bytes_read": 8557 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://microsoft.com/llms.txt", "observed_at": "2026-09-20T06:23:45.344727+00:00", "status": 404, "final_url": "https://www.microsoft.com/llms.txt", "content_type": "text/html", "truncated": true, "body_sha256": "c5f32da34f3c0eb52e4c435a8b4fa64043e9b0bfa4034bcc0972acc47c50c160", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 7, "domain": "akamai.net", "robots": { "requested_url": "https://akamai.net/robots.txt", "observed_at": "2026-09-20T06:23:44.578295+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://akamai.net/llms.txt", "observed_at": "2026-09-20T06:23:44.623384+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 8, "domain": "amazonaws.com", "robots": { "requested_url": "https://amazonaws.com/robots.txt", "observed_at": "2026-09-20T06:23:44.578731+00:00", "status": 200, "final_url": "https://aws.amazon.com:443/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "c1f42540a70f0dcd98d98565c46f486ed49a464f077ef1f1362c5470b638ae98", "bytes_read": 13822 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://amazonaws.com/llms.txt", "observed_at": "2026-09-20T06:23:45.319329+00:00", "status": 404, "final_url": "https://aws.amazon.com:443/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "ea9ad1ef00de802fade8f824de470c151271090781c8c425a14e09bccab34302", "bytes_read": 322200 }, "llms_state": "not_detected", "control": null }, { "rank": 9, "domain": "youtube.com", "robots": { "requested_url": "https://youtube.com/robots.txt", "observed_at": "2026-09-20T06:23:44.579136+00:00", "status": 200, "final_url": "https://www.youtube.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "96f8640ca357626e7d9a894057fef8b5746eb838c29ed68fc176af58c9348c6f", "bytes_read": 792 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://youtube.com/llms.txt", "observed_at": "2026-09-20T06:23:44.958864+00:00", "status": 404, "final_url": "https://www.youtube.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "2a97e1c36e8fa6fc1104b0b66c9339fbc2c263d404012a2daacf28ad601e37ca", "bytes_read": 755 }, "llms_state": "not_detected", "control": null }, { "rank": 10, "domain": "apple.com", "robots": { "requested_url": "https://apple.com/robots.txt", "observed_at": "2026-09-20T06:23:44.579494+00:00", "status": 200, "final_url": "https://www.apple.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "3115f45c98cfa19341978dcab5dfbc0fd3e0ff4eb3ab1fe8192c3fbf7227dee7", "bytes_read": 1018 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://apple.com/llms.txt", "observed_at": "2026-09-20T06:23:45.012339+00:00", "status": 404, "final_url": "https://www.apple.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "322073dc0c17ba6a7de24d13b72dbd933ba657868921b60a2f07a20068fb6fed", "bytes_read": 111090 }, "llms_state": "not_detected", "control": null }, { "rank": 11, "domain": "instagram.com", "robots": { "requested_url": "https://instagram.com/robots.txt", "observed_at": "2026-09-20T06:23:44.579899+00:00", "status": 200, "final_url": "https://www.instagram.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "da3a05f8ae504bd8bf58d56604a98be3fd79c727e95c46bb35bba43d64b7071b", "bytes_read": 6258 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://instagram.com/llms.txt", "observed_at": "2026-09-20T06:23:45.313853+00:00", "status": 200, "final_url": "https://www.instagram.com/llms.txt", "content_type": "text/html; charset=\"utf-8\"", "truncated": true, "body_sha256": "3e9a2a0ce5ae15d7ef075c74e15f88d023401825a4817eeff2f1987f4f75496f", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 12, "domain": "mail.ru", "robots": { "requested_url": "https://mail.ru/robots.txt", "observed_at": "2026-09-20T06:23:44.580236+00:00", "status": 200, "final_url": "https://mail.ru/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "d3fd6768e203ab4008918459ce8d1ccc9e31d703cc01c9765902bcfc894822bf", "bytes_read": 433 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://mail.ru/llms.txt", "observed_at": "2026-09-20T06:23:45.383194+00:00", "status": 200, "final_url": "https://account.mail.ru/login?autologin_exp=v22&autologin_project=home&errorCode=11300&errorText=invalid+user&page=https%3A%2F%2Fmail.ru%2F%3Fautologin%3D1789885426&state=c4cc2460db8e42408ae426b051e31c8b", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "a94906248d1b0e0e9bc266640bf049d805503e892280729d247a35b5f6791d9d", "bytes_read": 936 }, "llms_state": "not_detected", "control": null }, { "rank": 13, "domain": "fbcdn.net", "robots": { "requested_url": "https://fbcdn.net/robots.txt", "observed_at": "2026-09-20T06:23:44.580669+00:00", "status": 200, "final_url": "https://fbcdn.net/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "8155e929427b274a20a74eca43333ed5d5fe3ba4eea4035aea704f57a72f0542", "bytes_read": 3355 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://fbcdn.net/llms.txt", "observed_at": "2026-09-20T06:23:44.837464+00:00", "status": 200, "final_url": "https://www.facebook.com/", "content_type": "text/html; charset=\"utf-8\"", "truncated": false, "body_sha256": "d95960c6957e95f2f756021b0ad59e922e0b39da53532a8bad348a04226e81a5", "bytes_read": 449662 }, "llms_state": "not_detected", "control": null }, { "rank": 14, "domain": "ezviz7.com", "robots": { "requested_url": "https://ezviz7.com/robots.txt", "observed_at": "2026-09-20T06:23:44.581064+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ezviz7.com/llms.txt", "observed_at": "2026-09-20T06:23:44.856884+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 15, "domain": "twitter.com", "robots": { "requested_url": "https://twitter.com/robots.txt", "observed_at": "2026-09-20T06:23:44.581457+00:00", "status": 200, "final_url": "https://twitter.com/robots.txt", "content_type": "text/plain;charset=utf-8", "truncated": false, "body_sha256": "14a72e33b3e9af07e1e8f5f645bfa00ed3fe9f9c87249d861d369d33d3f3ddc6", "bytes_read": 2678 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "*" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "*" ] ] } }, "llms": { "requested_url": "https://twitter.com/llms.txt", "observed_at": "2026-09-20T06:23:44.779856+00:00", "status": 200, "final_url": "https://x.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "d9f702cbd2d1114d99a4c5f517fb0457d5057a89a8eda118dcb0d33e8f43967f", "bytes_read": 298692 }, "llms_state": "not_detected", "control": null }, { "rank": 16, "domain": "dzen.ru", "robots": { "requested_url": "https://dzen.ru/robots.txt", "observed_at": "2026-09-20T06:23:44.582034+00:00", "status": 200, "final_url": "https://dzen.ru/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "ae3497058e161cf934aea68ee8b34ad8e3be391f5b493b7987459f7c6e939f27", "bytes_read": 9956 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://dzen.ru/llms.txt", "observed_at": "2026-09-20T06:23:45.260156+00:00", "status": 200, "final_url": "https://sso.passport.yandex.ru/push?uuid=35724a47-5f52-41d7-809c-a132d0d234fe&retpath=https%3A%2F%2Fdzen.ru%2Fllms.txt%3Fis_autologin_ya%3Dtrue", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "c6e53b3c2cbeab48c827e244d6177e8ef803ed2a106c1be5e0641e4686a16650", "bytes_read": 2894 }, "llms_state": "not_detected", "control": null }, { "rank": 17, "domain": "linkedin.com", "robots": { "requested_url": "https://linkedin.com/robots.txt", "observed_at": "2026-09-20T06:23:44.625382+00:00", "status": 200, "final_url": "https://linkedin.com/robots.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "b8ea3ef60e6e70d653b9a8789147428698f543052b7836409da4dc899e7fdbc0", "bytes_read": 21248 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://linkedin.com/llms.txt", "observed_at": "2026-09-20T06:23:44.922898+00:00", "status": 200, "final_url": "https://linkedin.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "febbff77242f485c809315ce1d2fe1726d175f2f7d3dcfce0786fe7c20417e8e", "bytes_read": 21240 }, "llms_state": "not_detected", "control": null }, { "rank": 18, "domain": "googlevideo.com", "robots": { "requested_url": "https://googlevideo.com/robots.txt", "observed_at": "2026-09-20T06:23:44.858189+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://googlevideo.com/llms.txt", "observed_at": "2026-09-20T06:23:45.032888+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 19, "domain": "domaincontrol.com", "robots": { "requested_url": "https://domaincontrol.com/robots.txt", "observed_at": "2026-09-20T06:23:45.097490+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://domaincontrol.com/llms.txt", "observed_at": "2026-09-20T06:23:45.208638+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 20, "domain": "googletagmanager.com", "robots": { "requested_url": "https://googletagmanager.com/robots.txt", "observed_at": "2026-09-20T06:23:45.182943+00:00", "status": 404, "final_url": "https://googletagmanager.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2", "bytes_read": 1571 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://googletagmanager.com/llms.txt", "observed_at": "2026-09-20T06:23:45.437900+00:00", "status": 404, "final_url": "https://googletagmanager.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 21, "domain": "office.com", "robots": { "requested_url": "https://office.com/robots.txt", "observed_at": "2026-09-20T06:23:45.189097+00:00", "status": 200, "final_url": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=4765445b-32c6-49b0-83e6-1d93765276ca&redirect_uri=https%3A%2F%2Fwww.office.com%2Flandingv2&response_type=code%20id_token&scope=openid%20profile%20https%3A%2F%2Fwww.office.com%2Fv2%2FOfficeHome.All&response_mode=form_post&nonce=639254822260649384.OTM3ZjY3OWMtNDNkNC00ZGRhLWFhMjAtMjc0OTE1YzlkZWNjY2YyN2RiYzUtODdlNC00YTU1LThiYmUtNWExODBhMzQ3YzEy&ui_locales=en-US&mkt=en-US&client-request-id=74534dac-58de-4619-8021-7bf7583fb980&siwa=1&siwg=1&state=YrrAFBIMFnoyuMrVwC_R6X7MhoRXekco3lLgLjcAYrS6GKE1TbIUwSbvf-IHY14azFtGaMCBvrLYhhJmirXJ86mZmYS1ArQJekalSPsFfAnnaNMWIFRmah_SpZygpfsV07ChFsJCH50mrc7d5NEx54TyhOtnzVp-wZQPPcY4rvXop1gOF9_OV-rDPC0F967Wyp99AKb0Q636E_i2PaZjo9d56xK8eoeceM0p6cMg6tK_r83CcocmJY0JGh9Qj55aw33ZENBhQJ-_uazXPqLpSPmFqOBWwWDDCj4okPiJANDjoZd8dqk6kUh1j4ILFn5o6PjB74zAUJrpwOgUD_fUTXp1KFjJzB_JrjP0wMPpjLQ6vAFWg6-bCqMmWpAEGyjCNc69ntu7nMFj2WHsIvlO57IrV14MD0rn5OoqXnViyuYTmFGxTVhwRWMEP7jBSxsR4LRxQsQk9lQx8fGHujOwapWHATVITEFmnzetzO6D4MQOTlZ5biiQHxW3mZukUkfraq0XsXC-hn23nUovUZ_M2Oyu1C53frjMJA4q-XrQoTg&x-client-SKU=ID_NET8_0&x-client-ver=8.16.0.0", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "376dc4b8ef86624c2a9331b2d4128e18fc44ff04cc7989d31c3f47b2910070d1", "bytes_read": 59567 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://office.com/llms.txt", "observed_at": "2026-09-20T06:23:46.623290+00:00", "status": 200, "final_url": "https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id=4765445b-32c6-49b0-83e6-1d93765276ca&redirect_uri=https%3A%2F%2Fwww.office.com%2Flandingv2&response_type=code%20id_token&scope=openid%20profile%20https%3A%2F%2Fwww.office.com%2Fv2%2FOfficeHome.All&response_mode=form_post&nonce=639254822270996428.Nzg3OGFmMjEtM2Y4YS00ZjViLWEyMjctZjgwNTIwMTdkZjdkMjgzNjViNzktNWZlNS00YTgzLThkZDgtYzVmNzUyMjBlY2Zl&ui_locales=en-US&mkt=en-US&client-request-id=b73396e5-644d-4bcd-83c5-fe503e76e876&siwa=1&siwg=1&state=0srhaSa3dZWZuvPcETBGLqwCJhHAhT2ZJ9X1m7xLw-B04BxMIOzqPd5e0-clxmVhG-aNCA-SYroQLRo2sHGD2SbaPooTboWo06Xb1vaR-xfy5KyRA4sqnMzJbKnXqXm2N4DVCc_x6_lgQGWhAfr-dLGTaGoVhRLNhYs47Yd-hdJF447cIYy1BRBaHaO8tH4bt5Kmmvhyn5A7V1Z7QJD07_LDxRdtYzhHsapk06Kh5fbAMqZh9eG9v5R_h_fmJTMe3SN-ZEKE3nxt65moGxEm_REWlg1-ghQ5VbTAyfWXo5ERJw7_qwt9sum15SYFpi6rTGVl_odhwOlzqs3uSOmmww7mdXz8JSCJVpkdD6nNZBL9iaSdHwdNSXXTr5iNcfaerR-xOc51qx7gFjD0gsKkVjNGzV3gOd1oL_RY0ikoODnxKDk9SiZpntDIHgxZFuvOo5sOg44Eh3Uuu2PjoMY6vZ0ClupTzT9yLGLb2AdU0eZrrG1xeRBE-lnRMrHJx1TV0QoNbTpaSZFvd6B1qARha9JU7uwkvaVuX_Y-u7JoKM0&x-client-SKU=ID_NET8_0&x-client-ver=8.16.0.0", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "c22092b3b192d6f284835f9785433b85e04d1050214abefe86df408335b20802", "bytes_read": 59513 }, "llms_state": "not_detected", "control": null }, { "rank": 22, "domain": "live.com", "robots": { "requested_url": "https://live.com/robots.txt", "observed_at": "2026-09-20T06:23:45.210449+00:00", "status": 417, "final_url": "https://outlook.live.com/mail/", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://live.com/llms.txt", "observed_at": "2026-09-20T06:23:46.381919+00:00", "status": 417, "final_url": "https://outlook.live.com/mail/", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "unavailable", "control": null }, { "rank": 23, "domain": "akamaiedge.net", "robots": { "requested_url": "https://akamaiedge.net/robots.txt", "observed_at": "2026-09-20T06:23:45.266799+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://akamaiedge.net/llms.txt", "observed_at": "2026-09-20T06:23:45.332785+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 24, "domain": "amazon.com", "robots": { "requested_url": "https://amazon.com/robots.txt", "observed_at": "2026-09-20T06:23:45.320943+00:00", "status": 200, "final_url": "https://www.amazon.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "f62cfb6484ed7cc23380174e8cdeff3603d8815b1e022ddd4117460dd2408ba9", "bytes_read": 7887 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://amazon.com/llms.txt", "observed_at": "2026-09-20T06:23:46.016099+00:00", "status": 404, "final_url": "https://www.amazon.com/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "abeb2137f14f1973cb5949a7ea52daf3d9cb22e85cc8d8abd7e0bbff114f1048", "bytes_read": 2319 }, "llms_state": "not_detected", "control": null }, { "rank": 25, "domain": "gtld-servers.net", "robots": { "requested_url": "https://gtld-servers.net/robots.txt", "observed_at": "2026-09-20T06:23:45.338780+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://gtld-servers.net/llms.txt", "observed_at": "2026-09-20T06:23:45.416631+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 26, "domain": "akadns.net", "robots": { "requested_url": "https://akadns.net/robots.txt", "observed_at": "2026-09-20T06:23:45.366584+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://akadns.net/llms.txt", "observed_at": "2026-09-20T06:23:45.416913+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 27, "domain": "hicloudcam.com", "robots": { "requested_url": "https://hicloudcam.com/robots.txt", "observed_at": "2026-09-20T06:23:45.419883+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://hicloudcam.com/llms.txt", "observed_at": "2026-09-20T06:23:45.737761+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 28, "domain": "azure.com", "robots": { "requested_url": "https://azure.com/robots.txt", "observed_at": "2026-09-20T06:23:45.420246+00:00", "status": 200, "final_url": "https://azure.microsoft.com/en-us", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "83119a30d60bf2afddd32b9eafc188da341afa6bad92016751f391c98f46b260", "bytes_read": 463947 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://azure.com/llms.txt", "observed_at": "2026-09-20T06:23:47.144123+00:00", "status": 404, "final_url": "https://azure.microsoft.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "fad1f2074a37f9d7f78f71b054b2923a9f3064638a4b96c5d69383655f13c667", "bytes_read": 92534 }, "llms_state": "not_detected", "control": null }, { "rank": 29, "domain": "github.com", "robots": { "requested_url": "https://github.com/robots.txt", "observed_at": "2026-09-20T06:23:45.455354+00:00", "status": 200, "final_url": "https://github.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "384a9642ae240a8f6fb20a161ce568587d30e15650cf7fa1c293c26987aa203c", "bytes_read": 6397 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://github.com/llms.txt", "observed_at": "2026-09-20T06:23:45.745237+00:00", "status": 200, "final_url": "https://github.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "71d217ec3186152052ca1050932c0e6e48e766d3bc5a1359b50aa140436d297f", "bytes_read": 28658 }, "llms_state": "candidate", "control": { "requested_url": "https://github.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:46.057576+00:00", "status": 404, "final_url": "https://github.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5", "bytes_read": 9 } }, { "rank": 30, "domain": "wikipedia.org", "robots": { "requested_url": "https://wikipedia.org/robots.txt", "observed_at": "2026-09-20T06:23:45.555049+00:00", "status": 200, "final_url": "https://en.wikipedia.org/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "48b98d80c8444d37cdf50cc9a4d2257b66039477e581066d2356062a4cbd678f", "bytes_read": 28275 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://wikipedia.org/llms.txt", "observed_at": "2026-09-20T06:23:46.564221+00:00", "status": 404, "final_url": "https://en.wikipedia.org/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "76563b4eeb07e461f97c018d6f0744fdaf2b0af79ebd90410e0452733d164bdf", "bytes_read": 1868 }, "llms_state": "not_detected", "control": null }, { "rank": 31, "domain": "bing.com", "robots": { "requested_url": "https://bing.com/robots.txt", "observed_at": "2026-09-20T06:23:45.558804+00:00", "status": 200, "final_url": "https://www.bing.com:443/robots.txt?toWww=1&redig=07992974B65E4AED8DB0F240EF529E12", "content_type": "text/plain", "truncated": false, "body_sha256": "40d2da70c3b443225d5b01cc7b05117944980347637cc95ab285c6e4f5548235", "bytes_read": 2676 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://bing.com/llms.txt", "observed_at": "2026-09-20T06:23:46.453445+00:00", "status": 404, "final_url": "https://www.bing.com:443/llms.txt?toWww=1&redig=DFEFE667AAB24894B8EEDEC9E1313AFA", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "not_detected", "control": null }, { "rank": 32, "domain": "whatsapp.net", "robots": { "requested_url": "https://whatsapp.net/robots.txt", "observed_at": "2026-09-20T06:23:45.687759+00:00", "status": 200, "final_url": "https://www.whatsapp.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "b0dc29a03f811635b16c416f0c890b4acad4112533b78feed9bdb13de7156cf9", "bytes_read": 1180 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://whatsapp.net/llms.txt", "observed_at": "2026-09-20T06:23:46.212458+00:00", "status": 404, "final_url": "https://www.whatsapp.com/llms.txt", "content_type": "text/html; charset=\"utf-8\"", "truncated": false, "body_sha256": "7974bf34fdf420f7dc3506649e209735630f7809267af789a014a5932a48b239", "bytes_read": 257228 }, "llms_state": "not_detected", "control": null }, { "rank": 33, "domain": "apple-dns.net", "robots": { "requested_url": "https://apple-dns.net/robots.txt", "observed_at": "2026-09-20T06:23:45.745662+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://apple-dns.net/llms.txt", "observed_at": "2026-09-20T06:23:45.802264+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 34, "domain": "appsflyersdk.com", "robots": { "requested_url": "https://appsflyersdk.com/robots.txt", "observed_at": "2026-09-20T06:23:45.747859+00:00", "status": 403, "final_url": "https://appsflyersdk.com/robots.txt", "content_type": "text/xml", "truncated": false, "body_sha256": "95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83", "bytes_read": 146 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://appsflyersdk.com/llms.txt", "observed_at": "2026-09-20T06:23:46.018859+00:00", "status": 403, "final_url": "https://appsflyersdk.com/llms.txt", "content_type": "text/xml", "truncated": false, "body_sha256": "95769eb326e23b8cd1b7a9b82d884a713a9724f49b2794ac5b967d8cd706de83", "bytes_read": 146 }, "llms_state": "unavailable", "control": null }, { "rank": 35, "domain": "googleusercontent.com", "robots": { "requested_url": "https://googleusercontent.com/robots.txt", "observed_at": "2026-09-20T06:23:45.803665+00:00", "status": 404, "final_url": "https://googleusercontent.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2", "bytes_read": 1571 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://googleusercontent.com/llms.txt", "observed_at": "2026-09-20T06:23:46.019542+00:00", "status": 404, "final_url": "https://googleusercontent.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 36, "domain": "fastly.net", "robots": { "requested_url": "https://fastly.net/robots.txt", "observed_at": "2026-09-20T06:23:45.830801+00:00", "status": 200, "final_url": "https://www.fastly.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "23afd0d2868ab27736e15dd91bfa2b061fff83da62d60d131f335d21311dc6f3", "bytes_read": 398 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://fastly.net/llms.txt", "observed_at": "2026-09-20T06:23:46.311396+00:00", "status": 200, "final_url": "https://www.fastly.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "14d1f65e9da8401a0eb1c14fee4867dcb13db77753447a18f6febfff4074cd0e", "bytes_read": 15771 }, "llms_state": "candidate", "control": { "requested_url": "https://fastly.net/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:46.655671+00:00", "status": 404, "final_url": "https://www.fastly.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "c31138d8d37d4c99fdd41b009a93658d2c0d5ddca0b1a1669504a172106f09b6", "bytes_read": 231785 } }, { "rank": 37, "domain": "doubleclick.net", "robots": { "requested_url": "https://doubleclick.net/robots.txt", "observed_at": "2026-09-20T06:23:45.846299+00:00", "status": 200, "final_url": "https://doubleclick.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1aba570a1b32deb66941e0c29d4f37be71ee58aa1a624aa20754b3639538cf2c", "bytes_read": 6624 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://doubleclick.net/llms.txt", "observed_at": "2026-09-20T06:23:46.082367+00:00", "status": 404, "final_url": "https://doubleclick.net/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 38, "domain": "aaplimg.com", "robots": { "requested_url": "https://aaplimg.com/robots.txt", "observed_at": "2026-09-20T06:23:45.893913+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://aaplimg.com/llms.txt", "observed_at": "2026-09-20T06:23:45.942047+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 39, "domain": "microsoftonline.com", "robots": { "requested_url": "https://microsoftonline.com/robots.txt", "observed_at": "2026-09-20T06:23:45.943820+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://microsoftonline.com/llms.txt", "observed_at": "2026-09-20T06:23:45.994417+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 40, "domain": "office.net", "robots": { "requested_url": "https://office.net/robots.txt", "observed_at": "2026-09-20T06:23:46.000993+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://office.net/llms.txt", "observed_at": "2026-09-20T06:23:46.038833+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 41, "domain": "netflix.com", "robots": { "requested_url": "https://netflix.com/robots.txt", "observed_at": "2026-09-20T06:23:46.044952+00:00", "status": 200, "final_url": "https://www.netflix.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1e14fa0284e9757b52b48dde38b390effae68afdd72447088e72f18f49b1aa09", "bytes_read": 3790 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://netflix.com/llms.txt", "observed_at": "2026-09-20T06:23:46.577829+00:00", "status": 200, "final_url": "https://www.netflix.com/NotFound?prev=https%3A%2F%2Fwww.netflix.com%2Fllms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "6b9cee31f0d4e60ccc1e1cbd3bc6faff13e7ccb75a6932a6b6c048cf5b61c229", "bytes_read": 23137 }, "llms_state": "not_detected", "control": null }, { "rank": 42, "domain": "trafficmanager.net", "robots": { "requested_url": "https://trafficmanager.net/robots.txt", "observed_at": "2026-09-20T06:23:46.205723+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://trafficmanager.net/llms.txt", "observed_at": "2026-09-20T06:23:46.261382+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 43, "domain": "cloud.microsoft", "robots": { "requested_url": "https://cloud.microsoft/robots.txt", "observed_at": "2026-09-20T06:23:46.208799+00:00", "status": 200, "final_url": "https://m365.cloud.microsoft/", "content_type": "text/html", "truncated": false, "body_sha256": "3f2f5dc8fcb6d3aa1ee1fc853f68c0c8b9a2c2c9d350127120471681520628bb", "bytes_read": 79 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cloud.microsoft/llms.txt", "observed_at": "2026-09-20T06:23:46.906127+00:00", "status": 200, "final_url": "https://m365.cloud.microsoft/", "content_type": "text/html", "truncated": false, "body_sha256": "3f2f5dc8fcb6d3aa1ee1fc853f68c0c8b9a2c2c9d350127120471681520628bb", "bytes_read": 79 }, "llms_state": "not_detected", "control": null }, { "rank": 44, "domain": "gandi.net", "robots": { "requested_url": "https://gandi.net/robots.txt", "observed_at": "2026-09-20T06:23:46.264553+00:00", "status": 200, "final_url": "https://www.gandi.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "3fd451a6143f6a84977d65af0ae1e26a153466423e8d7e08e672f3d1466e1cdc", "bytes_read": 9023 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://gandi.net/llms.txt", "observed_at": "2026-09-20T06:23:47.511244+00:00", "status": 404, "final_url": "https://www.gandi.net/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "65bd387183652f66562a17e458a200df2aa477360cfa5c1e7325c4d80c911c66", "bytes_read": 118057 }, "llms_state": "not_detected", "control": null }, { "rank": 45, "domain": "sharepoint.com", "robots": { "requested_url": "https://sharepoint.com/robots.txt", "observed_at": "2026-09-20T06:23:46.278179+00:00", "status": null, "error": "TimeoutError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://sharepoint.com/llms.txt", "observed_at": "2026-09-20T06:23:58.266083+00:00", "status": null, "error": "TimeoutError" }, "llms_state": "unavailable", "control": null }, { "rank": 46, "domain": "youtu.be", "robots": { "requested_url": "https://youtu.be/robots.txt", "observed_at": "2026-09-20T06:23:46.314079+00:00", "status": 200, "final_url": "https://youtu.be/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "c290485c8002c51c24c95765948dbcec9ec88c1f181638d245418a36194fd1a0", "bytes_read": 511 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://youtu.be/llms.txt", "observed_at": "2026-09-20T06:23:46.561505+00:00", "status": 200, "final_url": "https://www.youtube.com/watch?v=llms.txt&feature=youtu.be", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "1dbb51f4eeb45692ce6eff20fd8b21db7f53bc9b6e9d2a989fc3a8af22e708e9", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 47, "domain": "digicert.com", "robots": { "requested_url": "https://digicert.com/robots.txt", "observed_at": "2026-09-20T06:23:46.372078+00:00", "status": 200, "final_url": "https://www.digicert.com/robots.txt", "content_type": "text/plain; charset=ISO-8859-1", "truncated": false, "body_sha256": "a139a54387d10bfd57f5223202cb56558934677d20460b789d4445d2549537f7", "bytes_read": 2657 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://digicert.com/llms.txt", "observed_at": "2026-09-20T06:23:47.186535+00:00", "status": 200, "final_url": "https://www.digicert.com/llms.txt", "content_type": "text/plain; charset=ISO-8859-1", "truncated": false, "body_sha256": "02169ec84a351a1d16d7d35ae9186f3edaa67dc827c2a7b55f9cea8f32abd439", "bytes_read": 8395 }, "llms_state": "candidate", "control": { "requested_url": "https://digicert.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:47.845973+00:00", "status": 404, "final_url": "https://www.digicert.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "abfcdb44fc46fb808a5780430dd23457aef7a83e1ee15d88c5b26cf790b72b51", "bytes_read": 99972 } }, { "rank": 48, "domain": "wordpress.org", "robots": { "requested_url": "https://wordpress.org/robots.txt", "observed_at": "2026-09-20T06:23:46.396049+00:00", "status": 200, "final_url": "https://wordpress.org/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "65185a701db3ab195bcb2f86657300e13153f758af110ac0ffeeea5dc19ab5dc", "bytes_read": 976 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://wordpress.org/llms.txt", "observed_at": "2026-09-20T06:23:46.700796+00:00", "status": 200, "final_url": "https://wordpress.org/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "731a4c979c79b5a350375f8b089fc26ed70b20d395de39ad305b716de233b4af", "bytes_read": 5765 }, "llms_state": "candidate", "control": { "requested_url": "https://wordpress.org/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:46.984647+00:00", "status": 404, "final_url": "https://wordpress.org/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "a0367ba9d0b37f006486da66b1680d50b9f39bbd04f6fbbda10cee5a476c6027", "bytes_read": 119228 } }, { "rank": 49, "domain": "skype.com", "robots": { "requested_url": "https://skype.com/robots.txt", "observed_at": "2026-09-20T06:23:46.551948+00:00", "status": 200, "final_url": "https://teams.live.com/free?source=skype", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "9857e5ea471fa1efe61f1d63fa6d0d4a56f07e85f8b8772a1e2d9079e57437be", "bytes_read": 61988 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://skype.com/llms.txt", "observed_at": "2026-09-20T06:23:47.804018+00:00", "status": 200, "final_url": "https://teams.live.com/free?source=skype", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "98803837253fcbc551e8ad25a9f69c319f9abecd11edacccd887c8459b005deb", "bytes_read": 61988 }, "llms_state": "not_detected", "control": null }, { "rank": 50, "domain": "x.com", "robots": { "requested_url": "https://x.com/robots.txt", "observed_at": "2026-09-20T06:23:46.909207+00:00", "status": 200, "final_url": "https://x.com/robots.txt", "content_type": "text/plain;charset=utf-8", "truncated": false, "body_sha256": "14a72e33b3e9af07e1e8f5f645bfa00ed3fe9f9c87249d861d369d33d3f3ddc6", "bytes_read": 2678 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "*" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "*" ] ] } }, "llms": { "requested_url": "https://x.com/llms.txt", "observed_at": "2026-09-20T06:23:47.094794+00:00", "status": 200, "final_url": "https://x.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "c16d7805d706f2f43ca681b8a04eb677a2c42263ff8bf848185aada249e40db3", "bytes_read": 298692 }, "llms_state": "not_detected", "control": null }, { "rank": 51, "domain": "icloud.com", "robots": { "requested_url": "https://icloud.com/robots.txt", "observed_at": "2026-09-20T06:23:47.016164+00:00", "status": 200, "final_url": "https://www.icloud.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "8590137a4eb5596205d92c30ce39ca340635e6b98a4666c534cb0aeea21d6602", "bytes_read": 1270 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://icloud.com/llms.txt", "observed_at": "2026-09-20T06:23:47.585232+00:00", "status": 404, "final_url": "https://www.icloud.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "14f5caee6ae566b8b12784aa24f2b700166484c32017546ac9458de5a518e4eb", "bytes_read": 3250 }, "llms_state": "not_detected", "control": null }, { "rank": 52, "domain": "cloudfront.net", "robots": { "requested_url": "https://cloudfront.net/robots.txt", "observed_at": "2026-09-20T06:23:47.206090+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cloudfront.net/llms.txt", "observed_at": "2026-09-20T06:23:47.321453+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 53, "domain": "goo.gl", "robots": { "requested_url": "https://goo.gl/robots.txt", "observed_at": "2026-09-20T06:23:47.225822+00:00", "status": 400, "final_url": "https://goo.gl/robots.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "bae5d66fbe371f66a934935346906d25bb35df6fb4cd5504a359cd324e89ea11", "bytes_read": 9258 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://goo.gl/llms.txt", "observed_at": "2026-09-20T06:23:47.727898+00:00", "status": 400, "final_url": "https://goo.gl/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "bae5d66fbe371f66a934935346906d25bb35df6fb4cd5504a359cd324e89ea11", "bytes_read": 9258 }, "llms_state": "unavailable", "control": null }, { "rank": 54, "domain": "tiktok.com", "robots": { "requested_url": "https://tiktok.com/robots.txt", "observed_at": "2026-09-20T06:23:47.315596+00:00", "status": 200, "final_url": "https://www.tiktok.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "835705a2ecb862f49a813b3aa4061e0edfefbb8318316b371f8fa2dba4018802", "bytes_read": 1567 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://tiktok.com/llms.txt", "observed_at": "2026-09-20T06:23:48.035718+00:00", "status": 200, "final_url": "https://www.tiktok.com/404?fromUrl=/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "cadf202595877bce74fd828ca51b19ab95bceb8f67d06a86a55cd0c1c9c60015", "bytes_read": 32982 }, "llms_state": "not_detected", "control": null }, { "rank": 55, "domain": "whatsapp.com", "robots": { "requested_url": "https://whatsapp.com/robots.txt", "observed_at": "2026-09-20T06:23:47.323270+00:00", "status": 200, "final_url": "https://www.whatsapp.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "b0dc29a03f811635b16c416f0c890b4acad4112533b78feed9bdb13de7156cf9", "bytes_read": 1180 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://whatsapp.com/llms.txt", "observed_at": "2026-09-20T06:23:47.737082+00:00", "status": 404, "final_url": "https://www.whatsapp.com/llms.txt", "content_type": "text/html; charset=\"utf-8\"", "truncated": false, "body_sha256": "d6344e411ae26740503897f53a59857aadf696747b61b1d498dad8a90efc7805", "bytes_read": 257256 }, "llms_state": "not_detected", "control": null }, { "rank": 56, "domain": "pinterest.com", "robots": { "requested_url": "https://pinterest.com/robots.txt", "observed_at": "2026-09-20T06:23:47.396397+00:00", "status": 200, "final_url": "https://www.pinterest.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "d6b2d2e5111deab042626a98141062225d75bafeaaec7d8dacfbb1f4021b60af", "bytes_read": 30906 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://pinterest.com/llms.txt", "observed_at": "2026-09-20T06:23:48.074572+00:00", "status": 200, "final_url": "https://www.pinterest.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "39951cf7e72631b6f53617cd8eb378efb22959726f155de82fd32a192f6d334a", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 57, "domain": "cloudflare.net", "robots": { "requested_url": "https://cloudflare.net/robots.txt", "observed_at": "2026-09-20T06:23:47.476513+00:00", "status": 200, "final_url": "https://cloudflare.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "de73d32142f114b8d18bbfd7fa5779ccee5f7a1753873402cfdab59444834ccd", "bytes_read": 207 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://cloudflare.net/llms.txt", "observed_at": "2026-09-20T06:23:47.784500+00:00", "status": 404, "final_url": "https://cloudflare.net/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "049e1b638af15c4f3525aed1b4be5d3fd6fe361d999601cbc978dfb34a8c3285", "bytes_read": 1286 }, "llms_state": "not_detected", "control": null }, { "rank": 58, "domain": "googledomains.com", "robots": { "requested_url": "https://googledomains.com/robots.txt", "observed_at": "2026-09-20T06:23:47.516380+00:00", "status": 200, "final_url": "https://domains.google/", "content_type": "text/html", "truncated": false, "body_sha256": "c28e4c26e9e4dadd661ce45ccdf3ecccf4344508eee36523bae5ad1d54645281", "bytes_read": 5354 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://googledomains.com/llms.txt", "observed_at": "2026-09-20T06:23:48.025841+00:00", "status": 200, "final_url": "https://domains.google/", "content_type": "text/html", "truncated": false, "body_sha256": "e90b599b5ca2a469763b5cf8b8d70411b493415fa05015a577f064ca387cc507", "bytes_read": 5354 }, "llms_state": "not_detected", "control": null }, { "rank": 59, "domain": "yahoo.com", "robots": { "requested_url": "https://yahoo.com/robots.txt", "observed_at": "2026-09-20T06:23:47.588017+00:00", "status": 200, "final_url": "https://www.yahoo.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "021adb45d8aa004e67dcc639527ab5745ba0fb28587c16c5cd4aea8b5eb886c9", "bytes_read": 1624 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://yahoo.com/llms.txt", "observed_at": "2026-09-20T06:23:48.139094+00:00", "status": 404, "final_url": "https://www.yahoo.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "4785cd0ed51dba3f9ceb0a89d5299ba6f46b9f2dc6d4c673ed2467d53b5c1da4", "bytes_read": 4591 }, "llms_state": "not_detected", "control": null }, { "rank": 60, "domain": "roblox.com", "robots": { "requested_url": "https://roblox.com/robots.txt", "observed_at": "2026-09-20T06:23:47.608502+00:00", "status": 200, "final_url": "https://www.roblox.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "a4a9450540cd8263a373527026c92df49b7289076e87a3bb5bb72a7c28bc5213", "bytes_read": 2181 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://roblox.com/llms.txt", "observed_at": "2026-09-20T06:23:48.341156+00:00", "status": 404, "final_url": "https://www.roblox.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "fe750dbcde716fe6ff28c139e520bd52c400ba2ed1891a93e5fc97b4408b42d2", "bytes_read": 67118 }, "llms_state": "not_detected", "control": null }, { "rank": 61, "domain": "googlesyndication.com", "robots": { "requested_url": "https://googlesyndication.com/robots.txt", "observed_at": "2026-09-20T06:23:47.610662+00:00", "status": 404, "final_url": "https://googlesyndication.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2", "bytes_read": 1571 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://googlesyndication.com/llms.txt", "observed_at": "2026-09-20T06:23:47.869323+00:00", "status": 404, "final_url": "https://googlesyndication.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 62, "domain": "gvt1.com", "robots": { "requested_url": "https://gvt1.com/robots.txt", "observed_at": "2026-09-20T06:23:47.830456+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://gvt1.com/llms.txt", "observed_at": "2026-09-20T06:23:47.877382+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 63, "domain": "pv-cdn.net", "robots": { "requested_url": "https://pv-cdn.net/robots.txt", "observed_at": "2026-09-20T06:23:47.879604+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://pv-cdn.net/llms.txt", "observed_at": "2026-09-20T06:23:47.914268+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 64, "domain": "spotify.com", "robots": { "requested_url": "https://spotify.com/robots.txt", "observed_at": "2026-09-20T06:23:47.916923+00:00", "status": 200, "final_url": "https://www.spotify.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "5c97777a4a9a3a8117e99acd1e557b07a4ddb40296648754fac99c36c8c594f9", "bytes_read": 592 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://spotify.com/llms.txt", "observed_at": "2026-09-20T06:23:48.355833+00:00", "status": 200, "final_url": "https://www.spotify.com/us/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "13c5b3dc801d584a4570a0964830d2876cd119d77626b6870245fc975e67ed32", "bytes_read": 50523 }, "llms_state": "not_detected", "control": null }, { "rank": 65, "domain": "windows.net", "robots": { "requested_url": "https://windows.net/robots.txt", "observed_at": "2026-09-20T06:23:47.978919+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://windows.net/llms.txt", "observed_at": "2026-09-20T06:23:48.217683+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 66, "domain": "wa.me", "robots": { "requested_url": "https://wa.me/robots.txt", "observed_at": "2026-09-20T06:23:48.075720+00:00", "status": 200, "final_url": "https://wa.me/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "6ceebd7d44206c973321a46e46b94102a8edcc37eea87394b2dd94ffe713df8d", "bytes_read": 1525 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://wa.me/llms.txt", "observed_at": "2026-09-20T06:23:48.349248+00:00", "status": 404, "final_url": "https://api.whatsapp.com/resolve/?deeplink=%2Fllms.txt¬_found=1", "content_type": "text/html; charset=\"utf-8\"", "truncated": false, "body_sha256": "4f9579b5da8a7d904525fe464e55306fefbe7b87d53bac07f9cf736dda053f98", "bytes_read": 260694 }, "llms_state": "not_detected", "control": null }, { "rank": 67, "domain": "windowsupdate.com", "robots": { "requested_url": "https://windowsupdate.com/robots.txt", "observed_at": "2026-09-20T06:23:48.106934+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://windowsupdate.com/llms.txt", "observed_at": "2026-09-20T06:23:48.147230+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 68, "domain": "msn.com", "robots": { "requested_url": "https://msn.com/robots.txt", "observed_at": "2026-09-20T06:23:48.149003+00:00", "status": 200, "final_url": "https://www.msn.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "aefbffa0a0a083e30637972046b42afc1569f7813c5687739a5ecdee0a6be4dd", "bytes_read": 5245 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://msn.com/llms.txt", "observed_at": "2026-09-20T06:23:48.695019+00:00", "status": 404, "final_url": "https://www.msn.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "10fcf682b17971154f37ba43feed59f3f054c12d46e8d9dab193696175e960ff", "bytes_read": 19143 }, "llms_state": "not_detected", "control": null }, { "rank": 69, "domain": "adobe.com", "robots": { "requested_url": "https://adobe.com/robots.txt", "observed_at": "2026-09-20T06:23:48.201027+00:00", "status": 200, "final_url": "https://www.adobe.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "41081faa0fd323774e225ab51e8a73262055b24ecc7732557229fb93938f9d84", "bytes_read": 10199 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://adobe.com/llms.txt", "observed_at": "2026-09-20T06:23:48.742761+00:00", "status": 200, "final_url": "https://www.adobe.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "811353cb603ff604ecd257b7e24643f1c7e157837015221a951d9d2db0c494d8", "bytes_read": 10499 }, "llms_state": "candidate", "control": { "requested_url": "https://adobe.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:49.063490+00:00", "status": 404, "final_url": "https://www.adobe.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "4cea9068d315a6b931c26a44bd0234f614fdd0155777173b1c93d51729c5f744", "bytes_read": 889 } }, { "rank": 70, "domain": "ntp.org", "robots": { "requested_url": "https://ntp.org/robots.txt", "observed_at": "2026-09-20T06:23:48.358348+00:00", "status": 200, "final_url": "https://www.ntp.org/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "d9066d29c1dfed62e1bbde2c102079a5ccc0c1e9f669644fd89fd652cdc156bb", "bytes_read": 13 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://ntp.org/llms.txt", "observed_at": "2026-09-20T06:23:49.195999+00:00", "status": 404, "final_url": "https://www.ntp.org/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "ce78ad119af14e88e0bd97553a54a70a036626840ba00e22b9aebc2022165394", "bytes_read": 6601 }, "llms_state": "not_detected", "control": null }, { "rank": 71, "domain": "myfritz.net", "robots": { "requested_url": "https://myfritz.net/robots.txt", "observed_at": "2026-09-20T06:23:48.370892+00:00", "status": 200, "final_url": "https://myfritz.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "0267852a7136ff65cf570bc69e024a5198e7d31ef564f90165cd5333250a62ac", "bytes_read": 269 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://myfritz.net/llms.txt", "observed_at": "2026-09-20T06:23:48.989184+00:00", "status": 200, "final_url": "https://www.myfritz.net/devices/", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "e611f62f66f11dc6f52ed18facd92905b82ffdb9bd1915d296181ca4c1a891b9", "bytes_read": 6885 }, "llms_state": "not_detected", "control": null }, { "rank": 72, "domain": "qq.com", "robots": { "requested_url": "https://qq.com/robots.txt", "observed_at": "2026-09-20T06:23:48.439900+00:00", "status": 200, "final_url": "https://www.qq.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1f5d4ddc6513b31998acadbb56fe02e8b9eb244a3eb64d36f446f146e0bc3d5f", "bytes_read": 71 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://qq.com/llms.txt", "observed_at": "2026-09-20T06:23:51.056725+00:00", "status": 200, "final_url": "https://h5.news.qq.com/static/babyhome.htm", "content_type": "text/html", "truncated": false, "body_sha256": "dd3ca8f2a7ee2f282645a07d06786a1fdb6f0caa1e090063805dc51f525c2464", "bytes_read": 4183 }, "llms_state": "not_detected", "control": null }, { "rank": 73, "domain": "cloudflare-dns.com", "robots": { "requested_url": "https://cloudflare-dns.com/robots.txt", "observed_at": "2026-09-20T06:23:48.575279+00:00", "status": 200, "final_url": "https://one.one.one.one/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "8fa3036c68bfcbd32365f6225d24333264093b7cd38d306e106b4dbdc934fd5b", "bytes_read": 1248 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cloudflare-dns.com/llms.txt", "observed_at": "2026-09-20T06:23:49.037024+00:00", "status": 404, "final_url": "https://one.one.one.one/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "8017fa1f7764a80571cc0a99c7dbdfaa33dad3ec31f19239d56d38b0a6f421fe", "bytes_read": 1285 }, "llms_state": "not_detected", "control": null }, { "rank": 74, "domain": "root-servers.net", "robots": { "requested_url": "https://root-servers.net/robots.txt", "observed_at": "2026-09-20T06:23:48.622662+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://root-servers.net/llms.txt", "observed_at": "2026-09-20T06:23:48.686290+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 75, "domain": "nginx.org", "robots": { "requested_url": "https://nginx.org/robots.txt", "observed_at": "2026-09-20T06:23:48.630809+00:00", "status": 200, "final_url": "https://nginx.org/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "8f5b42f5d23ca623fa42f917a5c90b8677ef8d7c9b5ffc0324bbea8e7067f26e", "bytes_read": 72 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://nginx.org/llms.txt", "observed_at": "2026-09-20T06:23:48.983152+00:00", "status": 404, "final_url": "https://nginx.org/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "24342a7369de0717a42a99e8cfb0d278f1118fd7c18274483b9bfe52a8de7082", "bytes_read": 2039 }, "llms_state": "not_detected", "control": null }, { "rank": 76, "domain": "chatgpt.com", "robots": { "requested_url": "https://chatgpt.com/robots.txt", "observed_at": "2026-09-20T06:23:48.637331+00:00", "status": 200, "final_url": "https://chatgpt.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "22c1e86846f6588b078eaba2dc513fd839d26740fb52ee3392127e1f218e5031", "bytes_read": 4316 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://chatgpt.com/llms.txt", "observed_at": "2026-09-20T06:23:48.811865+00:00", "status": 403, "final_url": "https://chatgpt.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "acbd424440105fb8c6f0ddffd8e0ec7981d0b9c9a08f3b592e35da66d5bd0c6e", "bytes_read": 8636 }, "llms_state": "unavailable", "control": null }, { "rank": 77, "domain": "zoom.us", "robots": { "requested_url": "https://zoom.us/robots.txt", "observed_at": "2026-09-20T06:23:48.688688+00:00", "status": 200, "final_url": "https://zoom.us/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "b3ed9f53f38c480b046b4d5a648aa7a879c6720957301506dc282b32fb2f5593", "bytes_read": 2005 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://zoom.us/llms.txt", "observed_at": "2026-09-20T06:23:48.929805+00:00", "status": 404, "final_url": "https://zoom.us/llms.txt", "content_type": "text/html;charset=utf-8", "truncated": true, "body_sha256": "138bfe3cf987628e337487dff4724099ea533146cfafdd5fbe4ae5866540d052", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 78, "domain": "cdninstagram.com", "robots": { "requested_url": "https://cdninstagram.com/robots.txt", "observed_at": "2026-09-20T06:23:48.729842+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cdninstagram.com/llms.txt", "observed_at": "2026-09-20T06:23:48.824046+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 79, "domain": "workers.dev", "robots": { "requested_url": "https://workers.dev/robots.txt", "observed_at": "2026-09-20T06:23:48.826299+00:00", "status": 200, "final_url": "https://www.cloudflare.com/products/workers/", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "5992f8cc771cd9bb06006794578a34ebd8e72386b4e34c86afa53551fbb07499", "bytes_read": 512001 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://workers.dev/llms.txt", "observed_at": "2026-09-20T06:23:49.785024+00:00", "status": 200, "final_url": "https://www.cloudflare.com/products/workers/", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "892e01b2865dfdbf537069d140cea7b331e315a27e42ec995908bac8754f5f2f", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 80, "domain": "akam.net", "robots": { "requested_url": "https://akam.net/robots.txt", "observed_at": "2026-09-20T06:23:48.829017+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://akam.net/llms.txt", "observed_at": "2026-09-20T06:23:48.905603+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 81, "domain": "tiktokcdn.com", "robots": { "requested_url": "https://tiktokcdn.com/robots.txt", "observed_at": "2026-09-20T06:23:48.907351+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://tiktokcdn.com/llms.txt", "observed_at": "2026-09-20T06:23:48.943127+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 82, "domain": "openai.com", "robots": { "requested_url": "https://openai.com/robots.txt", "observed_at": "2026-09-20T06:23:48.931864+00:00", "status": 200, "final_url": "https://openai.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498", "bytes_read": 98 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://openai.com/llms.txt", "observed_at": "2026-09-20T06:23:49.094775+00:00", "status": 403, "final_url": "https://openai.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "641128b48b3760e8d34847f862f7d3fd85a57418aea7126a4cb114bd9d835f76", "bytes_read": 9810 }, "llms_state": "unavailable", "control": null }, { "rank": 83, "domain": "tiktokv.com", "robots": { "requested_url": "https://tiktokv.com/robots.txt", "observed_at": "2026-09-20T06:23:48.932796+00:00", "status": 404, "final_url": "https://tiktokv.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248", "bytes_read": 144 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://tiktokv.com/llms.txt", "observed_at": "2026-09-20T06:23:49.770189+00:00", "status": 404, "final_url": "https://tiktokv.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "b282ac8512c5e029959282e6fd450dc119f5db4a072869eabf45e31440596248", "bytes_read": 144 }, "llms_state": "not_detected", "control": null }, { "rank": 84, "domain": "edgekey.net", "robots": { "requested_url": "https://edgekey.net/robots.txt", "observed_at": "2026-09-20T06:23:48.945689+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://edgekey.net/llms.txt", "observed_at": "2026-09-20T06:23:48.989822+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 85, "domain": "gvt2.com", "robots": { "requested_url": "https://gvt2.com/robots.txt", "observed_at": "2026-09-20T06:23:48.992518+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://gvt2.com/llms.txt", "observed_at": "2026-09-20T06:23:49.044159+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 86, "domain": "yandex.net", "robots": { "requested_url": "https://yandex.net/robots.txt", "observed_at": "2026-09-20T06:23:49.006082+00:00", "status": 404, "final_url": "https://yandex.net/robots.txt", "content_type": "", "truncated": false, "body_sha256": "0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5", "bytes_read": 9 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://yandex.net/llms.txt", "observed_at": "2026-09-20T06:23:49.771302+00:00", "status": 404, "final_url": "https://ya.ru/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "2ff41f2733d71b32b1e02e9e7a6960d9a908bca2aaa6020f2a9300594bb1c88a", "bytes_read": 11787 }, "llms_state": "not_detected", "control": null }, { "rank": 87, "domain": "baidu.com", "robots": { "requested_url": "https://baidu.com/robots.txt", "observed_at": "2026-09-20T06:23:49.026491+00:00", "status": 200, "final_url": "https://www.baidu.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "c28ebeffb7baf9bd6407109e80dc95d87c75c0eaecc20b23884cb33111cc5df0", "bytes_read": 2814 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://baidu.com/llms.txt", "observed_at": "2026-09-20T06:23:52.042468+00:00", "status": 404, "final_url": "https://www.baidu.com/llms.txt", "content_type": "text/html; charset=iso-8859-1", "truncated": false, "body_sha256": "0da006306ca67b9d88367bc1a63929b073ed36fb046b9b61695f934bfa544b89", "bytes_read": 206 }, "llms_state": "not_detected", "control": null }, { "rank": 88, "domain": "vimeo.com", "robots": { "requested_url": "https://vimeo.com/robots.txt", "observed_at": "2026-09-20T06:23:49.046904+00:00", "status": 200, "final_url": "https://vimeo.com/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "021ea0af6b29208b5ad678715c9e6671373ae3a937b8e879a366e5f497f88b10", "bytes_read": 22615 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://vimeo.com/llms.txt", "observed_at": "2026-09-20T06:23:50.089045+00:00", "status": 404, "final_url": "https://vimeo.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "4579e8925d4de56d8a55060215f23592bdd9b3661d1ac7c5b7c9735ca3c22c23", "bytes_read": 164629 }, "llms_state": "not_detected", "control": null }, { "rank": 89, "domain": "ytimg.com", "robots": { "requested_url": "https://ytimg.com/robots.txt", "observed_at": "2026-09-20T06:23:49.095575+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ytimg.com/llms.txt", "observed_at": "2026-09-20T06:23:49.142097+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 90, "domain": "windows.com", "robots": { "requested_url": "https://windows.com/robots.txt", "observed_at": "2026-09-20T06:23:49.144809+00:00", "status": 200, "final_url": "https://www.microsoft.com/en-us/windows/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "96a3c21a141b339a01a61f3e7a83250919c0c0afaaa07511d7308304c497546b", "bytes_read": 506087 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://windows.com/llms.txt", "observed_at": "2026-09-20T06:23:51.277590+00:00", "status": 200, "final_url": "https://www.microsoft.com/en-us/windows/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "96a3c21a141b339a01a61f3e7a83250919c0c0afaaa07511d7308304c497546b", "bytes_read": 506087 }, "llms_state": "not_detected", "control": null }, { "rank": 91, "domain": "f5.com", "robots": { "requested_url": "https://f5.com/robots.txt", "observed_at": "2026-09-20T06:23:49.146180+00:00", "status": 200, "final_url": "https://www.f5.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "eca371645a1484bc5d7dfc8e150f188b7e70d7754e6c00c1770de364f38e1401", "bytes_read": 156 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://f5.com/llms.txt", "observed_at": "2026-09-20T06:23:50.065059+00:00", "status": 404, "final_url": "https://www.f5.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "c5790bb85ed53ac58a738dc4af5419d8734fb46fda5a4bbcb1c9a55782ff548e", "bytes_read": 102146 }, "llms_state": "not_detected", "control": null }, { "rank": 92, "domain": "opera.com", "robots": { "requested_url": "https://opera.com/robots.txt", "observed_at": "2026-09-20T06:23:49.199171+00:00", "status": 200, "final_url": "https://www.opera.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "0c6b878d90d9783f0a0972e38888f8dfef8393e317be80a266a82f3da548f20f", "bytes_read": 994 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://opera.com/llms.txt", "observed_at": "2026-09-20T06:23:50.136313+00:00", "status": 200, "final_url": "https://www.opera.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "abefbde9fb5c1ef40e063c0d21aeff38c71e1f9899b3b7cc8d924a89deffb013", "bytes_read": 11613 }, "llms_state": "candidate", "control": { "requested_url": "https://opera.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:50.887557+00:00", "status": 404, "final_url": "https://www.opera.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "37702814e870f01f8f6db030391df74fcf7150321e3e4fbcb9a2b19846d5ee38", "bytes_read": 114536 } }, { "rank": 93, "domain": "sentry.io", "robots": { "requested_url": "https://sentry.io/robots.txt", "observed_at": "2026-09-20T06:23:49.217366+00:00", "status": 200, "final_url": "https://sentry.io/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "c0f7d803d97db6c0bf75804b97eed4a533f7249c27994e253ad4fa403652da96", "bytes_read": 140 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://sentry.io/llms.txt", "observed_at": "2026-09-20T06:23:49.433991+00:00", "status": 200, "final_url": "https://sentry.io/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "aff6ff87530b58b92f6aa67ecf456df9dfe3d7976084700915bfb73eed1d1258", "bytes_read": 3714 }, "llms_state": "candidate", "control": { "requested_url": "https://sentry.io/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:49.811287+00:00", "status": 200, "final_url": "https://sentry.io/auth/login/", "content_type": "text/html", "truncated": false, "body_sha256": "454f9c2c75089638b07117423b04d22a60911e495a094e4e3c83250f1d146129", "bytes_read": 21058 } }, { "rank": 94, "domain": "samsung.com", "robots": { "requested_url": "https://samsung.com/robots.txt", "observed_at": "2026-09-20T06:23:49.304592+00:00", "status": 200, "final_url": "https://www.samsung.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e07ce06f3220d0496ee9bec373ddb2e2fdc272391a4b0153c453bd39b08e39f8", "bytes_read": 10085 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://samsung.com/llms.txt", "observed_at": "2026-09-20T06:23:50.886255+00:00", "status": 200, "final_url": "https://www.samsung.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "d248d7acb33e01eff77c0c2fe1b2f25196ce8d9265ce133583435a80b686589f", "bytes_read": 4597 }, "llms_state": "candidate", "control": { "requested_url": "https://samsung.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:52.073519+00:00", "status": 404, "final_url": "https://www.samsung.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "864b491ab74e1116473e79ab8e9c4c8ae3c68966f6d20c294e58e242c25a819c", "bytes_read": 94161 } }, { "rank": 95, "domain": "nginx.com", "robots": { "requested_url": "https://nginx.com/robots.txt", "observed_at": "2026-09-20T06:23:49.324442+00:00", "status": 200, "final_url": "https://www.f5.com/products/nginx", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "f0e99a73c884eb176ed57710acc9d2372f52c737164edfde804e02d4b2d3d7bf", "bytes_read": 402836 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://nginx.com/llms.txt", "observed_at": "2026-09-20T06:23:50.717105+00:00", "status": 200, "final_url": "https://www.f5.com/products/nginx", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "f0e99a73c884eb176ed57710acc9d2372f52c737164edfde804e02d4b2d3d7bf", "bytes_read": 402836 }, "llms_state": "not_detected", "control": null }, { "rank": 96, "domain": "yandex.ru", "robots": { "requested_url": "https://yandex.ru/robots.txt", "observed_at": "2026-09-20T06:23:49.391970+00:00", "status": 200, "final_url": "https://yandex.ru/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "15a85b858cbe0939830a319acdc887709540e916e9f7e7b1f1c106527b65e41f", "bytes_read": 39582 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://yandex.ru/llms.txt", "observed_at": "2026-09-20T06:23:50.521140+00:00", "status": 404, "final_url": "https://yandex.ru/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "6b26dd7123f27b59a22a785e8cca212cb9ee3c923d830a1913c3b8d5fc9bf7f0", "bytes_read": 11791 }, "llms_state": "not_detected", "control": null }, { "rank": 97, "domain": "edgesuite.net", "robots": { "requested_url": "https://edgesuite.net/robots.txt", "observed_at": "2026-09-20T06:23:50.096978+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://edgesuite.net/llms.txt", "observed_at": "2026-09-20T06:23:50.136807+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 98, "domain": "highwebmedia.com", "robots": { "requested_url": "https://highwebmedia.com/robots.txt", "observed_at": "2026-09-20T06:23:50.141113+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://highwebmedia.com/llms.txt", "observed_at": "2026-09-20T06:23:50.169468+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 99, "domain": "mozilla.org", "robots": { "requested_url": "https://mozilla.org/robots.txt", "observed_at": "2026-09-20T06:23:50.171416+00:00", "status": 200, "final_url": "https://www.mozilla.org/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "cf0fd2842659297b0d6fce407bed86bf424ae6666051b881dbd72e871d687fe3", "bytes_read": 66 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://mozilla.org/llms.txt", "observed_at": "2026-09-20T06:23:50.638303+00:00", "status": 404, "final_url": "https://www.mozilla.org/en-US/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "9475c8da3d41214acce52bde2711027ed7122046b819573334ccf189f53956b4", "bytes_read": 9118 }, "llms_state": "not_detected", "control": null }, { "rank": 100, "domain": "t.me", "robots": { "requested_url": "https://t.me/robots.txt", "observed_at": "2026-09-20T06:23:50.246818+00:00", "status": 404, "final_url": "https://t.me/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "bd5e6537d1fa8b7e136d3589dd722410eb3836f3b4b8e21733fba881463f5272", "bytes_read": 153 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://t.me/llms.txt", "observed_at": "2026-09-20T06:23:50.730419+00:00", "status": 200, "final_url": "https://telegram.org/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "6fb274a1a5aa44ceacb50ed63ba2b34732e7dc1d3d7b9bebd70e173bc6e49f5f", "bytes_read": 19919 }, "llms_state": "not_detected", "control": null }, { "rank": 101, "domain": "wordpress.com", "robots": { "requested_url": "https://wordpress.com/robots.txt", "observed_at": "2026-09-20T06:23:50.273238+00:00", "status": 200, "final_url": "https://wordpress.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "462d4866929634bf5cc361d3af1d4b9cff0f744bca3a2f3bb7a6b6c67a1c496d", "bytes_read": 1387 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://wordpress.com/llms.txt", "observed_at": "2026-09-20T06:23:50.478571+00:00", "status": 200, "final_url": "https://wordpress.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "4c085ee948dd6fbdecf247242cd38b6ebde72b16fe78938438a9975503ff842b", "bytes_read": 7018 }, "llms_state": "candidate", "control": { "requested_url": "https://wordpress.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:50.646900+00:00", "status": 404, "final_url": "https://wordpress.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "c7311ee845efb3728587742d83a881bc514a3cf1139c5c4e37bc6d477351d407", "bytes_read": 144494 } }, { "rank": 102, "domain": "blogspot.com", "robots": { "requested_url": "https://blogspot.com/robots.txt", "observed_at": "2026-09-20T06:23:50.331112+00:00", "status": 200, "final_url": "https://www.blogger.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "2076a51a5ea6b870209770053655a5e2499bddd4ae6a6a40b5453b9898168a34", "bytes_read": 572 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://blogspot.com/llms.txt", "observed_at": "2026-09-20T06:23:50.834163+00:00", "status": 404, "final_url": "https://www.blogger.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "103bd81325370dd77244d0f6e2d92e8eb4852627d6a63deb7bf05a840409210a", "bytes_read": 168 }, "llms_state": "not_detected", "control": null }, { "rank": 103, "domain": "gwfb.net", "robots": { "requested_url": "https://gwfb.net/robots.txt", "observed_at": "2026-09-20T06:23:50.354637+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://gwfb.net/llms.txt", "observed_at": "2026-09-20T06:23:50.695786+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 104, "domain": "achmea.nl", "robots": { "requested_url": "https://achmea.nl/robots.txt", "observed_at": "2026-09-20T06:23:50.697504+00:00", "status": 200, "final_url": "https://www.achmea.nl/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "d9066d29c1dfed62e1bbde2c102079a5ccc0c1e9f669644fd89fd652cdc156bb", "bytes_read": 13 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://achmea.nl/llms.txt", "observed_at": "2026-09-20T06:23:51.796641+00:00", "status": 404, "final_url": "https://www.achmea.nl/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "7d9299d223988d669c0b3da524cf69eb9eae148a988e8074b0b185995fe76a37", "bytes_read": 40794 }, "llms_state": "not_detected", "control": null }, { "rank": 105, "domain": "dellemc.com", "robots": { "requested_url": "https://dellemc.com/robots.txt", "observed_at": "2026-09-20T06:23:50.774847+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://dellemc.com/llms.txt", "observed_at": "2026-09-20T06:24:10.832147+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 106, "domain": "actify.nl", "robots": { "requested_url": "https://actify.nl/robots.txt", "observed_at": "2026-09-20T06:23:50.830835+00:00", "status": 200, "final_url": "https://www.zilverenkruis.nl/gezonder-leven/actify/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "95ddbc76823137ff85fd07cab0c38523262bd69cdde4ce18581429cac6d5462a", "bytes_read": 373 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://actify.nl/llms.txt", "observed_at": "2026-09-20T06:23:52.430118+00:00", "status": 404, "final_url": "https://www.zilverenkruis.nl/gezonder-leven/actify/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "11918fe8f540f7b48c1eecf0e9ddfbc8f04674709881661d2b2d51eb35d2c122", "bytes_read": 240868 }, "llms_state": "not_detected", "control": null }, { "rank": 107, "domain": "a2z.com", "robots": { "requested_url": "https://a2z.com/robots.txt", "observed_at": "2026-09-20T06:23:50.981111+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://a2z.com/llms.txt", "observed_at": "2026-09-20T06:23:51.064790+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 108, "domain": "b-cdn.net", "robots": { "requested_url": "https://b-cdn.net/robots.txt", "observed_at": "2026-09-20T06:23:51.069002+00:00", "status": 200, "final_url": "https://bunny.net/", "content_type": "text/html", "truncated": true, "body_sha256": "fb76c24e584833a4129646a647093de3c28266d83e1e22981bd3c1561a27d2bb", "bytes_read": 512001 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://b-cdn.net/llms.txt", "observed_at": "2026-09-20T06:23:51.869299+00:00", "status": 200, "final_url": "https://bunny.net/", "content_type": "text/html", "truncated": true, "body_sha256": "fb76c24e584833a4129646a647093de3c28266d83e1e22981bd3c1561a27d2bb", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 109, "domain": "reddit.com", "robots": { "requested_url": "https://reddit.com/robots.txt", "observed_at": "2026-09-20T06:23:51.121920+00:00", "status": 200, "final_url": "https://www.reddit.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "41aea42110242175c7b4247f222fa95ac6caee62044af686b684785832781857", "bytes_read": 538 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://reddit.com/llms.txt", "observed_at": "2026-09-20T06:23:51.523811+00:00", "status": 200, "final_url": "https://www.reddit.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "465c058609f23f489620f26007a4bbb16d6ff3857198f0f4a7ad54a618fe5220", "bytes_read": 8405 }, "llms_state": "not_detected", "control": null }, { "rank": 110, "domain": "ui.com", "robots": { "requested_url": "https://ui.com/robots.txt", "observed_at": "2026-09-20T06:23:51.176186+00:00", "status": 200, "final_url": "https://ui.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "3a07e69b7c6f2ba775e98171bf02b7c40e7a4624ef9a0819de63a02d903f360e", "bytes_read": 367 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://ui.com/llms.txt", "observed_at": "2026-09-20T06:23:51.412380+00:00", "status": 404, "final_url": "https://ui.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "4f0a26d3f89010d73c9f59cfa5148894eaf4dcf21a6aa881e80d8d98cc486bc8", "bytes_read": 271731 }, "llms_state": "not_detected", "control": null }, { "rank": 111, "domain": "nic.ru", "robots": { "requested_url": "https://nic.ru/robots.txt", "observed_at": "2026-09-20T06:23:51.215512+00:00", "status": 200, "final_url": "https://nic.ru/robots.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "d0db68a344d1ad23628a7baec8bdbdf58a763073717bd3cf01ebba3d3478e8ef", "bytes_read": 13602 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://nic.ru/llms.txt", "observed_at": "2026-09-20T06:23:53.165339+00:00", "status": 200, "final_url": "https://nic.ru/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "d0db68a344d1ad23628a7baec8bdbdf58a763073717bd3cf01ebba3d3478e8ef", "bytes_read": 13602 }, "llms_state": "not_detected", "control": null }, { "rank": 112, "domain": "office365.com", "robots": { "requested_url": "https://office365.com/robots.txt", "observed_at": "2026-09-20T06:23:51.681566+00:00", "status": 403, "final_url": "https://office365.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "4686b7c61f163cc8b8c09e6f5645d6f30407f6033e314e62d1cdec1a745fa7b5", "bytes_read": 1892 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://office365.com/llms.txt", "observed_at": "2026-09-20T06:23:51.965585+00:00", "status": 403, "final_url": "https://office365.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "4686b7c61f163cc8b8c09e6f5645d6f30407f6033e314e62d1cdec1a745fa7b5", "bytes_read": 1892 }, "llms_state": "unavailable", "control": null }, { "rank": 113, "domain": "yccdn.ru", "robots": { "requested_url": "https://yccdn.ru/robots.txt", "observed_at": "2026-09-20T06:23:51.736480+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://yccdn.ru/llms.txt", "observed_at": "2026-09-20T06:23:52.010093+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 114, "domain": "google-analytics.com", "robots": { "requested_url": "https://google-analytics.com/robots.txt", "observed_at": "2026-09-20T06:23:51.749372+00:00", "status": 200, "final_url": "https://google-analytics.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "43b87e6af8d60fc13ffb0a175234363f7edbe90f5dfe39200215feb4c30a3de9", "bytes_read": 78 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://google-analytics.com/llms.txt", "observed_at": "2026-09-20T06:23:51.939165+00:00", "status": 404, "final_url": "https://google-analytics.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 115, "domain": "aws.dev", "robots": { "requested_url": "https://aws.dev/robots.txt", "observed_at": "2026-09-20T06:23:51.793725+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://aws.dev/llms.txt", "observed_at": "2026-09-20T06:23:51.894590+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 116, "domain": "europa.eu", "robots": { "requested_url": "https://europa.eu/robots.txt", "observed_at": "2026-09-20T06:23:51.897152+00:00", "status": 200, "final_url": "https://europa.eu/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "bede8dd1fdd4b10f8a4ba2f0087e13cce1d83a52c192323b7959fd8213aeeb31", "bytes_read": 4930 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://europa.eu/llms.txt", "observed_at": "2026-09-20T06:23:52.398322+00:00", "status": 404, "final_url": "https://european-union.europa.eu/not-found_en", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "9ceac76ce49d9acf9b2174d9288825e9770409d7065aa57a9ad85956ca18bab8", "bytes_read": 54349 }, "llms_state": "not_detected", "control": null }, { "rank": 117, "domain": "bit.ly", "robots": { "requested_url": "https://bit.ly/robots.txt", "observed_at": "2026-09-20T06:23:51.948252+00:00", "status": 200, "final_url": "https://bit.ly/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "aeddaf37d71afe500676c334c63a84340d99c91651b9d41bc0c4b2f9437d6cd8", "bytes_read": 123 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://bit.ly/llms.txt", "observed_at": "2026-09-20T06:23:52.180236+00:00", "status": 200, "final_url": "https://zu.lk/-/not-found", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "57f66e909f563a75ec4cff0a431e9c793f2941a74f61ab71de3aa783483bf4b4", "bytes_read": 102230 }, "llms_state": "not_detected", "control": null }, { "rank": 118, "domain": "okcdn.ru", "robots": { "requested_url": "https://okcdn.ru/robots.txt", "observed_at": "2026-09-20T06:23:52.012330+00:00", "status": 404, "final_url": "https://st.okcdn.ru/robots.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "9145e451d2330fa2b4c9f1accfc1d289eae481ab19f0f1030ca870946881b343", "bytes_read": 147 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://okcdn.ru/llms.txt", "observed_at": "2026-09-20T06:23:53.560479+00:00", "status": 404, "final_url": "https://st.okcdn.ru/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "9145e451d2330fa2b4c9f1accfc1d289eae481ab19f0f1030ca870946881b343", "bytes_read": 147 }, "llms_state": "not_detected", "control": null }, { "rank": 119, "domain": "app-analytics-services.com", "robots": { "requested_url": "https://app-analytics-services.com/robots.txt", "observed_at": "2026-09-20T06:23:52.152994+00:00", "status": 404, "final_url": "https://app-analytics-services.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2", "bytes_read": 1571 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://app-analytics-services.com/llms.txt", "observed_at": "2026-09-20T06:23:52.400808+00:00", "status": 404, "final_url": "https://app-analytics-services.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 120, "domain": "outlook.com", "robots": { "requested_url": "https://outlook.com/robots.txt", "observed_at": "2026-09-20T06:23:52.167877+00:00", "status": 417, "final_url": "https://outlook.com/mail/", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://outlook.com/llms.txt", "observed_at": "2026-09-20T06:23:53.301790+00:00", "status": 417, "final_url": "https://outlook.com/mail/", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "unavailable", "control": null }, { "rank": 121, "domain": "app-measurement.com", "robots": { "requested_url": "https://app-measurement.com/robots.txt", "observed_at": "2026-09-20T06:23:52.223193+00:00", "status": 404, "final_url": "https://app-measurement.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2", "bytes_read": 1571 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://app-measurement.com/llms.txt", "observed_at": "2026-09-20T06:23:52.455635+00:00", "status": 404, "final_url": "https://app-measurement.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 122, "domain": "registrar-servers.com", "robots": { "requested_url": "https://registrar-servers.com/robots.txt", "observed_at": "2026-09-20T06:23:52.570324+00:00", "status": 200, "final_url": "https://registrar-servers.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "331ea9090db0c9f6f597bd9840fd5b171830f6e0b3ba1cb24dfa91f0c95aedc1", "bytes_read": 26 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://registrar-servers.com/llms.txt", "observed_at": "2026-09-20T06:23:52.909842+00:00", "status": 404, "final_url": "https://registrar-servers.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "c01638037abccc0746d0db73f3a24f9d9afe2cb81403fb10b195976a25cbc576", "bytes_read": 112 }, "llms_state": "not_detected", "control": null }, { "rank": 123, "domain": "amazonvideo.com", "robots": { "requested_url": "https://amazonvideo.com/robots.txt", "observed_at": "2026-09-20T06:23:52.612969+00:00", "status": 200, "final_url": "https://www.primevideo.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e924a08cd574df9664040688a18d722c6f08c1f8a7352ea8748dcda4795d04c8", "bytes_read": 4348 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://amazonvideo.com/llms.txt", "observed_at": "2026-09-20T06:23:53.058458+00:00", "status": 404, "final_url": "https://www.primevideo.com/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "05440d44c35780e9fe03ae2d1aada60b5934291f8084d9ac99c8839c2a1aaa4b", "bytes_read": 6649 }, "llms_state": "not_detected", "control": null }, { "rank": 124, "domain": "criteo.com", "robots": { "requested_url": "https://criteo.com/robots.txt", "observed_at": "2026-09-20T06:23:52.696238+00:00", "status": 200, "final_url": "https://criteo.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "ed40002530b0bd8be508782552d6d35cd29d7b15ae6a3efbbd9317ea9381c5a8", "bytes_read": 2056 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://criteo.com/llms.txt", "observed_at": "2026-09-20T06:23:52.897587+00:00", "status": 404, "final_url": "https://www.criteo.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "a30d052f5a07bdf16d56fea1793ee8c1a2fc812ae463e6b3e4ec5f257a6a9fa2", "bytes_read": 327174 }, "llms_state": "not_detected", "control": null }, { "rank": 125, "domain": "github.io", "robots": { "requested_url": "https://github.io/robots.txt", "observed_at": "2026-09-20T06:23:52.858466+00:00", "status": 200, "final_url": "https://pages.github.com/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "c974a5f7c122118328ea0c3fa14e494ddf01fa6a77d0e70e6e3c6773125d81ad", "bytes_read": 14446 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://github.io/llms.txt", "observed_at": "2026-09-20T06:23:53.354992+00:00", "status": 200, "final_url": "https://pages.github.com/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "c974a5f7c122118328ea0c3fa14e494ddf01fa6a77d0e70e6e3c6773125d81ad", "bytes_read": 14446 }, "llms_state": "not_detected", "control": null }, { "rank": 126, "domain": "cdn77.org", "robots": { "requested_url": "https://cdn77.org/robots.txt", "observed_at": "2026-09-20T06:23:53.119142+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cdn77.org/llms.txt", "observed_at": "2026-09-20T06:23:53.173847+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 127, "domain": "vk.com", "robots": { "requested_url": "https://vk.com/robots.txt", "observed_at": "2026-09-20T06:23:53.170402+00:00", "status": 200, "final_url": "https://vk.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "573219d395c1aa085175955ea564938f81e5c31b8cd52ba15d49a2171ffb06c4", "bytes_read": 6175 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://vk.com/llms.txt", "observed_at": "2026-09-20T06:23:53.808877+00:00", "status": 404, "final_url": "https://vk.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "02df22b1683f5dfbe9b9b06e4a1332d2be7acc48714baa1c504edacfbaa97f39", "bytes_read": 13120 }, "llms_state": "not_detected", "control": null }, { "rank": 128, "domain": "trbcdn.net", "robots": { "requested_url": "https://trbcdn.net/robots.txt", "observed_at": "2026-09-20T06:23:53.175767+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://trbcdn.net/llms.txt", "observed_at": "2026-09-20T06:23:53.809130+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 129, "domain": "snapchat.com", "robots": { "requested_url": "https://snapchat.com/robots.txt", "observed_at": "2026-09-20T06:23:53.358104+00:00", "status": 200, "final_url": "https://www.snapchat.com:443/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "ee1ba42e1ce0f9bfcbd1a7620dc7c31c50917675fd01fae255b50ded35c18f8b", "bytes_read": 3997 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://snapchat.com/llms.txt", "observed_at": "2026-09-20T06:23:53.990005+00:00", "status": 404, "final_url": "https://www.snapchat.com:443/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "8932ecd55f69aa4d55d4a639b3517feb7221c7d78d7ef22e59e1322dc627ba1e", "bytes_read": 66368 }, "llms_state": "not_detected", "control": null }, { "rank": 130, "domain": "apache.org", "robots": { "requested_url": "https://apache.org/robots.txt", "observed_at": "2026-09-20T06:23:53.447330+00:00", "status": 200, "final_url": "https://apache.org/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e969244acbbadc64dd66254a7d2bb20017fd9ed3115290694c9a1f562a070c53", "bytes_read": 47 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://apache.org/llms.txt", "observed_at": "2026-09-20T06:23:53.912834+00:00", "status": 404, "final_url": "https://apache.org/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "68bd24dd9e6895274a94f815eb7d9e6494c8ae6f39e8799ce6ede5190aec76fa", "bytes_read": 24653 }, "llms_state": "not_detected", "control": null }, { "rank": 131, "domain": "amazon-adsystem.com", "robots": { "requested_url": "https://amazon-adsystem.com/robots.txt", "observed_at": "2026-09-20T06:23:53.495724+00:00", "status": null, "error": "RemoteDisconnected" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://amazon-adsystem.com/llms.txt", "observed_at": "2026-09-20T06:23:58.706412+00:00", "status": null, "error": "RemoteDisconnected" }, "llms_state": "unavailable", "control": null }, { "rank": 132, "domain": "dnsowl.com", "robots": { "requested_url": "https://dnsowl.com/robots.txt", "observed_at": "2026-09-20T06:23:53.588026+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://dnsowl.com/llms.txt", "observed_at": "2026-09-20T06:23:53.722444+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 133, "domain": "unity3d.com", "robots": { "requested_url": "https://unity3d.com/robots.txt", "observed_at": "2026-09-20T06:23:53.724715+00:00", "status": 200, "final_url": "https://unity.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1a90eb94d80776f80da593d687c6d1e61cd6c479568fc00b8bbb3b941eaa0c80", "bytes_read": 264 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://unity3d.com/llms.txt", "observed_at": "2026-09-20T06:23:54.574309+00:00", "status": 200, "final_url": "https://unity.com/llms.txt", "content_type": "text/markdown; charset=utf-8", "truncated": true, "body_sha256": "57ff5a342b7df88fa66997ae69ba7250e12555fbd717c573a720e780ce6bf4d4", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 134, "domain": "amazon.dev", "robots": { "requested_url": "https://amazon.dev/robots.txt", "observed_at": "2026-09-20T06:23:53.802765+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://amazon.dev/llms.txt", "observed_at": "2026-09-20T06:23:53.895131+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 135, "domain": "intuit.com", "robots": { "requested_url": "https://intuit.com/robots.txt", "observed_at": "2026-09-20T06:23:53.838137+00:00", "status": 200, "final_url": "https://www.intuit.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "99f9354d3dc7685b0a7c25eb9370fb367e240013a28fd6600bf705984c5c1444", "bytes_read": 3086 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://intuit.com/llms.txt", "observed_at": "2026-09-20T06:23:54.395224+00:00", "status": null, "error": "TimeoutError" }, "llms_state": "unavailable", "control": null }, { "rank": 136, "domain": "dropbox.com", "robots": { "requested_url": "https://dropbox.com/robots.txt", "observed_at": "2026-09-20T06:23:53.897403+00:00", "status": 200, "final_url": "https://www.dropbox.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "cc244a992405a4c1368861e7a1968a68ae1d48567b5802fe4d1f1e00d73974b3", "bytes_read": 801 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://dropbox.com/llms.txt", "observed_at": "2026-09-20T06:23:54.450168+00:00", "status": 200, "final_url": "https://www.dropbox.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "ad9ed13f7ca90a0a91d36799eac5aa2bf9bdc985e66ddf8b8583f83a30556171", "bytes_read": 209330 }, "llms_state": "candidate", "control": { "requested_url": "https://dropbox.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:54.960779+00:00", "status": 404, "final_url": "https://www.dropbox.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "7c3062f4433da04b86fb2a95156b3598d5e9e030494f9956755dcff563579a4a", "bytes_read": 1233 } }, { "rank": 137, "domain": "reg.ru", "robots": { "requested_url": "https://reg.ru/robots.txt", "observed_at": "2026-09-20T06:23:53.915090+00:00", "status": 200, "final_url": "https://reg.ru/robots.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "d0db68a344d1ad23628a7baec8bdbdf58a763073717bd3cf01ebba3d3478e8ef", "bytes_read": 13602 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://reg.ru/llms.txt", "observed_at": "2026-09-20T06:23:54.506158+00:00", "status": 200, "final_url": "https://reg.ru/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "d0db68a344d1ad23628a7baec8bdbdf58a763073717bd3cf01ebba3d3478e8ef", "bytes_read": 13602 }, "llms_state": "not_detected", "control": null }, { "rank": 138, "domain": "triplinkintl.com", "robots": { "requested_url": "https://triplinkintl.com/robots.txt", "observed_at": "2026-09-20T06:23:54.045120+00:00", "status": 200, "final_url": "https://www.triplinkintl.com/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "230baddcdcdb946857dcc28cd01657f7408f0de87bf6d31e4ab4e959a091384e", "bytes_read": 41919 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://triplinkintl.com/llms.txt", "observed_at": "2026-09-20T06:23:58.407183+00:00", "status": 200, "final_url": "https://www.triplinkintl.com/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "a545ac9b2b095caf9a84d9854b25791bbe598e6af1045fd82581982fce4d43fc", "bytes_read": 41919 }, "llms_state": "not_detected", "control": null }, { "rank": 139, "domain": "epicgames.com", "robots": { "requested_url": "https://epicgames.com/robots.txt", "observed_at": "2026-09-20T06:23:54.198656+00:00", "status": 403, "final_url": "https://www.epicgames.com/site/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "d6bfb00e46bf924022adb05e1bf8f67fc0a82d51ebcc7c9669ec59de47fdea0a", "bytes_read": 25388 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://epicgames.com/llms.txt", "observed_at": "2026-09-20T06:23:54.808612+00:00", "status": 403, "final_url": "https://www.epicgames.com/site/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "52e346e4693a712c6bcf7997eab1411675d7c2641df8d13bd8631fa59d830853", "bytes_read": 25382 }, "llms_state": "unavailable", "control": null }, { "rank": 140, "domain": "msftncsi.com", "robots": { "requested_url": "https://msftncsi.com/robots.txt", "observed_at": "2026-09-20T06:23:54.293324+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://msftncsi.com/llms.txt", "observed_at": "2026-09-20T06:23:54.338384+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 141, "domain": "kaspersky.com", "robots": { "requested_url": "https://kaspersky.com/robots.txt", "observed_at": "2026-09-20T06:23:54.340229+00:00", "status": 200, "final_url": "https://www.kaspersky.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "9efe9b0f74014e7bd3c10828a1af580cd30907b83f13b9aa52f001e3ab5ea3b7", "bytes_read": 1929 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://kaspersky.com/llms.txt", "observed_at": "2026-09-20T06:23:55.663254+00:00", "status": 200, "final_url": "https://www.kaspersky.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "95f6695565e741729ec2b77206288669f5da7f17028b6cae8f62d563cce22168", "bytes_read": 44633 }, "llms_state": "candidate", "control": { "requested_url": "https://kaspersky.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:57.031860+00:00", "status": 404, "final_url": "https://www.kaspersky.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "b16e15764b8bc06c5c3f9f19bc8b99fa48e7894aa5a6ccdad65da49bbf564793", "bytes_read": 19 } }, { "rank": 142, "domain": "dns.google", "robots": { "requested_url": "https://dns.google/robots.txt", "observed_at": "2026-09-20T06:23:54.342057+00:00", "status": 404, "final_url": "https://dns.google/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2", "bytes_read": 1571 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://dns.google/llms.txt", "observed_at": "2026-09-20T06:23:54.593105+00:00", "status": 404, "final_url": "https://dns.google/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 143, "domain": "nih.gov", "robots": { "requested_url": "https://nih.gov/robots.txt", "observed_at": "2026-09-20T06:23:54.444896+00:00", "status": 200, "final_url": "https://www.nih.gov/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "773fb8d35bb9a39d35335ee6db8dc5c912d2aacbfb823152d9c61cd647dd902d", "bytes_read": 2027 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://nih.gov/llms.txt", "observed_at": "2026-09-20T06:23:54.997262+00:00", "status": 404, "final_url": "https://www.nih.gov/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "650abef9eda9ca23564e405be3497376b5f87644ce605b5ea5c32615856d75e1", "bytes_read": 546 }, "llms_state": "not_detected", "control": null }, { "rank": 144, "domain": "discord.gg", "robots": { "requested_url": "https://discord.gg/robots.txt", "observed_at": "2026-09-20T06:23:54.522786+00:00", "status": 200, "final_url": "https://discord.gg/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "16ceb5ee3e0dc13aa9adf31a3ebbe45a1d965b8c2b9f72eaf84e5911e140ed95", "bytes_read": 23 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://discord.gg/llms.txt", "observed_at": "2026-09-20T06:23:54.671308+00:00", "status": 200, "final_url": "https://discord.com/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "2367d0f578df3a2dc6a5105dd222522f42078424a61f2c03c2ab89d7dd76d58f", "bytes_read": 170241 }, "llms_state": "not_detected", "control": null }, { "rank": 145, "domain": "afternic.com", "robots": { "requested_url": "https://afternic.com/robots.txt", "observed_at": "2026-09-20T06:23:54.665591+00:00", "status": 200, "final_url": "https://www.afternic.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "e0a36ac62f8496ae1152654bc23729c1b005c5ca366ff2be745c928d84702129", "bytes_read": 85 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://afternic.com/llms.txt", "observed_at": "2026-09-20T06:23:55.763889+00:00", "status": 404, "final_url": "https://www.afternic.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "f51f5eda082b95849a8aa098bc57964aeb40ecb5dbe161963737c31cb649ee09", "bytes_read": 318389 }, "llms_state": "not_detected", "control": null }, { "rank": 146, "domain": "static.microsoft", "robots": { "requested_url": "https://static.microsoft/robots.txt", "observed_at": "2026-09-20T06:23:54.668617+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://static.microsoft/llms.txt", "observed_at": "2026-09-20T06:23:54.700193+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 147, "domain": "iiko.it", "robots": { "requested_url": "https://iiko.it/robots.txt", "observed_at": "2026-09-20T06:23:54.702305+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://iiko.it/llms.txt", "observed_at": "2026-09-20T06:23:55.781630+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 148, "domain": "forms.gle", "robots": { "requested_url": "https://forms.gle/robots.txt", "observed_at": "2026-09-20T06:23:54.834523+00:00", "status": 400, "final_url": "https://forms.gle/robots.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "bae5d66fbe371f66a934935346906d25bb35df6fb4cd5504a359cd324e89ea11", "bytes_read": 9258 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://forms.gle/llms.txt", "observed_at": "2026-09-20T06:23:55.070542+00:00", "status": 400, "final_url": "https://forms.gle/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "bae5d66fbe371f66a934935346906d25bb35df6fb4cd5504a359cd324e89ea11", "bytes_read": 9258 }, "llms_state": "unavailable", "control": null }, { "rank": 149, "domain": "miit.gov.cn", "robots": { "requested_url": "https://miit.gov.cn/robots.txt", "observed_at": "2026-09-20T06:23:55.008833+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://miit.gov.cn/llms.txt", "observed_at": "2026-09-20T06:23:56.503224+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 150, "domain": "azurefd.net", "robots": { "requested_url": "https://azurefd.net/robots.txt", "observed_at": "2026-09-20T06:23:55.082131+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://azurefd.net/llms.txt", "observed_at": "2026-09-20T06:23:55.130000+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 151, "domain": "archive.org", "robots": { "requested_url": "https://archive.org/robots.txt", "observed_at": "2026-09-20T06:23:55.134835+00:00", "status": 200, "final_url": "https://archive.org/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "83e00f46c71e9fa872694af73fa9c15a615b9052b2f10820a3d146bc506c6161", "bytes_read": 238 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://archive.org/llms.txt", "observed_at": "2026-09-20T06:23:55.496338+00:00", "status": 404, "final_url": "https://archive.org/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "91ee43d65d9cfb290e66bfaab0ca535c4015a3408bc77b6569bdb835ab4ae44c", "bytes_read": 4963 }, "llms_state": "not_detected", "control": null }, { "rank": 152, "domain": "xiaomi.com", "robots": { "requested_url": "https://xiaomi.com/robots.txt", "observed_at": "2026-09-20T06:23:55.145656+00:00", "status": 403, "final_url": "https://www.mi.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "696296ff43fb868aea25cb549aa486e33cb006e14992daa8cf9f86b2aadea48e", "bytes_read": 378 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://xiaomi.com/llms.txt", "observed_at": "2026-09-20T06:23:56.755904+00:00", "status": 403, "final_url": "https://www.mi.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "a9859a3412647bd15bccb94bf22f78deaac9d758b5ea2848302d2a224b4cbc4e", "bytes_read": 376 }, "llms_state": "unavailable", "control": null }, { "rank": 153, "domain": "godaddy.com", "robots": { "requested_url": "https://godaddy.com/robots.txt", "observed_at": "2026-09-20T06:23:55.251259+00:00", "status": 403, "final_url": "https://www.godaddy.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "82ba08afcbad01b07527dd945af20f0cfebf77b9546d7bc222c17fd2f5dffb2d", "bytes_read": 381 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://godaddy.com/llms.txt", "observed_at": "2026-09-20T06:23:55.624381+00:00", "status": 403, "final_url": "https://www.godaddy.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "ad9a0415ac7a51057f89c57b2cd9f234b7ff156bf60b3a53e6ca72a30ef97ce5", "bytes_read": 379 }, "llms_state": "unavailable", "control": null }, { "rank": 154, "domain": "one.one", "robots": { "requested_url": "https://one.one/robots.txt", "observed_at": "2026-09-20T06:23:55.270351+00:00", "status": 200, "final_url": "https://www.one.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "52000dbde5b05eaed9a35eb0af6c57000f7b76d36ab0163a09432c35ff7b61f3", "bytes_read": 450 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://one.one/llms.txt", "observed_at": "2026-09-20T06:23:56.579452+00:00", "status": 200, "final_url": "https://www.one.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "12ee377eaa605c61293ba5c795db1003f28116d9be56fcb81f72033962a17f57", "bytes_read": 3764 }, "llms_state": "candidate", "control": { "requested_url": "https://one.one/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:57.546888+00:00", "status": 404, "final_url": "https://www.one.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "fcf449b1f71848df5f0fd6e314bdbe0826389a77328386f40179d549273bbd36", "bytes_read": 15088 } }, { "rank": 155, "domain": "vkuserphoto.ru", "robots": { "requested_url": "https://vkuserphoto.ru/robots.txt", "observed_at": "2026-09-20T06:23:55.286160+00:00", "status": 418, "final_url": "https://ps.userapi.com/robots.txt", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://vkuserphoto.ru/llms.txt", "observed_at": "2026-09-20T06:23:56.745564+00:00", "status": 418, "final_url": "https://ps.userapi.com/llms.txt", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "unavailable", "control": null }, { "rank": 156, "domain": "steamserver.net", "robots": { "requested_url": "https://steamserver.net/robots.txt", "observed_at": "2026-09-20T06:23:55.354095+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://steamserver.net/llms.txt", "observed_at": "2026-09-20T06:23:55.394341+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 157, "domain": "aliyuncs.com", "robots": { "requested_url": "https://aliyuncs.com/robots.txt", "observed_at": "2026-09-20T06:23:55.396963+00:00", "status": 404, "final_url": "https://aliyuncs.com/robots.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "aa9baf69df772978540751afb4be49ef0d2426e8794d15b0b35221a182b20943", "bytes_read": 611 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://aliyuncs.com/llms.txt", "observed_at": "2026-09-20T06:23:56.437526+00:00", "status": 404, "final_url": "https://aliyuncs.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "197e899523d6baaa367d586334ff9783337c3177d63314682e88fa234b01ce5f", "bytes_read": 609 }, "llms_state": "not_detected", "control": null }, { "rank": 158, "domain": "gravatar.com", "robots": { "requested_url": "https://gravatar.com/robots.txt", "observed_at": "2026-09-20T06:23:55.557323+00:00", "status": 200, "final_url": "https://gravatar.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "75e9d90f44e5644b58fb6ec2361a61d48214863bd10d37b7fd6595b9b2f1bf48", "bytes_read": 1022 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://gravatar.com/llms.txt", "observed_at": "2026-09-20T06:23:55.799903+00:00", "status": 200, "final_url": "https://gravatar.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "c07c986048f9e393b5bbb07332446daf593a4102091a5863e53ab92bb47806e7", "bytes_read": 21927 }, "llms_state": "candidate", "control": { "requested_url": "https://gravatar.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:56.042969+00:00", "status": 404, "final_url": "https://gravatar.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "8612cb3cb17a0422aa253c6c6eee818c4326ec24d5b45c02795660b4c6dc97e5", "bytes_read": 52009 } }, { "rank": 159, "domain": "nytimes.com", "robots": { "requested_url": "https://nytimes.com/robots.txt", "observed_at": "2026-09-20T06:23:55.783472+00:00", "status": 200, "final_url": "https://www.nytimes.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "543e2314ebeee347c8aa9ef3b5d6f9fdf4d502995970d7767f0bc2596bdec985", "bytes_read": 8547 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://nytimes.com/llms.txt", "observed_at": "2026-09-20T06:23:56.250506+00:00", "status": 404, "final_url": "https://www.nytimes.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "e9e5717f2113fdd93ab6545310cac13501e00127141790c5d718686c46d166fe", "bytes_read": 68678 }, "llms_state": "not_detected", "control": null }, { "rank": 160, "domain": "msftconnecttest.com", "robots": { "requested_url": "https://msftconnecttest.com/robots.txt", "observed_at": "2026-09-20T06:23:55.834788+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://msftconnecttest.com/llms.txt", "observed_at": "2026-09-20T06:23:55.877719+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 161, "domain": "adtrafficquality.google", "robots": { "requested_url": "https://adtrafficquality.google/robots.txt", "observed_at": "2026-09-20T06:23:55.878851+00:00", "status": 404, "final_url": "https://adtrafficquality.google/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2", "bytes_read": 1571 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://adtrafficquality.google/llms.txt", "observed_at": "2026-09-20T06:23:56.173975+00:00", "status": 404, "final_url": "https://adtrafficquality.google/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 162, "domain": "tumblr.com", "robots": { "requested_url": "https://tumblr.com/robots.txt", "observed_at": "2026-09-20T06:23:55.879915+00:00", "status": 200, "final_url": "https://www.tumblr.com/robots.txt", "content_type": "text/plain;charset=utf-8", "truncated": false, "body_sha256": "2892c51fb7e06e7629ccb662ecf47dec3b0bfa6a679b4e112f25f7565ac1fcb0", "bytes_read": 3452 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://tumblr.com/llms.txt", "observed_at": "2026-09-20T06:23:56.317219+00:00", "status": 404, "final_url": "https://www.tumblr.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "f6e42682c0b61c38bb66ba03641c2f37ce7f3119e961938dc53eb821da45a1c1", "bytes_read": 4972 }, "llms_state": "not_detected", "control": null }, { "rank": 163, "domain": "jomodns.com", "robots": { "requested_url": "https://jomodns.com/robots.txt", "observed_at": "2026-09-20T06:23:56.323167+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://jomodns.com/llms.txt", "observed_at": "2026-09-20T06:23:56.596902+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 164, "domain": "applovin.com", "robots": { "requested_url": "https://applovin.com/robots.txt", "observed_at": "2026-09-20T06:23:56.410948+00:00", "status": 200, "final_url": "https://applovin.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "6da1b8bd9130995f8f99a4c1485e47c6c1766a49e51e26a92447b78d2af5bda3", "bytes_read": 218 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://applovin.com/llms.txt", "observed_at": "2026-09-20T06:23:56.620874+00:00", "status": 200, "final_url": "https://applovin.com/en", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "f9f1ac348624b9140554236efad43f37728baca6f9b4ce4d3550de6345e27902", "bytes_read": 384136 }, "llms_state": "not_detected", "control": null }, { "rank": 165, "domain": "tm-azurefd.net", "robots": { "requested_url": "https://tm-azurefd.net/robots.txt", "observed_at": "2026-09-20T06:23:56.505601+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://tm-azurefd.net/llms.txt", "observed_at": "2026-09-20T06:23:56.589892+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 166, "domain": "webex.com", "robots": { "requested_url": "https://webex.com/robots.txt", "observed_at": "2026-09-20T06:23:56.591821+00:00", "status": 403, "final_url": "https://www.webex.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "d41789557a2ca7e3c8f9f019efb8d8340d96ccf3beba613d3434e7303153fa9a", "bytes_read": 381 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://webex.com/llms.txt", "observed_at": "2026-09-20T06:23:57.357348+00:00", "status": 200, "final_url": "https://www.webex.com/llms.txt", "content_type": "text/plain; charset=ISO-8859-1", "truncated": false, "body_sha256": "9c6cfc41bb1fe1e65c3c7f7a2939efd0e223391a4c9f117adf9dbb391f23dec0", "bytes_read": 36071 }, "llms_state": "unknown_control", "control": { "requested_url": "https://webex.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:58.487956+00:00", "status": 403, "final_url": "https://www.webex.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "71c7626da5e7f3b4ace490bb3e66147bd7d47a493afc814b8160e1eb9a8ad5ad", "bytes_read": 431 } }, { "rank": 167, "domain": "mts.ru", "robots": { "requested_url": "https://mts.ru/robots.txt", "observed_at": "2026-09-20T06:23:56.596626+00:00", "status": 200, "final_url": "https://moskva.mts.ru/robotstxt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "46a6e3c31f4f92c111a22e76b16f18b540446d1aa9817c68130ab3c88e72193f", "bytes_read": 6839 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://mts.ru/llms.txt", "observed_at": "2026-09-20T06:23:58.987531+00:00", "status": 404, "final_url": "https://moskva.mts.ru/llmstxt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "af7aaa7a5b21583863457afe21a10b99cd2ffe6c455020d81c87e48c20155f44", "bytes_read": 194127 }, "llms_state": "not_detected", "control": null }, { "rank": 168, "domain": "discord.com", "robots": { "requested_url": "https://discord.com/robots.txt", "observed_at": "2026-09-20T06:23:56.599005+00:00", "status": 200, "final_url": "https://discord.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "fd74e7f8abd01d90ce80462af73d0896ed098dd0a61b20f24bf1137dd37979a5", "bytes_read": 1774 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://discord.com/llms.txt", "observed_at": "2026-09-20T06:23:56.744100+00:00", "status": 404, "final_url": "https://discord.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "e3cdaf0edb8e675b785c0f69043028c749603063dc2ddab6edeb65b3fd3578ea", "bytes_read": 44839 }, "llms_state": "not_detected", "control": null }, { "rank": 169, "domain": "paypal.com", "robots": { "requested_url": "https://paypal.com/robots.txt", "observed_at": "2026-09-20T06:23:56.668959+00:00", "status": 200, "final_url": "https://www.paypal.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "9c2f2f57e870e9b8406b1b738f0afbae6434f9991f24812890de2a6e317fc083", "bytes_read": 2628 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://paypal.com/llms.txt", "observed_at": "2026-09-20T06:23:57.203676+00:00", "status": 200, "final_url": "https://www.paypal.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1e1be425396507ad6c9bdecdd342c9ada72019ec9a0572ec40c5804122932fa8", "bytes_read": 40103 }, "llms_state": "candidate", "control": { "requested_url": "https://paypal.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:57.658376+00:00", "status": 404, "final_url": "https://www.paypalobjects.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "24d099beac3d63d4eeb021e28d21ace493071ed0b1268852fc22f92211547a25", "bytes_read": 1108 } }, { "rank": 170, "domain": "dns-parking.com", "robots": { "requested_url": "https://dns-parking.com/robots.txt", "observed_at": "2026-09-20T06:23:56.748900+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://dns-parking.com/llms.txt", "observed_at": "2026-09-20T06:23:56.835346+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 171, "domain": "ax-msedge.net", "robots": { "requested_url": "https://ax-msedge.net/robots.txt", "observed_at": "2026-09-20T06:23:56.837298+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ax-msedge.net/llms.txt", "observed_at": "2026-09-20T06:23:56.907357+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 172, "domain": "example.com", "robots": { "requested_url": "https://example.com/robots.txt", "observed_at": "2026-09-20T06:23:56.909117+00:00", "status": 404, "final_url": "https://example.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "ff67a9d764d6a2367a187734e697f6a53217db9a21c101d410a113ca871a299d", "bytes_read": 559 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://example.com/llms.txt", "observed_at": "2026-09-20T06:23:57.117482+00:00", "status": 404, "final_url": "https://example.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "ff67a9d764d6a2367a187734e697f6a53217db9a21c101d410a113ca871a299d", "bytes_read": 559 }, "llms_state": "not_detected", "control": null }, { "rank": 173, "domain": "vedcdnlb.com", "robots": { "requested_url": "https://vedcdnlb.com/robots.txt", "observed_at": "2026-09-20T06:23:57.027577+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://vedcdnlb.com/llms.txt", "observed_at": "2026-09-20T06:23:57.846191+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 174, "domain": "3gppnetwork.org", "robots": { "requested_url": "https://3gppnetwork.org/robots.txt", "observed_at": "2026-09-20T06:23:57.203003+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://3gppnetwork.org/llms.txt", "observed_at": "2026-09-20T06:23:57.264506+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 175, "domain": "adobe.io", "robots": { "requested_url": "https://adobe.io/robots.txt", "observed_at": "2026-09-20T06:23:57.254015+00:00", "status": 200, "final_url": "https://developer.adobe.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "7b9b15178930b581386131eff19e6e51fdf72a1c7cc8dc9c545cdb1f6fc53786", "bytes_read": 92 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://adobe.io/llms.txt", "observed_at": "2026-09-20T06:23:58.216104+00:00", "status": 404, "final_url": "https://developer.adobe.com/llms.txt", "content_type": "", "truncated": false, "body_sha256": "d206e70de352d5dcff681d779fa2ec08816f6916b99f0cc717e07cedde448700", "bytes_read": 2592 }, "llms_state": "not_detected", "control": null }, { "rank": 176, "domain": "spectrum.com", "robots": { "requested_url": "https://spectrum.com/robots.txt", "observed_at": "2026-09-20T06:23:57.266556+00:00", "status": null, "error": "TimeoutError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://spectrum.com/llms.txt", "observed_at": "2026-09-20T06:24:07.899507+00:00", "status": null, "error": "TimeoutError" }, "llms_state": "unavailable", "control": null }, { "rank": 177, "domain": "azurewebsites.net", "robots": { "requested_url": "https://azurewebsites.net/robots.txt", "observed_at": "2026-09-20T06:23:57.708443+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://azurewebsites.net/llms.txt", "observed_at": "2026-09-20T06:23:57.770869+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 178, "domain": "vk.ru", "robots": { "requested_url": "https://vk.ru/robots.txt", "observed_at": "2026-09-20T06:23:57.773048+00:00", "status": 200, "final_url": "https://vk.ru/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "573219d395c1aa085175955ea564938f81e5c31b8cd52ba15d49a2171ffb06c4", "bytes_read": 6175 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://vk.ru/llms.txt", "observed_at": "2026-09-20T06:23:58.435900+00:00", "status": 404, "final_url": "https://vk.ru/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "02df22b1683f5dfbe9b9b06e4a1332d2be7acc48714baa1c504edacfbaa97f39", "bytes_read": 13120 }, "llms_state": "not_detected", "control": null }, { "rank": 179, "domain": "ggpht.com", "robots": { "requested_url": "https://ggpht.com/robots.txt", "observed_at": "2026-09-20T06:23:57.843152+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ggpht.com/llms.txt", "observed_at": "2026-09-20T06:23:57.890772+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 180, "domain": "shopify.com", "robots": { "requested_url": "https://shopify.com/robots.txt", "observed_at": "2026-09-20T06:23:57.849672+00:00", "status": 200, "final_url": "https://shopify.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "a55c031afe721803d5e722c76910b8b569511cce2fc45459933c60d8e0764aa7", "bytes_read": 62 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://shopify.com/llms.txt", "observed_at": "2026-09-20T06:23:58.030932+00:00", "status": 200, "final_url": "https://www.shopify.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "fd5d372a21847ff2333682645367699f10c44ded4a1dbf08d8e93d512beb765a", "bytes_read": 12520 }, "llms_state": "candidate", "control": { "requested_url": "https://shopify.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:23:58.576451+00:00", "status": 404, "final_url": "https://www.shopify.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "f50b199548b1df9325b0480dc3080b5cec14a8d271aaf4283d37ef995e6de052", "bytes_read": 11084 } }, { "rank": 181, "domain": "telekom.net", "robots": { "requested_url": "https://telekom.net/robots.txt", "observed_at": "2026-09-20T06:23:57.893473+00:00", "status": 200, "final_url": "https://www.telekom.com/", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "f3a651cc35c82933172715cff8750e04839abf6f340bef5b2bc14294570efdc3", "bytes_read": 221702 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://telekom.net/llms.txt", "observed_at": "2026-09-20T06:24:00.212837+00:00", "status": 200, "final_url": "https://www.telekom.com/", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "f3a651cc35c82933172715cff8750e04839abf6f340bef5b2bc14294570efdc3", "bytes_read": 221702 }, "llms_state": "not_detected", "control": null }, { "rank": 182, "domain": "base.org", "robots": { "requested_url": "https://base.org/robots.txt", "observed_at": "2026-09-20T06:23:58.344316+00:00", "status": null, "error": "TimeoutError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://base.org/llms.txt", "observed_at": "2026-09-20T06:24:08.678992+00:00", "status": null, "error": "TimeoutError" }, "llms_state": "unavailable", "control": null }, { "rank": 183, "domain": "hosting24.com", "robots": { "requested_url": "https://hosting24.com/robots.txt", "observed_at": "2026-09-20T06:23:58.488330+00:00", "status": 200, "final_url": "https://www.hostinger.com/", "content_type": "text/html", "truncated": true, "body_sha256": "5ac203f5fa68ad20d62e16ffb738c9ab89c88e8054ddb552cc3371b59d2a8cfc", "bytes_read": 512001 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://hosting24.com/llms.txt", "observed_at": "2026-09-20T06:23:59.176549+00:00", "status": 200, "final_url": "https://www.hostinger.com/", "content_type": "text/html", "truncated": true, "body_sha256": "5ac203f5fa68ad20d62e16ffb738c9ab89c88e8054ddb552cc3371b59d2a8cfc", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 184, "domain": "hichina.com", "robots": { "requested_url": "https://hichina.com/robots.txt", "observed_at": "2026-09-20T06:23:58.629710+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://hichina.com/llms.txt", "observed_at": "2026-09-20T06:23:58.865215+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 185, "domain": "doctolib.net", "robots": { "requested_url": "https://doctolib.net/robots.txt", "observed_at": "2026-09-20T06:23:58.763939+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://doctolib.net/llms.txt", "observed_at": "2026-09-20T06:23:58.793762+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 186, "domain": "paytmfirstgames.com", "robots": { "requested_url": "https://paytmfirstgames.com/robots.txt", "observed_at": "2026-09-20T06:23:58.797620+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://paytmfirstgames.com/llms.txt", "observed_at": "2026-09-20T06:23:58.928823+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 187, "domain": "cdn-apple.com", "robots": { "requested_url": "https://cdn-apple.com/robots.txt", "observed_at": "2026-09-20T06:23:58.873254+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cdn-apple.com/llms.txt", "observed_at": "2026-09-20T06:23:58.902805+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 188, "domain": "macromedia.com", "robots": { "requested_url": "https://macromedia.com/robots.txt", "observed_at": "2026-09-20T06:23:58.906489+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://macromedia.com/llms.txt", "observed_at": "2026-09-20T06:24:08.945829+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 189, "domain": "soundcloud.com", "robots": { "requested_url": "https://soundcloud.com/robots.txt", "observed_at": "2026-09-20T06:23:58.922442+00:00", "status": 200, "final_url": "https://soundcloud.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "7bbee7524de61fe4a1b4e1701bd55e9dde6ad96e3e5e97e04001b128ea890998", "bytes_read": 1441 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://soundcloud.com/llms.txt", "observed_at": "2026-09-20T06:23:59.171493+00:00", "status": 404, "final_url": "https://soundcloud.com/llms.txt", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "not_detected", "control": null }, { "rank": 190, "domain": "googleadservices.com", "robots": { "requested_url": "https://googleadservices.com/robots.txt", "observed_at": "2026-09-20T06:23:58.931264+00:00", "status": 404, "final_url": "https://googleadservices.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2", "bytes_read": 1571 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://googleadservices.com/llms.txt", "observed_at": "2026-09-20T06:23:59.258491+00:00", "status": 404, "final_url": "https://googleadservices.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 191, "domain": "flickr.com", "robots": { "requested_url": "https://flickr.com/robots.txt", "observed_at": "2026-09-20T06:23:58.989447+00:00", "status": 200, "final_url": "https://flickr.com/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "7f0af7ac6cba1389d7f1271821f9c4330b258c1e387803936a00442d19aae402", "bytes_read": 2130 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://flickr.com/llms.txt", "observed_at": "2026-09-20T06:23:59.285332+00:00", "status": 404, "final_url": "https://www.flickr.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "0ac454fa23c1fa86155f26e469b647b1da02f7758f0da4c3cec0fcdfdf9a1c12", "bytes_read": 197322 }, "llms_state": "not_detected", "control": null }, { "rank": 192, "domain": "w3.org", "robots": { "requested_url": "https://w3.org/robots.txt", "observed_at": "2026-09-20T06:23:58.992533+00:00", "status": 200, "final_url": "https://www.w3.org/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "f3947d244239d90298c7c3323a6d83042bb4f38c821f1bdefa449e7dfd9e2976", "bytes_read": 3812 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://w3.org/llms.txt", "observed_at": "2026-09-20T06:23:59.366896+00:00", "status": 404, "final_url": "https://www.w3.org/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "c13453619f8936463dc50c5538bd957779fd22bff15211efc6251bb71d6d8fcc", "bytes_read": 21638 }, "llms_state": "not_detected", "control": null }, { "rank": 193, "domain": "adsafeprotected.com", "robots": { "requested_url": "https://adsafeprotected.com/robots.txt", "observed_at": "2026-09-20T06:23:59.174877+00:00", "status": 200, "final_url": "https://adsafeprotected.com/skeleton.gif", "content_type": "image/gif", "truncated": false, "body_sha256": "42b976597a2d977d0e300f6d06bc903db389e5c112d33c1c8c249690a522d9f2", "bytes_read": 43 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://adsafeprotected.com/llms.txt", "observed_at": "2026-09-20T06:23:59.531832+00:00", "status": 200, "final_url": "https://adsafeprotected.com/skeleton.gif", "content_type": "image/gif", "truncated": false, "body_sha256": "42b976597a2d977d0e300f6d06bc903db389e5c112d33c1c8c249690a522d9f2", "bytes_read": 43 }, "llms_state": "not_detected", "control": null }, { "rank": 194, "domain": "namebrightdns.com", "robots": { "requested_url": "https://namebrightdns.com/robots.txt", "observed_at": "2026-09-20T06:23:59.390153+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://namebrightdns.com/llms.txt", "observed_at": "2026-09-20T06:23:59.471277+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 195, "domain": "nflxso.net", "robots": { "requested_url": "https://nflxso.net/robots.txt", "observed_at": "2026-09-20T06:23:59.471972+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://nflxso.net/llms.txt", "observed_at": "2026-09-20T06:23:59.526361+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 196, "domain": "lencr.org", "robots": { "requested_url": "https://lencr.org/robots.txt", "observed_at": "2026-09-20T06:23:59.474210+00:00", "status": 200, "final_url": "https://letsencrypt.org/docs/lencr.org/", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "63e1dbdb955b4ed70a8391ee1f3bc07b9de2515b3cbef576db4278dab1b071bc", "bytes_read": 28865 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://lencr.org/llms.txt", "observed_at": "2026-09-20T06:24:00.250386+00:00", "status": 200, "final_url": "https://letsencrypt.org/docs/lencr.org/", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "63e1dbdb955b4ed70a8391ee1f3bc07b9de2515b3cbef576db4278dab1b071bc", "bytes_read": 28865 }, "llms_state": "not_detected", "control": null }, { "rank": 197, "domain": "bytefcdn-oversea.com", "robots": { "requested_url": "https://bytefcdn-oversea.com/robots.txt", "observed_at": "2026-09-20T06:23:59.528341+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://bytefcdn-oversea.com/llms.txt", "observed_at": "2026-09-20T06:23:59.596717+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 198, "domain": "rubiconproject.com", "robots": { "requested_url": "https://rubiconproject.com/robots.txt", "observed_at": "2026-09-20T06:23:59.603228+00:00", "status": 200, "final_url": "https://www.rubiconproject.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "85fb7c9f9f8d912342d9598fc395a8a1674d5c5384bbbfa28c23fcc3f15f3ad8", "bytes_read": 189 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://rubiconproject.com/llms.txt", "observed_at": "2026-09-20T06:24:00.049599+00:00", "status": 200, "final_url": "https://www.rubiconproject.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "01bf6470ee4c6fa792156a2fa2474f199575a70b5a839e53224fc7c0a2cc06f2", "bytes_read": 6641 }, "llms_state": "candidate", "control": { "requested_url": "https://rubiconproject.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:00.433387+00:00", "status": 404, "final_url": "https://www.rubiconproject.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 } }, { "rank": 199, "domain": "creativecommons.org", "robots": { "requested_url": "https://creativecommons.org/robots.txt", "observed_at": "2026-09-20T06:23:59.616978+00:00", "status": 200, "final_url": "https://creativecommons.org/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "d7a6362bab284aaa0b08d19dcbd455a2054914002c9859e14061e15e071f4705", "bytes_read": 177 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://creativecommons.org/llms.txt", "observed_at": "2026-09-20T06:23:59.827438+00:00", "status": 404, "final_url": "https://creativecommons.org/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "0ce0051a879b45fd2a5d5d2015383b60bad0e4bbf622c6eca47f8b739263a430", "bytes_read": 32162 }, "llms_state": "not_detected", "control": null }, { "rank": 200, "domain": "duckdns.org", "robots": { "requested_url": "https://duckdns.org/robots.txt", "observed_at": "2026-09-20T06:23:59.658081+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://duckdns.org/llms.txt", "observed_at": "2026-09-20T06:24:09.742604+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 201, "domain": "medium.com", "robots": { "requested_url": "https://medium.com/robots.txt", "observed_at": "2026-09-20T06:23:59.784417+00:00", "status": 200, "final_url": "https://medium.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "fe87c9fc79edeba118b4f5184b2472913f4057172783cf4bd4dae7c997c16e74", "bytes_read": 884 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://medium.com/llms.txt", "observed_at": "2026-09-20T06:24:00.012253+00:00", "status": 404, "final_url": "https://medium.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "631ee223e82d786cd7be91b133f15159a57a7f164c74ba73ff06aa243c0f90dc", "bytes_read": 77657 }, "llms_state": "not_detected", "control": null }, { "rank": 202, "domain": "cnn.com", "robots": { "requested_url": "https://cnn.com/robots.txt", "observed_at": "2026-09-20T06:23:59.805795+00:00", "status": 200, "final_url": "https://www.cnn.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "4eed8bbc909b9bc8816b2974b4cdcd4d7c9f15b620a40e31cb0f6452aae32aa0", "bytes_read": 3456 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://cnn.com/llms.txt", "observed_at": "2026-09-20T06:24:00.226276+00:00", "status": 404, "final_url": "https://www.cnn.com/llms.txt", "content_type": "text/html", "truncated": true, "body_sha256": "e9e857c84e5e409fecd9fe905e6f80b0f7755b9fae1fa99ee54643ee098b18a2", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 203, "domain": "vungle.com", "robots": { "requested_url": "https://vungle.com/robots.txt", "observed_at": "2026-09-20T06:23:59.983775+00:00", "status": 200, "final_url": "https://vungle.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "3b58901ca623912285b3383a045337e7af98f930d6836a30fe11f27f9f8a19c5", "bytes_read": 82 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://vungle.com/llms.txt", "observed_at": "2026-09-20T06:24:00.190090+00:00", "status": 404, "final_url": "https://vungle.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "d8531a1ac331414d357ac014ac8813b93c2c0ea1756542ad8bacb6d352ed4891", "bytes_read": 60 }, "llms_state": "not_detected", "control": null }, { "rank": 204, "domain": "theguardian.com", "robots": { "requested_url": "https://theguardian.com/robots.txt", "observed_at": "2026-09-20T06:24:00.187435+00:00", "status": 200, "final_url": "https://www.theguardian.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "3387cd8d393a0c257e687ce63fcf30558ada1eb1fc167bfc51cb216e29005db2", "bytes_read": 2873 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://theguardian.com/llms.txt", "observed_at": "2026-09-20T06:24:00.654987+00:00", "status": 404, "final_url": "https://www.theguardian.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "5a504aff032fe8faab92965fc40e0c4bcaa7709d90347f0d14415572e6a0586a", "bytes_read": 24345 }, "llms_state": "not_detected", "control": null }, { "rank": 205, "domain": "ttdns2.com", "robots": { "requested_url": "https://ttdns2.com/robots.txt", "observed_at": "2026-09-20T06:24:00.432506+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ttdns2.com/llms.txt", "observed_at": "2026-09-20T06:24:00.488526+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 206, "domain": "ebay.com", "robots": { "requested_url": "https://ebay.com/robots.txt", "observed_at": "2026-09-20T06:24:00.490353+00:00", "status": 200, "final_url": "https://www.ebay.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "221797cd7a314aeba19bb83d7868af3ea04325e4f2f8fd9b8ba465649e091970", "bytes_read": 24164 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://ebay.com/llms.txt", "observed_at": "2026-09-20T06:24:01.678823+00:00", "status": 404, "final_url": "https://pages.ebay.com/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "6ce11de4922dd9d3a64383576a51e53afa647f542d116be7f7b8ccd46c99f08a", "bytes_read": 73587 }, "llms_state": "not_detected", "control": null }, { "rank": 207, "domain": "oracle.com", "robots": { "requested_url": "https://oracle.com/robots.txt", "observed_at": "2026-09-20T06:24:00.807100+00:00", "status": 403, "final_url": "https://www.oracle.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "24ee2e32771082a65c9d093bf734f26e4856b9a2c0418075916b8ab78c23c979", "bytes_read": 1450 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://oracle.com/llms.txt", "observed_at": "2026-09-20T06:24:01.740642+00:00", "status": 403, "final_url": "https://www.oracle.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "000b11c4a3e2a38424cc778e46e722b2538d12dfda8d3a69e9eb8267eabd6d0d", "bytes_read": 1450 }, "llms_state": "unavailable", "control": null }, { "rank": 208, "domain": "taboola.com", "robots": { "requested_url": "https://taboola.com/robots.txt", "observed_at": "2026-09-20T06:24:00.823737+00:00", "status": 200, "final_url": "https://www.taboola.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e87ccc5901966b901c2c6b160cbb9b0ca10b587361d00edf291cc4b61a3251a7", "bytes_read": 2350 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://taboola.com/llms.txt", "observed_at": "2026-09-20T06:24:01.360975+00:00", "status": 200, "final_url": "https://www.taboola.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "f5fd95f07de69a9beed7de943be2e2ab0c8b9480689d7b2df938ba46dfc9a657", "bytes_read": 276297 }, "llms_state": "candidate", "control": { "requested_url": "https://taboola.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:01.947958+00:00", "status": 404, "final_url": "https://www.taboola.com/page-not-found/?ref=%2Fsuede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "cb533b864d71c248fa56bf90c410fb3d57d0e3f32950b147ee9398e41d8414a5", "bytes_read": 66806 } }, { "rank": 209, "domain": "shifen.com", "robots": { "requested_url": "https://shifen.com/robots.txt", "observed_at": "2026-09-20T06:24:00.988466+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://shifen.com/llms.txt", "observed_at": "2026-09-20T06:24:11.309688+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 210, "domain": "lsrelayaccess.com", "robots": { "requested_url": "https://lsrelayaccess.com/robots.txt", "observed_at": "2026-09-20T06:24:00.989336+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://lsrelayaccess.com/llms.txt", "observed_at": "2026-09-20T06:24:21.573445+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 211, "domain": "t-online.de", "robots": { "requested_url": "https://t-online.de/robots.txt", "observed_at": "2026-09-20T06:24:01.008931+00:00", "status": 200, "final_url": "https://www.t-online.de/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "aefcf62006193c765ed22c8c6e46711397b52d962c0875c6c43e1cdae0b4153c", "bytes_read": 2206 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://t-online.de/llms.txt", "observed_at": "2026-09-20T06:24:01.857263+00:00", "status": 404, "final_url": "https://www.t-online.de/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "f5779fc3e16db06405216fae8af6b7a5f7e81c3c979fe46142fb50d7cb895dbc", "bytes_read": 13425 }, "llms_state": "not_detected", "control": null }, { "rank": 212, "domain": "ksyuncdn.com", "robots": { "requested_url": "https://ksyuncdn.com/robots.txt", "observed_at": "2026-09-20T06:24:01.084408+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ksyuncdn.com/llms.txt", "observed_at": "2026-09-20T06:24:01.374009+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 213, "domain": "doi.org", "robots": { "requested_url": "https://doi.org/robots.txt", "observed_at": "2026-09-20T06:24:01.319435+00:00", "status": 404, "final_url": "https://www.doi.org/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "9518442b21a1c248277789569b6e13e0136300b26ec11740de8034d7601d5f9e", "bytes_read": 7625 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://doi.org/llms.txt", "observed_at": "2026-09-20T06:24:01.721858+00:00", "status": 404, "final_url": "https://www.doi.org/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "9518442b21a1c248277789569b6e13e0136300b26ec11740de8034d7601d5f9e", "bytes_read": 7625 }, "llms_state": "not_detected", "control": null }, { "rank": 214, "domain": "youtube-nocookie.com", "robots": { "requested_url": "https://youtube-nocookie.com/robots.txt", "observed_at": "2026-09-20T06:24:01.375855+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://youtube-nocookie.com/llms.txt", "observed_at": "2026-09-20T06:24:01.530146+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 215, "domain": "ailawandorder.com", "robots": { "requested_url": "https://ailawandorder.com/robots.txt", "observed_at": "2026-09-20T06:24:01.633272+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ailawandorder.com/llms.txt", "observed_at": "2026-09-20T06:24:02.228545+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 216, "domain": "capcutapi.com", "robots": { "requested_url": "https://capcutapi.com/robots.txt", "observed_at": "2026-09-20T06:24:02.369582+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://capcutapi.com/llms.txt", "observed_at": "2026-09-20T06:24:02.425355+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 217, "domain": "bbc.com", "robots": { "requested_url": "https://bbc.com/robots.txt", "observed_at": "2026-09-20T06:24:02.427009+00:00", "status": 200, "final_url": "https://www.bbc.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "34a6212c178b5845816e4e5b00245d758290ee8109b4507fda890b393ce0b52e", "bytes_read": 6130 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://bbc.com/llms.txt", "observed_at": "2026-09-20T06:24:03.106295+00:00", "status": 404, "final_url": "https://www.bbc.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "1d76e54236d324a5855ecc22ec9b25267ddcef008983b3e4773bc89172e5d24f", "bytes_read": 32428 }, "llms_state": "not_detected", "control": null }, { "rank": 218, "domain": "forbes.com", "robots": { "requested_url": "https://forbes.com/robots.txt", "observed_at": "2026-09-20T06:24:02.494095+00:00", "status": 200, "final_url": "https://www.forbes.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "29e98ab89e88236d171f529a26cb95984f1021926f62c26e9a0656cebf364b5d", "bytes_read": 33649 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://forbes.com/llms.txt", "observed_at": "2026-09-20T06:24:02.954303+00:00", "status": 404, "final_url": "https://www.forbes.com/llms.txt", "content_type": "text/html", "truncated": true, "body_sha256": "d604e01807fb98963ec13d7faf616b135e8c8e227045182ab3982d65a3fdc431", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 219, "domain": "yandexcloud.net", "robots": { "requested_url": "https://yandexcloud.net/robots.txt", "observed_at": "2026-09-20T06:24:02.786838+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://yandexcloud.net/llms.txt", "observed_at": "2026-09-20T06:24:02.973202+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 220, "domain": "fastly-edge.com", "robots": { "requested_url": "https://fastly-edge.com/robots.txt", "observed_at": "2026-09-20T06:24:02.879031+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://fastly-edge.com/llms.txt", "observed_at": "2026-09-20T06:24:02.945243+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 221, "domain": "sciencedirect.com", "robots": { "requested_url": "https://sciencedirect.com/robots.txt", "observed_at": "2026-09-20T06:24:02.915052+00:00", "status": 403, "final_url": "https://www.sciencedirect.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": true, "body_sha256": "b89d72ac2f0f06a404ab8f74920a621414b29c3ab6c41a2d55b572a02a706426", "bytes_read": 512001 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://sciencedirect.com/llms.txt", "observed_at": "2026-09-20T06:24:03.377072+00:00", "status": 403, "final_url": "https://www.sciencedirect.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": true, "body_sha256": "b89d72ac2f0f06a404ab8f74920a621414b29c3ab6c41a2d55b572a02a706426", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 222, "domain": "sfx.ms", "robots": { "requested_url": "https://sfx.ms/robots.txt", "observed_at": "2026-09-20T06:24:02.953303+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://sfx.ms/llms.txt", "observed_at": "2026-09-20T06:24:02.978908+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 223, "domain": "cdngslb.com", "robots": { "requested_url": "https://cdngslb.com/robots.txt", "observed_at": "2026-09-20T06:24:02.974761+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cdngslb.com/llms.txt", "observed_at": "2026-09-20T06:24:13.091222+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 224, "domain": "twitch.tv", "robots": { "requested_url": "https://twitch.tv/robots.txt", "observed_at": "2026-09-20T06:24:02.980335+00:00", "status": 200, "final_url": "https://www.twitch.tv/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "3bf83d9ad4057c9f328fab1cf3b6aa74d3bc7e3c5125e0f1ace3b27a36c04da6", "bytes_read": 1167 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://twitch.tv/llms.txt", "observed_at": "2026-09-20T06:24:03.549956+00:00", "status": 200, "final_url": "https://www.twitch.tv/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "96f0309232e73a9b3a40044334d5bfe8c0f58872489bd16c506de8d5ae32438f", "bytes_read": 197025 }, "llms_state": "not_detected", "control": null }, { "rank": 225, "domain": "researchgate.net", "robots": { "requested_url": "https://researchgate.net/robots.txt", "observed_at": "2026-09-20T06:24:03.190556+00:00", "status": 200, "final_url": "https://www.researchgate.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "4b9963df3f3529b6ee2b812decf2dadfde6d5eb041f547401dcf4915ac4bf1cb", "bytes_read": 293 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://researchgate.net/llms.txt", "observed_at": "2026-09-20T06:24:03.551368+00:00", "status": 403, "final_url": "https://www.researchgate.net/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "abc753a5c69b13535bb66dab0d1b40418e90be8612f695adc256087845d160bd", "bytes_read": 23047 }, "llms_state": "unavailable", "control": null }, { "rank": 226, "domain": "akamaized.net", "robots": { "requested_url": "https://akamaized.net/robots.txt", "observed_at": "2026-09-20T06:24:03.475081+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://akamaized.net/llms.txt", "observed_at": "2026-09-20T06:24:03.525103+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 227, "domain": "pages.dev", "robots": { "requested_url": "https://pages.dev/robots.txt", "observed_at": "2026-09-20T06:24:03.526696+00:00", "status": 200, "final_url": "https://www.cloudflare.com/products/pages/", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "a89c2e5de07c1ebad3b8cc867ff1e4963ff10cb37bb58b95a6f5d889d79d8146", "bytes_read": 512001 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://pages.dev/llms.txt", "observed_at": "2026-09-20T06:24:04.556616+00:00", "status": 200, "final_url": "https://www.cloudflare.com/products/pages/", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "4b158d710c944f07228ea3961ed9e186cb3e8056ffb73d7a147ead51558b3e63", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 228, "domain": "gmail.com", "robots": { "requested_url": "https://gmail.com/robots.txt", "observed_at": "2026-09-20T06:24:03.637561+00:00", "status": 200, "final_url": "https://gmail.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "60326aa3f254024b81aa351c7ef22579798612dc2f1bf7c755ed671b44fcb4d2", "bytes_read": 129 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://gmail.com/llms.txt", "observed_at": "2026-09-20T06:24:03.961031+00:00", "status": 404, "final_url": "https://gmail.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 229, "domain": "mit.edu", "robots": { "requested_url": "https://mit.edu/robots.txt", "observed_at": "2026-09-20T06:24:03.867429+00:00", "status": 200, "final_url": "https://web.mit.edu/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "bad4e8ead1b0414e6d9f15a570ae195b73c69f2fd02e3e2516e2991289675a47", "bytes_read": 199 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://mit.edu/llms.txt", "observed_at": "2026-09-20T06:24:04.680027+00:00", "status": 404, "final_url": "https://web.mit.edu/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "168c3da82cb9b3f68dcdf04c29761893dd113ebfb16a9e87c21698d681459c21", "bytes_read": 25731 }, "llms_state": "not_detected", "control": null }, { "rank": 230, "domain": "ozon.ru", "robots": { "requested_url": "https://ozon.ru/robots.txt", "observed_at": "2026-09-20T06:24:03.944898+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ozon.ru/llms.txt", "observed_at": "2026-09-20T06:24:24.623167+00:00", "status": 307, "final_url": "https://ozon.ru/llms.txt?__rr=9", "content_type": "text/html", "truncated": false, "body_sha256": "8f03491247cbfa8a2e60e0f7ec62d63b5070659f60383a1c81abeb2b20221be3", "bytes_read": 164 }, "llms_state": "unavailable", "control": null }, { "rank": 231, "domain": "ubuntu.com", "robots": { "requested_url": "https://ubuntu.com/robots.txt", "observed_at": "2026-09-20T06:24:03.953908+00:00", "status": 200, "final_url": "https://ubuntu.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "16e26f4cb730b111c334f112f075115ef4a50cf8df110581f3d4ee032a47cca9", "bytes_read": 5511 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [] } }, "llms": { "requested_url": "https://ubuntu.com/llms.txt", "observed_at": "2026-09-20T06:24:04.436508+00:00", "status": 200, "final_url": "https://ubuntu.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "73127cc2f34665b4661f5405ac319b54b0556db4ecfce6a773d8acbe71e79274", "bytes_read": 12879 }, "llms_state": "candidate", "control": { "requested_url": "https://ubuntu.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:04.978413+00:00", "status": 404, "final_url": "https://ubuntu.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "7dddeddfac15d311c10bd65df518af935c58b78903839dad31261c156f0ed404", "bytes_read": 56605 } }, { "rank": 232, "domain": "userapi.com", "robots": { "requested_url": "https://userapi.com/robots.txt", "observed_at": "2026-09-20T06:24:04.089748+00:00", "status": 200, "final_url": "https://vk.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "573219d395c1aa085175955ea564938f81e5c31b8cd52ba15d49a2171ffb06c4", "bytes_read": 6175 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://userapi.com/llms.txt", "observed_at": "2026-09-20T06:24:05.392537+00:00", "status": 404, "final_url": "https://vk.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "02df22b1683f5dfbe9b9b06e4a1332d2be7acc48714baa1c504edacfbaa97f39", "bytes_read": 13120 }, "llms_state": "not_detected", "control": null }, { "rank": 233, "domain": "pki.goog", "robots": { "requested_url": "https://pki.goog/robots.txt", "observed_at": "2026-09-20T06:24:04.515378+00:00", "status": 200, "final_url": "https://pki.goog/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "18bec4c62e19bead2c2e6e19a36c1ba6be280c92ca6a3ad4230c3f1c9c7b2bb4", "bytes_read": 69 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://pki.goog/llms.txt", "observed_at": "2026-09-20T06:24:04.848838+00:00", "status": 404, "final_url": "https://pki.goog/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 234, "domain": "nr-data.net", "robots": { "requested_url": "https://nr-data.net/robots.txt", "observed_at": "2026-09-20T06:24:04.682377+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://nr-data.net/llms.txt", "observed_at": "2026-09-20T06:24:04.713562+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 235, "domain": "azureedge.net", "robots": { "requested_url": "https://azureedge.net/robots.txt", "observed_at": "2026-09-20T06:24:04.715666+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://azureedge.net/llms.txt", "observed_at": "2026-09-20T06:24:04.744108+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 236, "domain": "bbc.co.uk", "robots": { "requested_url": "https://bbc.co.uk/robots.txt", "observed_at": "2026-09-20T06:24:04.746035+00:00", "status": 200, "final_url": "https://www.bbc.co.uk/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "0d588e695ee40aa6310e79289f06d56d8003e2b448144b300e9682b26110ccff", "bytes_read": 4907 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://bbc.co.uk/llms.txt", "observed_at": "2026-09-20T06:24:05.339335+00:00", "status": 404, "final_url": "https://www.bbc.co.uk/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "1d76e54236d324a5855ecc22ec9b25267ddcef008983b3e4773bc89172e5d24f", "bytes_read": 32428 }, "llms_state": "not_detected", "control": null }, { "rank": 237, "domain": "miui.com", "robots": { "requested_url": "https://miui.com/robots.txt", "observed_at": "2026-09-20T06:24:05.068948+00:00", "status": 404, "final_url": "https://miui.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "91fe1bb2ce31dfa39896c9a04ee23274a7e9dd0ad1fc91ec191e8b7307f652f2", "bytes_read": 169 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://miui.com/llms.txt", "observed_at": "2026-09-20T06:24:06.948626+00:00", "status": 404, "final_url": "https://miui.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "91fe1bb2ce31dfa39896c9a04ee23274a7e9dd0ad1fc91ec191e8b7307f652f2", "bytes_read": 169 }, "llms_state": "not_detected", "control": null }, { "rank": 238, "domain": "nist.gov", "robots": { "requested_url": "https://nist.gov/robots.txt", "observed_at": "2026-09-20T06:24:05.074524+00:00", "status": 200, "final_url": "https://www.nist.gov/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "0fed8806709b6ff2716921723b5077d1c19d152f760db983af60988cae0031dd", "bytes_read": 2425 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://nist.gov/llms.txt", "observed_at": "2026-09-20T06:24:05.559834+00:00", "status": 404, "final_url": "https://www.nist.gov/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "9f2a00f25e72882b5eccd6317bec66ab599e54b2cd6d4ff92bb6552d871b8f06", "bytes_read": 547 }, "llms_state": "not_detected", "control": null }, { "rank": 239, "domain": "stripe.com", "robots": { "requested_url": "https://stripe.com/robots.txt", "observed_at": "2026-09-20T06:24:05.340870+00:00", "status": 200, "final_url": "https://stripe.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "23166bdd2477d69a2fbd405ae5901154a36f5de79876de4616494458fb340002", "bytes_read": 643 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://stripe.com/llms.txt", "observed_at": "2026-09-20T06:24:05.725633+00:00", "status": 200, "final_url": "https://stripe.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "d1251f7b78e9f0d58ccb13f6dfe409d0950c9937899031f22e405c6865d9d9b4", "bytes_read": 69923 }, "llms_state": "candidate", "control": { "requested_url": "https://stripe.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:06.107398+00:00", "status": 404, "final_url": "https://stripe.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "a391d5cf13ba509532d666d3fa8f6d491dcdd7a11833df82abf02987523bf3d3", "bytes_read": 370272 } }, { "rank": 240, "domain": "pubmatic.com", "robots": { "requested_url": "https://pubmatic.com/robots.txt", "observed_at": "2026-09-20T06:24:05.691831+00:00", "status": 403, "final_url": "https://pubmatic.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "ca9c2981c49b98414fe957b3eca3fe57896cf6e1787393bcca07c2339680b180", "bytes_read": 919 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://pubmatic.com/llms.txt", "observed_at": "2026-09-20T06:24:05.904994+00:00", "status": 403, "final_url": "https://pubmatic.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "bfc6544dbcdcdb3c2314716f882b309fcf0c65f992ad0ff9fbe0a482852e829d", "bytes_read": 919 }, "llms_state": "unavailable", "control": null }, { "rank": 241, "domain": "cpanel.net", "robots": { "requested_url": "https://cpanel.net/robots.txt", "observed_at": "2026-09-20T06:24:05.776969+00:00", "status": 200, "final_url": "https://www.cpanel.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "aaa4243b4f8014855ae9a7a81b7cb96ff3c621c239ad6c805263b99029ddfda0", "bytes_read": 297 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://cpanel.net/llms.txt", "observed_at": "2026-09-20T06:24:06.395492+00:00", "status": 404, "final_url": "https://www.cpanel.net/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "8bef56cac49256bcdbddc192afd0cc64e6362aff6df59d22d961ff62a5f8739a", "bytes_read": 107817 }, "llms_state": "not_detected", "control": null }, { "rank": 242, "domain": "t.co", "robots": { "requested_url": "https://t.co/robots.txt", "observed_at": "2026-09-20T06:24:05.874859+00:00", "status": 200, "final_url": "https://t.co/robots.txt", "content_type": "text/plain;charset=utf-8", "truncated": false, "body_sha256": "1f4a1996aeeb85d3c39832b77beb5e4ea328e5701781a9f30f5caf291193865c", "bytes_read": 150 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://t.co/llms.txt", "observed_at": "2026-09-20T06:24:06.026222+00:00", "status": 404, "final_url": "https://t.co/llms.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "b4056ca07cbe8a3c71d16206c45c9d5515c74912647eb67100390f95a8a88600", "bytes_read": 3656 }, "llms_state": "not_detected", "control": null }, { "rank": 243, "domain": "canva.com", "robots": { "requested_url": "https://canva.com/robots.txt", "observed_at": "2026-09-20T06:24:06.095175+00:00", "status": 200, "final_url": "https://www.canva.com/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "f24cf8d8e655e2163ff6cd0d4cbb402a5ddaf886d9dc2bf34a61a951aeff5289", "bytes_read": 6438 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://canva.com/llms.txt", "observed_at": "2026-09-20T06:24:06.595091+00:00", "status": 404, "final_url": "https://www.canva.com/llms.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "9f370c7c635353dd89cd5662357774e05e6561dffc83e9be27d7b825dc8e31e2", "bytes_read": 1997 }, "llms_state": "not_detected", "control": null }, { "rank": 244, "domain": "wsdvs.com", "robots": { "requested_url": "https://wsdvs.com/robots.txt", "observed_at": "2026-09-20T06:24:06.173412+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://wsdvs.com/llms.txt", "observed_at": "2026-09-20T06:24:06.260441+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 245, "domain": "comcast.net", "robots": { "requested_url": "https://comcast.net/robots.txt", "observed_at": "2026-09-20T06:24:06.262169+00:00", "status": 403, "final_url": "https://www.xfinity.com/", "content_type": "text/html", "truncated": false, "body_sha256": "5dde8ef6019462fc196b7fb20f68afc526a5b2ec744ecd5a524973723dbbb809", "bytes_read": 369 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://comcast.net/llms.txt", "observed_at": "2026-09-20T06:24:07.287002+00:00", "status": 403, "final_url": "https://www.xfinity.com/", "content_type": "text/html", "truncated": false, "body_sha256": "dc25ffc9525e3c1e33704c2a1ab5487a590f930d44a18efccf79c4af5a37a50b", "bytes_read": 369 }, "llms_state": "unavailable", "control": null }, { "rank": 246, "domain": "playstation.net", "robots": { "requested_url": "https://playstation.net/robots.txt", "observed_at": "2026-09-20T06:24:06.697704+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://playstation.net/llms.txt", "observed_at": "2026-09-20T06:24:06.737222+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 247, "domain": "rbxcdn.com", "robots": { "requested_url": "https://rbxcdn.com/robots.txt", "observed_at": "2026-09-20T06:24:06.738764+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://rbxcdn.com/llms.txt", "observed_at": "2026-09-20T06:24:06.814593+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 248, "domain": "adsrvr.org", "robots": { "requested_url": "https://adsrvr.org/robots.txt", "observed_at": "2026-09-20T06:24:06.816303+00:00", "status": 403, "final_url": "https://adsrvr.org/robots.txt", "content_type": "application/xml", "truncated": false, "body_sha256": "a824bc7739e226e1b40ea0f8c4e4f4c6f796fc3b4abfa6e9abe3bd119a30d938", "bytes_read": 111 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://adsrvr.org/llms.txt", "observed_at": "2026-09-20T06:24:07.075664+00:00", "status": 403, "final_url": "https://adsrvr.org/llms.txt", "content_type": "application/xml", "truncated": false, "body_sha256": "a824bc7739e226e1b40ea0f8c4e4f4c6f796fc3b4abfa6e9abe3bd119a30d938", "bytes_read": 111 }, "llms_state": "unavailable", "control": null }, { "rank": 249, "domain": "meraki.com", "robots": { "requested_url": "https://meraki.com/robots.txt", "observed_at": "2026-09-20T06:24:06.939159+00:00", "status": 200, "final_url": "https://meraki.cisco.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "97df1dd76ec8eb07a9c37668f22939d03c9083147637fac401a7a963c84fbf63", "bytes_read": 2213 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://meraki.com/llms.txt", "observed_at": "2026-09-20T06:24:07.694086+00:00", "status": null, "error": "TimeoutError" }, "llms_state": "unavailable", "control": null }, { "rank": 250, "domain": "sourceforge.net", "robots": { "requested_url": "https://sourceforge.net/robots.txt", "observed_at": "2026-09-20T06:24:06.944787+00:00", "status": 200, "final_url": "https://sourceforge.net/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "09c4d5c44ceab0a22862415966b082c33a125c4763b7fb8f248bd5e73b45f034", "bytes_read": 5837 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://sourceforge.net/llms.txt", "observed_at": "2026-09-20T06:24:07.286526+00:00", "status": 200, "final_url": "https://sourceforge.net/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": true, "body_sha256": "32f5fd8ff516b1bc91a709250491fa39ab62c8345a92435d52895c304d32a7a1", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 251, "domain": "imcmdb.net", "robots": { "requested_url": "https://imcmdb.net/robots.txt", "observed_at": "2026-09-20T06:24:07.009112+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://imcmdb.net/llms.txt", "observed_at": "2026-09-20T06:24:17.196569+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 252, "domain": "roku.com", "robots": { "requested_url": "https://roku.com/robots.txt", "observed_at": "2026-09-20T06:24:07.286792+00:00", "status": 200, "final_url": "https://www.roku.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "066ab71c9b66d7b45625514669f588125f1a455552ca8cc79f9d496b680b4cd1", "bytes_read": 303 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://roku.com/llms.txt", "observed_at": "2026-09-20T06:24:07.851630+00:00", "status": 200, "final_url": "https://www.roku.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "57c97b0e5ede9818015bc7bc25fc88e169e9c505ce409cfe3a672b8d92522193", "bytes_read": 15683 }, "llms_state": "candidate", "control": { "requested_url": "https://roku.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:08.376255+00:00", "status": 404, "final_url": "https://www.roku.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "2a196cf866b6d9b8f859a1d30ed26769785795a826116dab166b93528e2e46f5", "bytes_read": 118563 } }, { "rank": 253, "domain": "omtrdc.net", "robots": { "requested_url": "https://omtrdc.net/robots.txt", "observed_at": "2026-09-20T06:24:07.801590+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://omtrdc.net/llms.txt", "observed_at": "2026-09-20T06:24:07.850954+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 254, "domain": "statsigapi.net", "robots": { "requested_url": "https://statsigapi.net/robots.txt", "observed_at": "2026-09-20T06:24:07.853429+00:00", "status": 403, "final_url": "https://statsigapi.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "b5865d15f4439d43678f8f1165ca58106c42c0f1aff9419dfc642fa4dcdac269", "bytes_read": 19 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://statsigapi.net/llms.txt", "observed_at": "2026-09-20T06:24:08.155182+00:00", "status": 403, "final_url": "https://statsigapi.net/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "b5865d15f4439d43678f8f1165ca58106c42c0f1aff9419dfc642fa4dcdac269", "bytes_read": 19 }, "llms_state": "unavailable", "control": null }, { "rank": 255, "domain": "cisco.com", "robots": { "requested_url": "https://cisco.com/robots.txt", "observed_at": "2026-09-20T06:24:08.024661+00:00", "status": 200, "final_url": "https://www.cisco.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "12098444d8aefbb839686eba5ef56272ea3ae74adfb1cc3bee380ac92ad13393", "bytes_read": 16189 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://cisco.com/llms.txt", "observed_at": "2026-09-20T06:24:09.077242+00:00", "status": 200, "final_url": "https://www.cisco.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e47eb32547c6fb01757bfbdc9d99ec33516b91cb99284f1c56540b4651021545", "bytes_read": 1196 }, "llms_state": "candidate", "control": { "requested_url": "https://cisco.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:09.707650+00:00", "status": 404, "final_url": "https://www.cisco.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "56a7f7a81bffe7920dca03b3132343a60376d7da08de5456390b5e45c0fb385b", "bytes_read": 32278 } }, { "rank": 256, "domain": "wikimedia.org", "robots": { "requested_url": "https://wikimedia.org/robots.txt", "observed_at": "2026-09-20T06:24:08.080536+00:00", "status": 404, "final_url": "https://www.wikimedia.org/robots.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "1f2e52511f1afed16ea642f4ce0b3da6669fee995d9c7f5cd9575789aa27d2ff", "bytes_read": 1874 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://wikimedia.org/llms.txt", "observed_at": "2026-09-20T06:24:08.621103+00:00", "status": 404, "final_url": "https://www.wikimedia.org/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "76563b4eeb07e461f97c018d6f0744fdaf2b0af79ebd90410e0452733d164bdf", "bytes_read": 1868 }, "llms_state": "not_detected", "control": null }, { "rank": 257, "domain": "ripn.net", "robots": { "requested_url": "https://ripn.net/robots.txt", "observed_at": "2026-09-20T06:24:08.377655+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ripn.net/llms.txt", "observed_at": "2026-09-20T06:24:08.587305+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 258, "domain": "android.com", "robots": { "requested_url": "https://android.com/robots.txt", "observed_at": "2026-09-20T06:24:08.589113+00:00", "status": 200, "final_url": "https://www.android.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "0a7b28459b8a6ea07fec6fa8a85e04521a516387762531c8b3ded73ecc10ecf9", "bytes_read": 1890 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://android.com/llms.txt", "observed_at": "2026-09-20T06:24:08.956709+00:00", "status": 404, "final_url": "https://www.android.com/llms.txt", "content_type": "text/html; charset=ISO-8859-1", "truncated": false, "body_sha256": "994c4957c2e1e17e63da543e320df4c9aeac598a7c7393a0f67c46d7dcbbf52f", "bytes_read": 226139 }, "llms_state": "not_detected", "control": null }, { "rank": 259, "domain": "alidns.com", "robots": { "requested_url": "https://alidns.com/robots.txt", "observed_at": "2026-09-20T06:24:08.916443+00:00", "status": 200, "final_url": "https://alidns.com/robots.txt", "content_type": "application/json; charset=utf-8", "truncated": false, "body_sha256": "7aa725e853bdb23fea945d1db53403112491cebcf0e0ea377746074250303f80", "bytes_read": 64 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://alidns.com/llms.txt", "observed_at": "2026-09-20T06:24:11.330936+00:00", "status": 200, "final_url": "https://alidns.com/llms.txt", "content_type": "application/json; charset=utf-8", "truncated": false, "body_sha256": "7aa725e853bdb23fea945d1db53403112491cebcf0e0ea377746074250303f80", "bytes_read": 64 }, "llms_state": "not_detected", "control": null }, { "rank": 260, "domain": "telegram.org", "robots": { "requested_url": "https://telegram.org/robots.txt", "observed_at": "2026-09-20T06:24:08.967053+00:00", "status": 404, "final_url": "https://telegram.org/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "bd5e6537d1fa8b7e136d3589dd722410eb3836f3b4b8e21733fba881463f5272", "bytes_read": 153 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://telegram.org/llms.txt", "observed_at": "2026-09-20T06:24:09.393235+00:00", "status": 200, "final_url": "https://telegram.org/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "81cdcc24b95a89d5426e6f29f217c64c3806bb29e36e43447d7d6944f841c9fa", "bytes_read": 19911 }, "llms_state": "not_detected", "control": null }, { "rank": 261, "domain": "hubspot.com", "robots": { "requested_url": "https://hubspot.com/robots.txt", "observed_at": "2026-09-20T06:24:09.364227+00:00", "status": 200, "final_url": "https://www.hubspot.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "74374bc3165f346bbb5959ad5d2948e2e16f1a364ba8ccb49663cb14eaae96af", "bytes_read": 12606 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://hubspot.com/llms.txt", "observed_at": "2026-09-20T06:24:09.938175+00:00", "status": 200, "final_url": "https://www.hubspot.com/llms.txt", "content_type": "text/plain; charset=x-macroman", "truncated": false, "body_sha256": "e1d66e610875c2e7f320de1c026b9fb8c49759d7670bdbd318630ae7d536c491", "bytes_read": 35564 }, "llms_state": "candidate", "control": { "requested_url": "https://hubspot.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:10.360718+00:00", "status": 404, "final_url": "https://www.hubspot.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "3326768b28b3b4c649a8ba7c29b51229f551b4f9f407df3846693f379e7753db", "bytes_read": 500957 } }, { "rank": 262, "domain": "hcaptcha.com", "robots": { "requested_url": "https://hcaptcha.com/robots.txt", "observed_at": "2026-09-20T06:24:09.397690+00:00", "status": 200, "final_url": "https://www.hcaptcha.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "27d24f57ae50401547cf1f9ff1ceed290928d997acf8db046756f5d4487febe2", "bytes_read": 109 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://hcaptcha.com/llms.txt", "observed_at": "2026-09-20T06:24:09.808604+00:00", "status": 200, "final_url": "https://www.hcaptcha.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "61cc9845baaaf131cfe4a89ac964f08f454b491aa0ea4324db5f04c519b62997", "bytes_read": 10217 }, "llms_state": "candidate", "control": { "requested_url": "https://hcaptcha.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:10.079882+00:00", "status": 404, "final_url": "https://www.hcaptcha.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "874b4dd4e0c93580649e88cb685278e9a74f9ca9d501f6f739f00f33b38dd509", "bytes_read": 3273 } }, { "rank": 263, "domain": "adnxs.com", "robots": { "requested_url": "https://adnxs.com/robots.txt", "observed_at": "2026-09-20T06:24:09.941632+00:00", "status": 404, "final_url": "https://adnxs.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "aa7f41d067fa6df8c37c71cbb1a6c7f2baa2f70697943313beaa382b8e24181c", "bytes_read": 153 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://adnxs.com/llms.txt", "observed_at": "2026-09-20T06:24:10.292638+00:00", "status": 404, "final_url": "https://adnxs.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "aa7f41d067fa6df8c37c71cbb1a6c7f2baa2f70697943313beaa382b8e24181c", "bytes_read": 153 }, "llms_state": "not_detected", "control": null }, { "rank": 264, "domain": "alibabadns.com", "robots": { "requested_url": "https://alibabadns.com/robots.txt", "observed_at": "2026-09-20T06:24:10.274793+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://alibabadns.com/llms.txt", "observed_at": "2026-09-20T06:24:10.611579+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 265, "domain": "scorecardresearch.com", "robots": { "requested_url": "https://scorecardresearch.com/robots.txt", "observed_at": "2026-09-20T06:24:10.341094+00:00", "status": 404, "final_url": "https://scorecardresearch.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "dc1d54dab6ec8c00f70137927504e4f222c8395f10760b6beecfcfa94e08249f", "bytes_read": 1245 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://scorecardresearch.com/llms.txt", "observed_at": "2026-09-20T06:24:10.564267+00:00", "status": 404, "final_url": "https://scorecardresearch.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "dc1d54dab6ec8c00f70137927504e4f222c8395f10760b6beecfcfa94e08249f", "bytes_read": 1245 }, "llms_state": "not_detected", "control": null }, { "rank": 266, "domain": "media-amazon.com", "robots": { "requested_url": "https://media-amazon.com/robots.txt", "observed_at": "2026-09-20T06:24:10.555870+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://media-amazon.com/llms.txt", "observed_at": "2026-09-20T06:24:10.605044+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 267, "domain": "imdb.com", "robots": { "requested_url": "https://imdb.com/robots.txt", "observed_at": "2026-09-20T06:24:10.607632+00:00", "status": 200, "final_url": "https://www.imdb.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "8c80a1f3076cb1ea938bcd8ed7d29f855ac81076e79c9c57fd3a31b84b969b3c", "bytes_read": 1947 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://imdb.com/llms.txt", "observed_at": "2026-09-20T06:24:11.211433+00:00", "status": 202, "final_url": "https://www.imdb.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "unavailable", "control": null }, { "rank": 268, "domain": "who.int", "robots": { "requested_url": "https://who.int/robots.txt", "observed_at": "2026-09-20T06:24:10.613938+00:00", "status": 200, "final_url": "https://www.who.int/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "c6957df319872ec37b752ec18e78f276e8957a8116af8f0e0a7cd3eb6ade78e1", "bytes_read": 18983 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://who.int/llms.txt", "observed_at": "2026-09-20T06:24:11.024884+00:00", "status": 404, "final_url": "https://www.who.int/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "f03af4c1142d4658ed1890601f5e039096b58ac4b0f67f6d914e994e506cf20c", "bytes_read": 72505 }, "llms_state": "not_detected", "control": null }, { "rank": 269, "domain": "booking.com", "robots": { "requested_url": "https://booking.com/robots.txt", "observed_at": "2026-09-20T06:24:10.761212+00:00", "status": 200, "final_url": "https://booking.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "d4d6f0840e2ed2b056ab44353eee55b1c8ddbbe2be3c0e82c4c942b0e93bfe6b", "bytes_read": 39491 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://booking.com/llms.txt", "observed_at": "2026-09-20T06:24:10.991756+00:00", "status": 404, "final_url": "https://www.booking.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "4bde8e4b38a7672b13b82eb5d53e89d69a1fb89ed019f394107244d5f18670b4", "bytes_read": 435157 }, "llms_state": "not_detected", "control": null }, { "rank": 270, "domain": "forter.com", "robots": { "requested_url": "https://forter.com/robots.txt", "observed_at": "2026-09-20T06:24:11.604830+00:00", "status": 200, "final_url": "https://forter.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "816144fcf584068f993830623dcecaab44c89fe6519ab4f4bbae21e70d2f1a7f", "bytes_read": 1253 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://forter.com/llms.txt", "observed_at": "2026-09-20T06:24:11.982455+00:00", "status": 200, "final_url": "https://forter.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "ed506da59fbdb5caf269fe35857dd55dcf3a456a21b254146218abc9127df943", "bytes_read": 20379 }, "llms_state": "candidate", "control": { "requested_url": "https://forter.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:12.148655+00:00", "status": 404, "final_url": "https://forter.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 } }, { "rank": 271, "domain": "wixsite.com", "robots": { "requested_url": "https://wixsite.com/robots.txt", "observed_at": "2026-09-20T06:24:11.621964+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://wixsite.com/llms.txt", "observed_at": "2026-09-20T06:24:11.670530+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 272, "domain": "linktr.ee", "robots": { "requested_url": "https://linktr.ee/robots.txt", "observed_at": "2026-09-20T06:24:11.623298+00:00", "status": 200, "final_url": "https://linktr.ee/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "f42d3a0a83cbb3f48bad5f8f984c7c412dacf36af1e7fcecfac9a8c073d95312", "bytes_read": 10873 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://linktr.ee/llms.txt", "observed_at": "2026-09-20T06:24:11.858394+00:00", "status": 200, "final_url": "https://linktr.ee/llms.txt", "content_type": "text/markdown; charset=utf-8", "truncated": false, "body_sha256": "0b4073401c7b5e55ffd90138ce686ca7615bccb17725946e2da0de66dc58a284", "bytes_read": 7913 }, "llms_state": "candidate", "control": { "requested_url": "https://linktr.ee/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:11.996854+00:00", "status": 404, "final_url": "https://linktr.ee/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "5c9ba0ebfa4d5bf38c15823d774e326228b4b699f4cf527801a48fd7e261a4a9", "bytes_read": 4954 } }, { "rank": 273, "domain": "name-services.com", "robots": { "requested_url": "https://name-services.com/robots.txt", "observed_at": "2026-09-20T06:24:11.672285+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://name-services.com/llms.txt", "observed_at": "2026-09-20T06:24:11.759927+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 274, "domain": "spov-msedge.net", "robots": { "requested_url": "https://spov-msedge.net/robots.txt", "observed_at": "2026-09-20T06:24:11.761690+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://spov-msedge.net/llms.txt", "observed_at": "2026-09-20T06:24:11.834146+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 275, "domain": "launchpad.net", "robots": { "requested_url": "https://launchpad.net/robots.txt", "observed_at": "2026-09-20T06:24:11.835813+00:00", "status": 200, "final_url": "https://launchpad.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "9cf965426fc3764630313a7ed1830e9094b55ded1520e46af85ea2ade4e10357", "bytes_read": 1881 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://launchpad.net/llms.txt", "observed_at": "2026-09-20T06:24:12.257305+00:00", "status": 404, "final_url": "https://launchpad.net/llms.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "ab84fb49e19edb20bfda1b789544e177a69e0a21c4ef2555f83875c8235801c5", "bytes_read": 9595 }, "llms_state": "not_detected", "control": null }, { "rank": 276, "domain": "ttvnw.net", "robots": { "requested_url": "https://ttvnw.net/robots.txt", "observed_at": "2026-09-20T06:24:12.141519+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ttvnw.net/llms.txt", "observed_at": "2026-09-20T06:24:12.173616+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 277, "domain": "facebook.net", "robots": { "requested_url": "https://facebook.net/robots.txt", "observed_at": "2026-09-20T06:24:12.175638+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://facebook.net/llms.txt", "observed_at": "2026-09-20T06:24:12.212008+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 278, "domain": "amon.tech", "robots": { "requested_url": "https://amon.tech/robots.txt", "observed_at": "2026-09-20T06:24:12.213696+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://amon.tech/llms.txt", "observed_at": "2026-09-20T06:24:12.607620+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 279, "domain": "mangosip.ru", "robots": { "requested_url": "https://mangosip.ru/robots.txt", "observed_at": "2026-09-20T06:24:12.264845+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://mangosip.ru/llms.txt", "observed_at": "2026-09-20T06:24:22.715181+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 280, "domain": "aliyuncsslbintl.com", "robots": { "requested_url": "https://aliyuncsslbintl.com/robots.txt", "observed_at": "2026-09-20T06:24:12.410500+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://aliyuncsslbintl.com/llms.txt", "observed_at": "2026-09-20T06:24:12.615351+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 281, "domain": "ibm.com", "robots": { "requested_url": "https://ibm.com/robots.txt", "observed_at": "2026-09-20T06:24:12.617329+00:00", "status": 200, "final_url": "https://www.ibm.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "764391437703b48b4886c320984d9633fef6a088e840ac47ae9d1d8bc78069c2", "bytes_read": 4540 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://ibm.com/llms.txt", "observed_at": "2026-09-20T06:24:13.259788+00:00", "status": 404, "final_url": "https://www.ibm.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "73948440a68a8955787c14c57b3214113d86639b0ca4460cea9a1218f1a65150", "bytes_read": 79283 }, "llms_state": "not_detected", "control": null }, { "rank": 282, "domain": "msedge.net", "robots": { "requested_url": "https://msedge.net/robots.txt", "observed_at": "2026-09-20T06:24:12.754572+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://msedge.net/llms.txt", "observed_at": "2026-09-20T06:24:22.817723+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 283, "domain": "telecid.ru", "robots": { "requested_url": "https://telecid.ru/robots.txt", "observed_at": "2026-09-20T06:24:12.813719+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://telecid.ru/llms.txt", "observed_at": "2026-09-20T06:24:13.495336+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 284, "domain": "reuters.com", "robots": { "requested_url": "https://reuters.com/robots.txt", "observed_at": "2026-09-20T06:24:12.830343+00:00", "status": 200, "final_url": "https://www.reuters.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "99a592c6446327bc8a068bc5995c22b5803b44fd26e2ab52333dcd83bd7ba363", "bytes_read": 4601 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://reuters.com/llms.txt", "observed_at": "2026-09-20T06:24:13.553810+00:00", "status": 401, "final_url": "https://www.reuters.com/llms.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "ac7ab33869a06f832be360d0b7dd327bbbda39100c2618af884103ced841c622", "bytes_read": 771 }, "llms_state": "unavailable", "control": null }, { "rank": 285, "domain": "shiabank.com", "robots": { "requested_url": "https://shiabank.com/robots.txt", "observed_at": "2026-09-20T06:24:13.811385+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://shiabank.com/llms.txt", "observed_at": "2026-09-20T06:24:23.970911+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 286, "domain": "2mdn.net", "robots": { "requested_url": "https://2mdn.net/robots.txt", "observed_at": "2026-09-20T06:24:13.854207+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://2mdn.net/llms.txt", "observed_at": "2026-09-20T06:24:13.907732+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 287, "domain": "appsflyer.com", "robots": { "requested_url": "https://appsflyer.com/robots.txt", "observed_at": "2026-09-20T06:24:13.909586+00:00", "status": 200, "final_url": "https://www.appsflyer.com:443/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "f50c566d5b7c25d1b078e3c5763e8fb96fb29c81b59d3dd268396ddf8b8f47e9", "bytes_read": 3116 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://appsflyer.com/llms.txt", "observed_at": "2026-09-20T06:24:14.540486+00:00", "status": 404, "final_url": "https://www.appsflyer.com:443/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 }, "llms_state": "not_detected", "control": null }, { "rank": 288, "domain": "dnsmadeeasy.com", "robots": { "requested_url": "https://dnsmadeeasy.com/robots.txt", "observed_at": "2026-09-20T06:24:14.044576+00:00", "status": 404, "final_url": "https://dnsmadeeasy.com/robots.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "e9c03a92d0ceb33c664013efd8dc449b94071f8a565a5dcba9804f3e52021085", "bytes_read": 99972 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://dnsmadeeasy.com/llms.txt", "observed_at": "2026-09-20T06:24:14.757274+00:00", "status": 404, "final_url": "https://dnsmadeeasy.com/llms.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "7b9c22ed32230cf1b6bb387955f8a44fd53002ae9eb7e577e06535051fda516e", "bytes_read": 99960 }, "llms_state": "not_detected", "control": null }, { "rank": 289, "domain": "springer.com", "robots": { "requested_url": "https://springer.com/robots.txt", "observed_at": "2026-09-20T06:24:14.933879+00:00", "status": 200, "final_url": "https://www.springer.com/robots.txt", "content_type": "", "truncated": false, "body_sha256": "9deaa339bbddb3b20c37c05b7537188f86204eb966c6fd98b8eac31cd406e696", "bytes_read": 286 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ], [ "allow", "/$" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ], [ "allow", "/$" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ], [ "allow", "/$" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ], [ "allow", "/$" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ], [ "allow", "/$" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ], [ "allow", "/$" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ], [ "allow", "/$" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ], [ "allow", "/$" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ], [ "allow", "/$" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ], [ "allow", "/$" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ], [ "allow", "/$" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ], [ "allow", "/$" ] ] } }, "llms": { "requested_url": "https://springer.com/llms.txt", "observed_at": "2026-09-20T06:24:15.555913+00:00", "status": 200, "final_url": "https://link.springer.com/brands/springer?error=cookies_not_supported&code=4080e32a-adc7-407a-9a9d-0643acb8c868", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "ccb85f86dc9edbf709ba2ee08e0b48301c3a212d321acc55b4e0e5c692f28702", "bytes_read": 271230 }, "llms_state": "not_detected", "control": null }, { "rank": 290, "domain": "namecheap.com", "robots": { "requested_url": "https://namecheap.com/robots.txt", "observed_at": "2026-09-20T06:24:15.373197+00:00", "status": 200, "final_url": "https://www.namecheap.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1a17e921688aa02ea543de8dcdc63a8aabee32b73761644bd87deefb2de48420", "bytes_read": 1359 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://namecheap.com/llms.txt", "observed_at": "2026-09-20T06:24:15.937375+00:00", "status": 403, "final_url": "https://www.namecheap.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "40c22ae5fde12b205ece419acbde8d86ecba51861fd6a0edae74d16005358cc3", "bytes_read": 5623 }, "llms_state": "unavailable", "control": null }, { "rank": 291, "domain": "telegram.me", "robots": { "requested_url": "https://telegram.me/robots.txt", "observed_at": "2026-09-20T06:24:16.307887+00:00", "status": 200, "final_url": "https://telegram.org/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "c8cce5d6bcdfff5a4eed0d914e8ca31aebcb69c266e1d07d1942797bc42bd952", "bytes_read": 19920 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://telegram.me/llms.txt", "observed_at": "2026-09-20T06:24:17.392121+00:00", "status": 200, "final_url": "https://telegram.org/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "3d5761b13b1dbf04a30a4597ecfd19a70ff44310ac083beae6651f98a937cd1b", "bytes_read": 19920 }, "llms_state": "not_detected", "control": null }, { "rank": 292, "domain": "drom.ru", "robots": { "requested_url": "https://drom.ru/robots.txt", "observed_at": "2026-09-20T06:24:17.761078+00:00", "status": 200, "final_url": "https://www.drom.ru/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "ccd1ac3ade44ede88f85f1609ab043d76886ea9152c9490b90481eb1b0745872", "bytes_read": 10797 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://drom.ru/llms.txt", "observed_at": "2026-09-20T06:24:18.810436+00:00", "status": 404, "final_url": "https://www.drom.ru/llms.txt", "content_type": "text/html; charset=windows-1251", "truncated": false, "body_sha256": "485fbbe7ee9126878c277328d00c1831bbd56237474f494fc9cd52791e44a4d2", "bytes_read": 50234 }, "llms_state": "not_detected", "control": null }, { "rank": 293, "domain": "mzstatic.com", "robots": { "requested_url": "https://mzstatic.com/robots.txt", "observed_at": "2026-09-20T06:24:17.960140+00:00", "status": 200, "final_url": "https://www.apple.com/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "dc369091de51ce34939ea51841873096ea2efa5e25bcfc53e3548ff229140a5f", "bytes_read": 312173 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://mzstatic.com/llms.txt", "observed_at": "2026-09-20T06:24:18.826583+00:00", "status": 200, "final_url": "https://www.apple.com/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "dc369091de51ce34939ea51841873096ea2efa5e25bcfc53e3548ff229140a5f", "bytes_read": 312173 }, "llms_state": "not_detected", "control": null }, { "rank": 294, "domain": "tinyurl.com", "robots": { "requested_url": "https://tinyurl.com/robots.txt", "observed_at": "2026-09-20T06:24:18.383570+00:00", "status": 200, "final_url": "https://tinyurl.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "627f35c9623a02c9e4281db4b6caaade781c73eba803bd3c6ec7fb78238aa42e", "bytes_read": 154 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://tinyurl.com/llms.txt", "observed_at": "2026-09-20T06:24:18.654481+00:00", "status": 404, "final_url": "https://tinyurl.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "bd8b07ee979885cbf54008ff0fc100888f2b427669eef6d070664cd769b918e0", "bytes_read": 28280 }, "llms_state": "not_detected", "control": null }, { "rank": 295, "domain": "tiktokcdn-us.com", "robots": { "requested_url": "https://tiktokcdn-us.com/robots.txt", "observed_at": "2026-09-20T06:24:18.875952+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://tiktokcdn-us.com/llms.txt", "observed_at": "2026-09-20T06:24:18.921353+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 296, "domain": "hp.com", "robots": { "requested_url": "https://hp.com/robots.txt", "observed_at": "2026-09-20T06:24:18.895759+00:00", "status": 200, "final_url": "https://www.hp.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "05a4d45bf193432243653fafacaf4fbd13926550a76711d0c09d8a996f83727c", "bytes_read": 17151 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://hp.com/llms.txt", "observed_at": "2026-09-20T06:24:19.737604+00:00", "status": 404, "final_url": "https://www.hp.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "63c0de7c6c7aea5753ef7db15d995b9046322af26b640d6458456e0d2a932d80", "bytes_read": 317803 }, "llms_state": "not_detected", "control": null }, { "rank": 297, "domain": "salesforce.com", "robots": { "requested_url": "https://salesforce.com/robots.txt", "observed_at": "2026-09-20T06:24:18.923462+00:00", "status": 200, "final_url": "https://www.salesforce.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "905d8800d5cc13bac380697922770b4c201bd32e570b858ff481cab029bbe20d", "bytes_read": 14256 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://salesforce.com/llms.txt", "observed_at": "2026-09-20T06:24:19.759386+00:00", "status": 200, "final_url": "https://www.salesforce.com/llms.txt?bc=DB", "content_type": "text/plain; charset=UTF-8", "truncated": true, "body_sha256": "236623e9f63501676f2607980301b18b8a83dbcbbaf00e28d2d82d3d71b866f8", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 298, "domain": "crpt.ru", "robots": { "requested_url": "https://crpt.ru/robots.txt", "observed_at": "2026-09-20T06:24:18.949199+00:00", "status": 200, "final_url": "https://crpt.ru/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://crpt.ru/llms.txt", "observed_at": "2026-09-20T06:24:19.986198+00:00", "status": 200, "final_url": "https://crpt.ru/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "not_detected", "control": null }, { "rank": 299, "domain": "dell.com", "robots": { "requested_url": "https://dell.com/robots.txt", "observed_at": "2026-09-20T06:24:19.609764+00:00", "status": 200, "final_url": "https://www.dell.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "f0ecc44961d338e6be5ee72a09ae6d2b8e498df266efd81ab294a936fb0c8aa0", "bytes_read": 29624 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://dell.com/llms.txt", "observed_at": "2026-09-20T06:24:21.572981+00:00", "status": 200, "final_url": "https://www.dell.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "16a64bf0b10d239022e26aa26680bad38d45b68e4e370a65614b40316a000568", "bytes_read": 133835 }, "llms_state": "candidate", "control": { "requested_url": "https://dell.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:22.590883+00:00", "status": 200, "final_url": "https://www.dell.com/en-us", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "e66fc4a9a8aa7bfe1c90e0905376e10f4ea86d3e2bc0c76032d5b528858272a9", "bytes_read": 343112 } }, { "rank": 300, "domain": "byteoversea.net", "robots": { "requested_url": "https://byteoversea.net/robots.txt", "observed_at": "2026-09-20T06:24:19.748781+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://byteoversea.net/llms.txt", "observed_at": "2026-09-20T06:24:19.781938+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 301, "domain": "nature.com", "robots": { "requested_url": "https://nature.com/robots.txt", "observed_at": "2026-09-20T06:24:19.784379+00:00", "status": 200, "final_url": "https://www.nature.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "747a13c2ef654bad8533e43ba7f5db2d1f2568880087855c817a549bf8ccdbc9", "bytes_read": 2721 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://nature.com/llms.txt", "observed_at": "2026-09-20T06:24:20.317248+00:00", "status": 404, "final_url": "https://www.nature.com/llms.txt?error=cookies_not_supported&code=6d243036-c726-435c-a88e-fa42fbb6ff89", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "9b4496bdb39a11a83b9455d78f6fbcfd4effe3108461459c7a600de0adf9bde7", "bytes_read": 99145 }, "llms_state": "not_detected", "control": null }, { "rank": 302, "domain": "inmobi.com", "robots": { "requested_url": "https://inmobi.com/robots.txt", "observed_at": "2026-09-20T06:24:19.851577+00:00", "status": 200, "final_url": "https://www.inmobi.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "cb5c17b8a79312998e06fb5f135be7e75528b730f42e3fc8a27962852eff6ad9", "bytes_read": 98 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://inmobi.com/llms.txt", "observed_at": "2026-09-20T06:24:20.403133+00:00", "status": 404, "final_url": "https://www.inmobi.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "1f48dde1f9240bf70f278c4aef5abcaa8e055ba96830f59f1b4d0b9e0c6f900f", "bytes_read": 27846 }, "llms_state": "not_detected", "control": null }, { "rank": 303, "domain": "jsdelivr.net", "robots": { "requested_url": "https://jsdelivr.net/robots.txt", "observed_at": "2026-09-20T06:24:20.716198+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://jsdelivr.net/llms.txt", "observed_at": "2026-09-20T06:24:30.756543+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 304, "domain": "steampowered.com", "robots": { "requested_url": "https://steampowered.com/robots.txt", "observed_at": "2026-09-20T06:24:20.801812+00:00", "status": 200, "final_url": "https://store.steampowered.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "d7943b2b8e930d73ec59a8613fcf01ebe9209663d893e12048034b9734f6240a", "bytes_read": 303 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://steampowered.com/llms.txt", "observed_at": "2026-09-20T06:24:21.900066+00:00", "status": 200, "final_url": "https://store.steampowered.com/", "content_type": "text/html; charset=UTF-8", "truncated": true, "body_sha256": "80df2b24eb3d5269071d22f778969cdde2093569dbf7c8b1afddc7ce9742b2d2", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 305, "domain": "weibo.com", "robots": { "requested_url": "https://weibo.com/robots.txt", "observed_at": "2026-09-20T06:24:20.866988+00:00", "status": 200, "final_url": "https://weibo.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "15b0db632716dd8564dda06ed3db8ed48cd35722eb65ac87eed134f82e9eb85e", "bytes_read": 968 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://weibo.com/llms.txt", "observed_at": "2026-09-20T06:24:22.106948+00:00", "status": 200, "final_url": "https://passport.weibo.com/visitor/visitor?entry=miniblog&a=enter&url=https%3A%2F%2Fweibo.com%2Fllms.txt&domain=weibo.com&ua=SuedeCrawlerIndex%2F2.0%20%28%2Bhttps%3A%2F%2Fsuedeai.ai%2Fblog%2Fai-crawler-access-index%29&_rand=1789885462938&sudaref=", "content_type": "text/html", "truncated": false, "body_sha256": "d0d9f036e3e672350a99c2c7239f06a3d223def7f218112de842f499cb6d2cef", "bytes_read": 9534 }, "llms_state": "not_detected", "control": null }, { "rank": 306, "domain": "autodesk.com", "robots": { "requested_url": "https://autodesk.com/robots.txt", "observed_at": "2026-09-20T06:24:21.322892+00:00", "status": 403, "final_url": "https://www.autodesk.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "fa6de2ff59d6c878258375f67bac1c16d27854088330255984e28fe7a175f54d", "bytes_read": 382 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://autodesk.com/llms.txt", "observed_at": "2026-09-20T06:24:22.049267+00:00", "status": 403, "final_url": "https://www.autodesk.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "53576f808e345d310c1697775d170f6ea837d1ec66ddf3433341cdcd8d20bbca", "bytes_read": 380 }, "llms_state": "unavailable", "control": null }, { "rank": 307, "domain": "www.gov.uk", "robots": { "requested_url": "https://www.gov.uk/robots.txt", "observed_at": "2026-09-20T06:24:21.849588+00:00", "status": 200, "final_url": "https://www.gov.uk/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "459835d29527b9c00f00f6af00f45c73a909af1944e81fefa153f585e1857305", "bytes_read": 1097 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [] } }, "llms": { "requested_url": "https://www.gov.uk/llms.txt", "observed_at": "2026-09-20T06:24:22.130479+00:00", "status": 404, "final_url": "https://www.gov.uk/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "6242fd3cb20b9416cba085eff3d605583b2103ab743cd8ec02c916d466881d7d", "bytes_read": 52491 }, "llms_state": "not_detected", "control": null }, { "rank": 308, "domain": "clarity.ms", "robots": { "requested_url": "https://clarity.ms/robots.txt", "observed_at": "2026-09-20T06:24:21.914575+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://clarity.ms/llms.txt", "observed_at": "2026-09-20T06:24:22.216295+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 309, "domain": "ok.ru", "robots": { "requested_url": "https://ok.ru/robots.txt", "observed_at": "2026-09-20T06:24:22.081495+00:00", "status": 200, "final_url": "https://ok.ru/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "5fa78036b382d6307d228182901f38e3ba17c90dc96cba7f099502f0dadf483b", "bytes_read": 6025 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://ok.ru/llms.txt", "observed_at": "2026-09-20T06:24:22.835556+00:00", "status": 404, "final_url": "https://ok.ru/llms.txt", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "not_detected", "control": null }, { "rank": 310, "domain": "wac-msedge.net", "robots": { "requested_url": "https://wac-msedge.net/robots.txt", "observed_at": "2026-09-20T06:24:22.447905+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://wac-msedge.net/llms.txt", "observed_at": "2026-09-20T06:24:22.504888+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 311, "domain": "amplitude.com", "robots": { "requested_url": "https://amplitude.com/robots.txt", "observed_at": "2026-09-20T06:24:22.507057+00:00", "status": 200, "final_url": "https://amplitude.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "9a58c15a331d8009022c488e1aab7a6b7901566096d504b64287b0f92554b81d", "bytes_read": 2385 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://amplitude.com/llms.txt", "observed_at": "2026-09-20T06:24:23.333832+00:00", "status": 200, "final_url": "https://amplitude.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "887a00530eee8e12a4f56d28ac1d4e7c522874022d51a381939a4fbc27c54423", "bytes_read": 12993 }, "llms_state": "candidate", "control": { "requested_url": "https://amplitude.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:23.969515+00:00", "status": 404, "final_url": "https://amplitude.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "b2cc62912b9079917e8142a2ab1c1b7aa2334818376b519bd4d4d3e2e6bafdf2", "bytes_read": 123712 } }, { "rank": 312, "domain": "alibaba.com", "robots": { "requested_url": "https://alibaba.com/robots.txt", "observed_at": "2026-09-20T06:24:22.524061+00:00", "status": 200, "final_url": "https://www.alibaba.com/atlassitemapsitenet/static/www_alibaba_com/robots.txt", "content_type": "", "truncated": false, "body_sha256": "a4a98eee7f6330d376232fa8eaef77ab924702e2fdd91e071b4f2c27501b8cae", "bytes_read": 5406 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://alibaba.com/llms.txt", "observed_at": "2026-09-20T06:24:23.483352+00:00", "status": 200, "final_url": "https://error.alibaba.com/error404.htm", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "16aa368239a980508641c4eda161cf7a7f30fec54bb288e22f9aa99a640b6b10", "bytes_read": 269900 }, "llms_state": "not_detected", "control": null }, { "rank": 313, "domain": "avast.com", "robots": { "requested_url": "https://avast.com/robots.txt", "observed_at": "2026-09-20T06:24:22.994313+00:00", "status": 200, "final_url": "https://www.avast.com/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "8bf069f2ef136e4121be44de1981f45f4774fdfed473df7f77a520367bb6d9ef", "bytes_read": 440 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://avast.com/llms.txt", "observed_at": "2026-09-20T06:24:23.803819+00:00", "status": 200, "final_url": "https://www.avast.com/llms.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "6154500d582ff4307083ebb678afa7b9b80cb5569921b7c131a5bfb72e857115", "bytes_read": 5625 }, "llms_state": "candidate", "control": { "requested_url": "https://avast.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:24.847019+00:00", "status": 404, "final_url": "https://www.avast.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "23bae7cb30b882fb70d53cba998dcb8841617ec170199569f71cf9206e4fc444", "bytes_read": 88201 } }, { "rank": 314, "domain": "weather.com", "robots": { "requested_url": "https://weather.com/robots.txt", "observed_at": "2026-09-20T06:24:23.096354+00:00", "status": 200, "final_url": "https://weather.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "28adb9aed15bb249f60992768684e6321d30d6ab2fdfcb34d0ed9dce111a1e38", "bytes_read": 3539 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://weather.com/llms.txt", "observed_at": "2026-09-20T06:24:23.321053+00:00", "status": 200, "final_url": "https://weather.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "e5b49dcc90d2a356203a123178d265cba081608375186a9efda54f5e7cadc888", "bytes_read": 9101 }, "llms_state": "candidate", "control": { "requested_url": "https://weather.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:23.498289+00:00", "status": 404, "final_url": "https://weather.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "7d2569c947fced5f5b8a9e50af61a396dc040eadd795017c87d746967c39df28", "bytes_read": 168332 } }, { "rank": 315, "domain": "samsungcloud.com", "robots": { "requested_url": "https://samsungcloud.com/robots.txt", "observed_at": "2026-09-20T06:24:23.398033+00:00", "status": 200, "final_url": "https://www.samsungcloud.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "c46cb537f640e0b9194aef1e038dac520c739d6b0aba40a5f2aeac76995d652c", "bytes_read": 125 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://samsungcloud.com/llms.txt", "observed_at": "2026-09-20T06:24:24.193635+00:00", "status": 404, "final_url": "https://www.samsungcloud.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "5984786022817806b3ec371afc3c7c82683f97a27d3d0e6d8039a0c5385fdcfa", "bytes_read": 43208 }, "llms_state": "not_detected", "control": null }, { "rank": 316, "domain": "arubanetworks.com", "robots": { "requested_url": "https://arubanetworks.com/robots.txt", "observed_at": "2026-09-20T06:24:23.631487+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://arubanetworks.com/llms.txt", "observed_at": "2026-09-20T06:24:23.784380+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 317, "domain": "twc.com", "robots": { "requested_url": "https://twc.com/robots.txt", "observed_at": "2026-09-20T06:24:23.853624+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://twc.com/llms.txt", "observed_at": "2026-09-20T06:24:34.145341+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 318, "domain": "debian.org", "robots": { "requested_url": "https://debian.org/robots.txt", "observed_at": "2026-09-20T06:24:23.911373+00:00", "status": 404, "final_url": "https://www.debian.org/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "ee47bbebee1079c6126833d00ab2858357fd3ebd57c3e99b94d67a3dc84c58e9", "bytes_read": 7468 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://debian.org/llms.txt", "observed_at": "2026-09-20T06:24:24.354333+00:00", "status": 404, "final_url": "https://www.debian.org/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "ee47bbebee1079c6126833d00ab2858357fd3ebd57c3e99b94d67a3dc84c58e9", "bytes_read": 7468 }, "llms_state": "not_detected", "control": null }, { "rank": 319, "domain": "harvard.edu", "robots": { "requested_url": "https://harvard.edu/robots.txt", "observed_at": "2026-09-20T06:24:24.465003+00:00", "status": 200, "final_url": "https://www.harvard.edu/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "14353df5d44dabd47535f2f91e1fbfdf3e0cc8de10bfe8f5d79e11b1d9cc2474", "bytes_read": 748 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://harvard.edu/llms.txt", "observed_at": "2026-09-20T06:24:24.908243+00:00", "status": 404, "final_url": "https://www.harvard.edu/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "8ec8c40f54387d1bec79bd143e10d14f2ac13786fb8c82befba11a20c2d37cdd", "bytes_read": 109944 }, "llms_state": "not_detected", "control": null }, { "rank": 320, "domain": "wiley.com", "robots": { "requested_url": "https://wiley.com/robots.txt", "observed_at": "2026-09-20T06:24:24.566047+00:00", "status": 200, "final_url": "https://www.wiley.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "5593e986c2951bcc5e6167b7f24c51ccc98120ce465e1e6066b2acd34ff076fe", "bytes_read": 51 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://wiley.com/llms.txt", "observed_at": "2026-09-20T06:24:25.068580+00:00", "status": 404, "final_url": "https://www.wiley.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "516cc2590df3bf474b3d8f1571b3624a3bbbd026e1b7e74015d4414c909d59c4", "bytes_read": 365710 }, "llms_state": "not_detected", "control": null }, { "rank": 321, "domain": "ea.com", "robots": { "requested_url": "https://ea.com/robots.txt", "observed_at": "2026-09-20T06:24:24.710161+00:00", "status": 200, "final_url": "https://www.ea.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "a12b6bc756d5fcc533f2f5942b150c136d9d75282f75e0e19c02cb835e2316ba", "bytes_read": 1904 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://ea.com/llms.txt", "observed_at": "2026-09-20T06:24:25.618468+00:00", "status": 404, "final_url": "https://www.ea.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "191f8dd888b1018d405da1ef76edb10c847db269eb0e95a94ed5ac795ad7299e", "bytes_read": 79684 }, "llms_state": "not_detected", "control": null }, { "rank": 322, "domain": "mtgglobals.com", "robots": { "requested_url": "https://mtgglobals.com/robots.txt", "observed_at": "2026-09-20T06:24:24.805637+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://mtgglobals.com/llms.txt", "observed_at": "2026-09-20T06:24:24.860375+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 323, "domain": "douyincdn.com", "robots": { "requested_url": "https://douyincdn.com/robots.txt", "observed_at": "2026-09-20T06:24:24.840455+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://douyincdn.com/llms.txt", "observed_at": "2026-09-20T06:24:24.917048+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 324, "domain": "samsungqbe.com", "robots": { "requested_url": "https://samsungqbe.com/robots.txt", "observed_at": "2026-09-20T06:24:24.862180+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://samsungqbe.com/llms.txt", "observed_at": "2026-09-20T06:24:24.914562+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 325, "domain": "heytapmobile.com", "robots": { "requested_url": "https://heytapmobile.com/robots.txt", "observed_at": "2026-09-20T06:24:24.916741+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://heytapmobile.com/llms.txt", "observed_at": "2026-09-20T06:24:24.980933+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 326, "domain": "shalltry.com", "robots": { "requested_url": "https://shalltry.com/robots.txt", "observed_at": "2026-09-20T06:24:24.919813+00:00", "status": 403, "final_url": "https://shalltry.com/robots.txt", "content_type": "application/xml", "truncated": false, "body_sha256": "a824bc7739e226e1b40ea0f8c4e4f4c6f796fc3b4abfa6e9abe3bd119a30d938", "bytes_read": 111 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://shalltry.com/llms.txt", "observed_at": "2026-09-20T06:24:25.367697+00:00", "status": 403, "final_url": "https://shalltry.com/llms.txt", "content_type": "application/xml", "truncated": false, "body_sha256": "a824bc7739e226e1b40ea0f8c4e4f4c6f796fc3b4abfa6e9abe3bd119a30d938", "bytes_read": 111 }, "llms_state": "unavailable", "control": null }, { "rank": 327, "domain": "php.net", "robots": { "requested_url": "https://php.net/robots.txt", "observed_at": "2026-09-20T06:24:24.982628+00:00", "status": 200, "final_url": "https://www.php.net/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "f94f2698fb4ea74efc64b0b61afff9a2fb05982f38fe6fbd0e4521cf668cd2e4", "bytes_read": 292 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://php.net/llms.txt", "observed_at": "2026-09-20T06:24:25.711514+00:00", "status": 404, "final_url": "https://www.php.net/manual-lookup.php?pattern=llms.txt&lang=en&scope=404quickref", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "202f91b2cbab2acec11ebe5088205f50c8b794da434dff608493ea1cd2cf1e81", "bytes_read": 22138 }, "llms_state": "not_detected", "control": null }, { "rank": 328, "domain": "mozilla.com", "robots": { "requested_url": "https://mozilla.com/robots.txt", "observed_at": "2026-09-20T06:24:25.517527+00:00", "status": 200, "final_url": "https://www.mozilla.org/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "cf0fd2842659297b0d6fce407bed86bf424ae6666051b881dbd72e871d687fe3", "bytes_read": 66 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://mozilla.com/llms.txt", "observed_at": "2026-09-20T06:24:26.215667+00:00", "status": 404, "final_url": "https://www.mozilla.org/en-US/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "9475c8da3d41214acce52bde2711027ed7122046b819573334ccf189f53956b4", "bytes_read": 9118 }, "llms_state": "not_detected", "control": null }, { "rank": 329, "domain": "akamaihd.net", "robots": { "requested_url": "https://akamaihd.net/robots.txt", "observed_at": "2026-09-20T06:24:25.582476+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://akamaihd.net/llms.txt", "observed_at": "2026-09-20T06:24:25.618837+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 330, "domain": "aliyun.com", "robots": { "requested_url": "https://aliyun.com/robots.txt", "observed_at": "2026-09-20T06:24:25.614424+00:00", "status": 200, "final_url": "https://www.aliyun.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "ccd19f4b14057b506526c588b37f9b5abe9faad1344ea3ee46e7de2ced7d4e44", "bytes_read": 316 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://aliyun.com/llms.txt", "observed_at": "2026-09-20T06:24:29.903059+00:00", "status": 404, "final_url": "https://cn.aliyun.com/llms.txt?from_alibabacloud=", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "6d919dd7592a6dc824b9b98f7f27a6e691da80fef43b2a12fd4ab9e6b34fa989", "bytes_read": 42766 }, "llms_state": "not_detected", "control": null }, { "rank": 331, "domain": "googleblog.com", "robots": { "requested_url": "https://googleblog.com/robots.txt", "observed_at": "2026-09-20T06:24:25.621091+00:00", "status": 200, "final_url": "https://blog.google/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "1815b563c56cf59a8f4c5c9ee76c3aa707a2e50bc361b96dde030c73d57e95a2", "bytes_read": 358745 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://googleblog.com/llms.txt", "observed_at": "2026-09-20T06:24:26.727131+00:00", "status": 200, "final_url": "https://blog.google/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "1815b563c56cf59a8f4c5c9ee76c3aa707a2e50bc361b96dde030c73d57e95a2", "bytes_read": 358745 }, "llms_state": "not_detected", "control": null }, { "rank": 332, "domain": "hostinger.com", "robots": { "requested_url": "https://hostinger.com/robots.txt", "observed_at": "2026-09-20T06:24:25.770278+00:00", "status": 200, "final_url": "https://www.hostinger.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "dd4647a6ea6ea70ba9512751f977193b679f946bd033fed46d2fd63943c89b16", "bytes_read": 21090 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://hostinger.com/llms.txt", "observed_at": "2026-09-20T06:24:26.120722+00:00", "status": 200, "final_url": "https://www.hostinger.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "5e2b28543415ad9d0271763652fb9d6e8c75885e91d19f293187b4d1de4b7dd8", "bytes_read": 12481 }, "llms_state": "candidate", "control": { "requested_url": "https://hostinger.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:26.503496+00:00", "status": 404, "final_url": "https://www.hostinger.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "f8eb947af1e7a394eb6faad20b3976cba2519ea8ec9685d6f7dfea2085a316a7", "bytes_read": 6611 } }, { "rank": 333, "domain": "amazon.co.uk", "robots": { "requested_url": "https://amazon.co.uk/robots.txt", "observed_at": "2026-09-20T06:24:25.936602+00:00", "status": 200, "final_url": "https://www.amazon.co.uk/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "b176460b07987e270b1f24d39317764ed6f7e335c818c348de86613884003ab2", "bytes_read": 7258 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://amazon.co.uk/llms.txt", "observed_at": "2026-09-20T06:24:27.101856+00:00", "status": 503, "final_url": "https://www.amazon.co.uk/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "b5762236bd45a44d490ad74c3d25291934465da0b08242d265de8f61c7b36926", "bytes_read": 1427 }, "llms_state": "unavailable", "control": null }, { "rank": 334, "domain": "spaceweb.pro", "robots": { "requested_url": "https://spaceweb.pro/robots.txt", "observed_at": "2026-09-20T06:24:26.482802+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://spaceweb.pro/llms.txt", "observed_at": "2026-09-20T06:24:37.910987+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 335, "domain": "awswaf.com", "robots": { "requested_url": "https://awswaf.com/robots.txt", "observed_at": "2026-09-20T06:24:26.807727+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://awswaf.com/llms.txt", "observed_at": "2026-09-20T06:24:26.855466+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 336, "domain": "etsy.com", "robots": { "requested_url": "https://etsy.com/robots.txt", "observed_at": "2026-09-20T06:24:26.857556+00:00", "status": 200, "final_url": "https://www.etsy.com/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "3e5220b7fc73c79989d3bac507d711cc577338a9bdfdc92083d7613e0c5f7c9d", "bytes_read": 52773 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://etsy.com/llms.txt", "observed_at": "2026-09-20T06:24:28.954532+00:00", "status": 404, "final_url": "https://www.etsy.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "1cd789d8d945f0d1114bbefb8edeb5e5af326f94fe354254acdefa9ed3dd830b", "bytes_read": 54346 }, "llms_state": "not_detected", "control": null }, { "rank": 337, "domain": "trueconf.net", "robots": { "requested_url": "https://trueconf.net/robots.txt", "observed_at": "2026-09-20T06:24:26.892025+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://trueconf.net/llms.txt", "observed_at": "2026-09-20T06:24:27.495134+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 338, "domain": "datadoghq.com", "robots": { "requested_url": "https://datadoghq.com/robots.txt", "observed_at": "2026-09-20T06:24:27.204475+00:00", "status": 200, "final_url": "https://www.datadoghq.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "31eeea994b43f6d7f90690150759601a19c7a76f19827a51b18c89c6cbb1076b", "bytes_read": 525 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://datadoghq.com/llms.txt", "observed_at": "2026-09-20T06:24:27.662504+00:00", "status": 200, "final_url": "https://www.datadoghq.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": true, "body_sha256": "e874d199e949cea8e2fb517e2edb2f28c841d0ddc658e465258dc4aff7cc8d95", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 339, "domain": "myshopify.com", "robots": { "requested_url": "https://myshopify.com/robots.txt", "observed_at": "2026-09-20T06:24:27.747377+00:00", "status": 404, "final_url": "https://myshopify.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "aca34a237cc49ea34c2c42a5fad3363cf426e9ff67c81f68cb73d4c69ff9e0ff", "bytes_read": 22 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://myshopify.com/llms.txt", "observed_at": "2026-09-20T06:24:28.003538+00:00", "status": 404, "final_url": "https://myshopify.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "aca34a237cc49ea34c2c42a5fad3363cf426e9ff67c81f68cb73d4c69ff9e0ff", "bytes_read": 22 }, "llms_state": "not_detected", "control": null }, { "rank": 340, "domain": "smartadserver.com", "robots": { "requested_url": "https://smartadserver.com/robots.txt", "observed_at": "2026-09-20T06:24:27.866888+00:00", "status": 200, "final_url": "https://www.equativ.com/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "9dc59f6b0d2ae57705e475ba27b4302160370434954179ee316d48cebd72f113", "bytes_read": 120977 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://smartadserver.com/llms.txt", "observed_at": "2026-09-20T06:24:28.958197+00:00", "status": 200, "final_url": "https://www.equativ.com/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "9dc59f6b0d2ae57705e475ba27b4302160370434954179ee316d48cebd72f113", "bytes_read": 120977 }, "llms_state": "not_detected", "control": null }, { "rank": 341, "domain": "crashlytics.com", "robots": { "requested_url": "https://crashlytics.com/robots.txt", "observed_at": "2026-09-20T06:24:27.915783+00:00", "status": 200, "final_url": "https://firebase.google.com/products/crashlytics/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "a1ce78b4b650d9b0780607a48e9d0b47be66286046a178e48d46dac2501a86cb", "bytes_read": 511993 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://crashlytics.com/llms.txt", "observed_at": "2026-09-20T06:24:28.963950+00:00", "status": 200, "final_url": "https://firebase.google.com/products/crashlytics/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "76ff9ec56f151ac732be4361a399db765da5f33b9fe9167d081f0e5b5507847e", "bytes_read": 511989 }, "llms_state": "not_detected", "control": null }, { "rank": 342, "domain": "bilibili.com", "robots": { "requested_url": "https://bilibili.com/robots.txt", "observed_at": "2026-09-20T06:24:27.995065+00:00", "status": 200, "final_url": "https://www.bilibili.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "3c7225a7df79889d6fa7392f0798c1c001b0d0a83aea2dbf50214767e2610e3b", "bytes_read": 65 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://bilibili.com/llms.txt", "observed_at": "2026-09-20T06:24:29.624437+00:00", "status": 200, "final_url": "https://www.bilibili.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "361bcbb49345b0a7c1a3af92168c0d5a7b3ba74cec9331b2629a8724d6c336a3", "bytes_read": 5541 }, "llms_state": "not_detected", "control": null }, { "rank": 343, "domain": "worldnic.com", "robots": { "requested_url": "https://worldnic.com/robots.txt", "observed_at": "2026-09-20T06:24:28.263048+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://worldnic.com/llms.txt", "observed_at": "2026-09-20T06:24:38.404457+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 344, "domain": "media.net", "robots": { "requested_url": "https://media.net/robots.txt", "observed_at": "2026-09-20T06:24:28.397997+00:00", "status": 200, "final_url": "https://www.media.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "7fa2f2008692cab1af7c413ba0af97435d2b137ecb42afc8b4d38fd7cf3d48f2", "bytes_read": 337 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://media.net/llms.txt", "observed_at": "2026-09-20T06:24:29.218613+00:00", "status": 404, "final_url": "https://www.media.net/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "1669dbec0a206698007959f7eade4cabd7d5cd5a59b038a84b429f554b0e1da2", "bytes_read": 77738 }, "llms_state": "not_detected", "control": null }, { "rank": 345, "domain": "tbcache.com", "robots": { "requested_url": "https://tbcache.com/robots.txt", "observed_at": "2026-09-20T06:24:29.345571+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://tbcache.com/llms.txt", "observed_at": "2026-09-20T06:24:49.472708+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 346, "domain": "mobile.de", "robots": { "requested_url": "https://mobile.de/robots.txt", "observed_at": "2026-09-20T06:24:29.492255+00:00", "status": 200, "final_url": "https://www.mobile.de/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "615fd4d22633c6e8aff5a70d646f3cc32d9a22e8ecaed2634d0d7b1a21907532", "bytes_read": 1142 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://mobile.de/llms.txt", "observed_at": "2026-09-20T06:24:30.206744+00:00", "status": 403, "final_url": "https://www.mobile.de/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "95257f6094a7eb55db8ce5d9c403469daff4410c86f620f8393d3d3c68d472e6", "bytes_read": 7702 }, "llms_state": "unavailable", "control": null }, { "rank": 347, "domain": "name.com", "robots": { "requested_url": "https://name.com/robots.txt", "observed_at": "2026-09-20T06:24:29.807194+00:00", "status": 200, "final_url": "https://www.name.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "fa1599318bb9d98e374c4cfe2aadb99394092c2beac0a3fd62831bc299c953c2", "bytes_read": 1944 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://name.com/llms.txt", "observed_at": "2026-09-20T06:24:30.756270+00:00", "status": 200, "final_url": "https://www.name.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "236ce4c7a060cbbe51478eb011882a8fcb358a35e05d2ab24c461857017c4059", "bytes_read": 549 }, "llms_state": "not_detected", "control": null }, { "rank": 348, "domain": "gnu.org", "robots": { "requested_url": "https://gnu.org/robots.txt", "observed_at": "2026-09-20T06:24:29.844697+00:00", "status": 200, "final_url": "https://www.gnu.org/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "0e4faeb5a566b55bee4e90a4737ddbb926989b9133635e8257bb8e692d9b16fd", "bytes_read": 2862 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://gnu.org/llms.txt", "observed_at": "2026-09-20T06:24:30.410142+00:00", "status": 404, "final_url": "https://www.gnu.org/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "c46a74e41610f0786f1f91b7f4627d6717e6e0663bdb63887141adab14d21e71", "bytes_read": 11549 }, "llms_state": "not_detected", "control": null }, { "rank": 349, "domain": "trustpilot.com", "robots": { "requested_url": "https://trustpilot.com/robots.txt", "observed_at": "2026-09-20T06:24:30.485960+00:00", "status": 200, "final_url": "https://www.trustpilot.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "dd3d85ddfb30095b937c8ebabb513899681d3fc4a6f5532b63b2cd4cc1fc21ba", "bytes_read": 20976 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://trustpilot.com/llms.txt", "observed_at": "2026-09-20T06:24:31.400164+00:00", "status": 403, "final_url": "https://www.trustpilot.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "92d97ed45d6cc267c657b864c4deee1c2d896cda361adc660bc16538a75700f2", "bytes_read": 991 }, "llms_state": "unavailable", "control": null }, { "rank": 350, "domain": "doubleverify.com", "robots": { "requested_url": "https://doubleverify.com/robots.txt", "observed_at": "2026-09-20T06:24:30.836430+00:00", "status": 200, "final_url": "https://doubleverify.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "ea8e85a2b5420fc4539c4997c85a6821500c4e4a4ebb410b0c926f648fddb5ce", "bytes_read": 165 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://doubleverify.com/llms.txt", "observed_at": "2026-09-20T06:24:31.046254+00:00", "status": 404, "final_url": "https://doubleverify.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "6533668da375f293bd519c06bed5b9f28a569987ccc8f09521d9da48911222ac", "bytes_read": 115368 }, "llms_state": "not_detected", "control": null }, { "rank": 351, "domain": "naver.com", "robots": { "requested_url": "https://naver.com/robots.txt", "observed_at": "2026-09-20T06:24:30.912210+00:00", "status": 200, "final_url": "https://www.naver.com/robots.txt", "content_type": "application/octet-stream", "truncated": false, "body_sha256": "5342900c4c6bb95dbfdb9faf9cd447ed78a1f9c3699be58c370d2133658bc26b", "bytes_read": 91 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://naver.com/llms.txt", "observed_at": "2026-09-20T06:24:32.843950+00:00", "status": 404, "final_url": "https://www.naver.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "118ece62ca8ee95241c93fb95afd12d015aabcf9baf2861e953b3e5fdcf8f5f1", "bytes_read": 3657 }, "llms_state": "not_detected", "control": null }, { "rank": 352, "domain": "inner-active.mobi", "robots": { "requested_url": "https://inner-active.mobi/robots.txt", "observed_at": "2026-09-20T06:24:31.251244+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://inner-active.mobi/llms.txt", "observed_at": "2026-09-20T06:24:51.317083+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 353, "domain": "casalemedia.com", "robots": { "requested_url": "https://casalemedia.com/robots.txt", "observed_at": "2026-09-20T06:24:31.543420+00:00", "status": 200, "final_url": "https://www.indexexchange.com/", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "5d05a0174c1a0c03f9f3f6082331d60e2003decbfd5feb41f77f93a13bad1f86", "bytes_read": 113354 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://casalemedia.com/llms.txt", "observed_at": "2026-09-20T06:24:32.718370+00:00", "status": 200, "final_url": "https://www.indexexchange.com/", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "5d05a0174c1a0c03f9f3f6082331d60e2003decbfd5feb41f77f93a13bad1f86", "bytes_read": 113354 }, "llms_state": "not_detected", "control": null }, { "rank": 354, "domain": "slack.com", "robots": { "requested_url": "https://slack.com/robots.txt", "observed_at": "2026-09-20T06:24:31.565316+00:00", "status": 200, "final_url": "https://slack.com/robots.txt", "content_type": "text/plain;charset=utf-8", "truncated": false, "body_sha256": "5cf267512c33523c02640ab26da932e2f6ccd78d88ff17cd8a540630d03104c6", "bytes_read": 520 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://slack.com/llms.txt", "observed_at": "2026-09-20T06:24:31.853770+00:00", "status": 200, "final_url": "https://slack.com/llms.txt", "content_type": "text/plain;charset=utf-8", "truncated": false, "body_sha256": "d08ba41e1e7bbccb335dd566f975bb9e93ad9f65bfa48a5c0fb109bc0095e267", "bytes_read": 40672 }, "llms_state": "candidate", "control": { "requested_url": "https://slack.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:32.274685+00:00", "status": 404, "final_url": "https://slack.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "6e1e86d3f1c47de6f31ef7fd40136ec7162d4ebb1a9346e742f2066adfa88f94", "bytes_read": 45166 } }, { "rank": 355, "domain": "checkpoint.com", "robots": { "requested_url": "https://checkpoint.com/robots.txt", "observed_at": "2026-09-20T06:24:32.340050+00:00", "status": 200, "final_url": "https://www.checkpoint.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1f8adfdecef9041e841e8ba8a46304ac4d0538d546a14524ff159bc844f1a1f1", "bytes_read": 1346 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://checkpoint.com/llms.txt", "observed_at": "2026-09-20T06:24:33.288629+00:00", "status": 200, "final_url": "https://www.checkpoint.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "b3e5090e9c404c4020bb94cd166a3e5ea68623178a05a964bca0d7f9ca50c601", "bytes_read": 12490 }, "llms_state": "candidate", "control": { "requested_url": "https://checkpoint.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:33.675503+00:00", "status": 404, "final_url": "https://www.checkpoint.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 } }, { "rank": 356, "domain": "vecdnlb.com", "robots": { "requested_url": "https://vecdnlb.com/robots.txt", "observed_at": "2026-09-20T06:24:32.581096+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://vecdnlb.com/llms.txt", "observed_at": "2026-09-20T06:24:32.645408+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 357, "domain": "licdn.com", "robots": { "requested_url": "https://licdn.com/robots.txt", "observed_at": "2026-09-20T06:24:32.647976+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://licdn.com/llms.txt", "observed_at": "2026-09-20T06:24:32.679778+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 358, "domain": "selectel.ru", "robots": { "requested_url": "https://selectel.ru/robots.txt", "observed_at": "2026-09-20T06:24:32.682141+00:00", "status": 200, "final_url": "https://selectel.ru/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "f769518456fe0d3d3be324b7d18f3d70d415bfa50def79ff1218bcc597545477", "bytes_read": 1664 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://selectel.ru/llms.txt", "observed_at": "2026-09-20T06:24:33.649383+00:00", "status": 200, "final_url": "https://selectel.ru/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "bf943ec9ffaaf35dfe35bd9f691f3225a15fef5e2cb5d1f80a28437bade7fa86", "bytes_read": 509974 }, "llms_state": "unknown_control", "control": { "requested_url": "https://selectel.ru/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:35.340830+00:00", "status": 404, "final_url": "https://selectel.ru/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html;charset=utf-8", "truncated": true, "body_sha256": "e20ddbf232da901deb7a3d54d189ae95fcce748d498935b9890eae4a67ed4c30", "bytes_read": 512001 } }, { "rank": 359, "domain": "opendns.com", "robots": { "requested_url": "https://opendns.com/robots.txt", "observed_at": "2026-09-20T06:24:32.718704+00:00", "status": 200, "final_url": "https://www.opendns.com:443/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e5c4b84484ee4216e9373be99380320c25dd94805f99f0a805846f087636553f", "bytes_read": 24 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://opendns.com/llms.txt", "observed_at": "2026-09-20T06:24:33.325682+00:00", "status": 404, "final_url": "https://www.opendns.com:443/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "e646e6eb2d53864abc2ad075681b8efcfdb713113a136349a8f8b1cda08bad03", "bytes_read": 8064 }, "llms_state": "not_detected", "control": null }, { "rank": 360, "domain": "un.org", "robots": { "requested_url": "https://un.org/robots.txt", "observed_at": "2026-09-20T06:24:32.820166+00:00", "status": 200, "final_url": "https://www.un.org/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "f7186c2f743522cf71ee3def5babc565f04af28e65ff2d6793d4ac0a294ca875", "bytes_read": 2226 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://un.org/llms.txt", "observed_at": "2026-09-20T06:24:33.405713+00:00", "status": 404, "final_url": "https://www.un.org/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "d131e38fd1342d11a54fc205c84a0ee1ce2d3d5fafa07dd6b4447f01dc4c9c77", "bytes_read": 188 }, "llms_state": "not_detected", "control": null }, { "rank": 361, "domain": "cdc.gov", "robots": { "requested_url": "https://cdc.gov/robots.txt", "observed_at": "2026-09-20T06:24:33.351951+00:00", "status": 200, "final_url": "https://www.cdc.gov/index.html", "content_type": "text/html", "truncated": false, "body_sha256": "c5551730900818ed6279f1741051b8bd20fde7b805e5797a11cf04dea733ffb1", "bytes_read": 61600 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cdc.gov/llms.txt", "observed_at": "2026-09-20T06:24:33.855719+00:00", "status": 200, "final_url": "https://www.cdc.gov/index.html", "content_type": "text/html", "truncated": false, "body_sha256": "c5551730900818ed6279f1741051b8bd20fde7b805e5797a11cf04dea733ffb1", "bytes_read": 61600 }, "llms_state": "not_detected", "control": null }, { "rank": 362, "domain": "tradingview.com", "robots": { "requested_url": "https://tradingview.com/robots.txt", "observed_at": "2026-09-20T06:24:33.870374+00:00", "status": 200, "final_url": "https://www.tradingview.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "417f07c0a1a6e8050bab7082327ce84acacab18f2932cf273e101947e787ab36", "bytes_read": 1100 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [] } }, "llms": { "requested_url": "https://tradingview.com/llms.txt", "observed_at": "2026-09-20T06:24:34.376829+00:00", "status": 404, "final_url": "https://www.tradingview.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "85f648c3d9ea3f8b238f603276214fec8bf8c48b743c0cc277db020e65406f3a", "bytes_read": 146830 }, "llms_state": "not_detected", "control": null }, { "rank": 363, "domain": "allawnos.com", "robots": { "requested_url": "https://allawnos.com/robots.txt", "observed_at": "2026-09-20T06:24:33.870611+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://allawnos.com/llms.txt", "observed_at": "2026-09-20T06:24:33.913668+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 364, "domain": "washingtonpost.com", "robots": { "requested_url": "https://washingtonpost.com/robots.txt", "observed_at": "2026-09-20T06:24:33.915779+00:00", "status": 200, "final_url": "https://www.washingtonpost.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "918970c4ae59aa1f1f31577e14eec569caf79d69660bd06c381516e4d5c49426", "bytes_read": 5804 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://washingtonpost.com/llms.txt", "observed_at": "2026-09-20T06:24:34.587265+00:00", "status": null, "error": "TimeoutError" }, "llms_state": "unavailable", "control": null }, { "rank": 365, "domain": "weebly.com", "robots": { "requested_url": "https://weebly.com/robots.txt", "observed_at": "2026-09-20T06:24:33.973401+00:00", "status": 200, "final_url": "https://www.weebly.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "9a3d2c43e22adf6fa051b580c7b4ad025be069a223aa49ee996334eeb52a511a", "bytes_read": 777 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://weebly.com/llms.txt", "observed_at": "2026-09-20T06:24:34.567899+00:00", "status": 404, "final_url": "https://www.weebly.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "06f39fd0ff86ca2f837fb70fe38559c8bab1a65213d8bef69a3a27202dc94038", "bytes_read": 4133 }, "llms_state": "not_detected", "control": null }, { "rank": 366, "domain": "dnspod.net", "robots": { "requested_url": "https://dnspod.net/robots.txt", "observed_at": "2026-09-20T06:24:34.046754+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://dnspod.net/llms.txt", "observed_at": "2026-09-20T06:24:44.185554+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 367, "domain": "zendesk.com", "robots": { "requested_url": "https://zendesk.com/robots.txt", "observed_at": "2026-09-20T06:24:34.096708+00:00", "status": 200, "final_url": "https://www.zendesk.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "febf032bdcc18f83dbd17c75b2903f569c43c3f21f364616b2cda97714c36166", "bytes_read": 1028 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://zendesk.com/llms.txt", "observed_at": "2026-09-20T06:24:34.652401+00:00", "status": 200, "final_url": "https://www.zendesk.com/llms.txt", "content_type": "text/plain", "truncated": true, "body_sha256": "d59506eb503d2dd5133af538761cdd3128003c979d9516bd87eeaa52b3335509", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 368, "domain": "nvidia.com", "robots": { "requested_url": "https://nvidia.com/robots.txt", "observed_at": "2026-09-20T06:24:34.255848+00:00", "status": 200, "final_url": "https://www.nvidia.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "5069f0e6713ea599a59d48d5fb160ac859af14dcce9ec2a87fb9f554fe0f7125", "bytes_read": 3725 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://nvidia.com/llms.txt", "observed_at": "2026-09-20T06:24:34.970913+00:00", "status": 200, "final_url": "https://www.nvidia.com/llms.txt", "content_type": "text/markdown; charset=UTF-8", "truncated": false, "body_sha256": "069e726170096ccb0357322dc6d66d6cc9680914903367dd0f1b60a2c0b163d2", "bytes_read": 2610 }, "llms_state": "candidate", "control": { "requested_url": "https://nvidia.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:35.690398+00:00", "status": 404, "final_url": "https://www.nvidia.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "39c4b47477c2afe03816da775fb6fbf6feaf01afff763374472c03459c48bf24", "bytes_read": 297307 } }, { "rank": 369, "domain": "hostgator.com", "robots": { "requested_url": "https://hostgator.com/robots.txt", "observed_at": "2026-09-20T06:24:34.897788+00:00", "status": 403, "final_url": "https://www.hostgator.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "f964831bf348612e00553b07358b1708d0da8d91adf57829bc78681c0b058ea1", "bytes_read": 387773 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://hostgator.com/llms.txt", "observed_at": "2026-09-20T06:24:35.452684+00:00", "status": 404, "final_url": "https://www.hostgator.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "c195dde6276123043ecba0febe94390312595b3eed90f5c4a004d4c438626afb", "bytes_read": 98215 }, "llms_state": "not_detected", "control": null }, { "rank": 370, "domain": "edgcdn.net", "robots": { "requested_url": "https://edgcdn.net/robots.txt", "observed_at": "2026-09-20T06:24:35.021898+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://edgcdn.net/llms.txt", "observed_at": "2026-09-20T06:24:35.104211+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 371, "domain": "live-video.net", "robots": { "requested_url": "https://live-video.net/robots.txt", "observed_at": "2026-09-20T06:24:35.106831+00:00", "status": 403, "final_url": "https://live-video.net/robots.txt", "content_type": "application/xml", "truncated": false, "body_sha256": "a824bc7739e226e1b40ea0f8c4e4f4c6f796fc3b4abfa6e9abe3bd119a30d938", "bytes_read": 111 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://live-video.net/llms.txt", "observed_at": "2026-09-20T06:24:35.524271+00:00", "status": 403, "final_url": "https://live-video.net/llms.txt", "content_type": "application/xml", "truncated": false, "body_sha256": "a824bc7739e226e1b40ea0f8c4e4f4c6f796fc3b4abfa6e9abe3bd119a30d938", "bytes_read": 111 }, "llms_state": "unavailable", "control": null }, { "rank": 372, "domain": "indeed.com", "robots": { "requested_url": "https://indeed.com/robots.txt", "observed_at": "2026-09-20T06:24:35.121200+00:00", "status": 200, "final_url": "https://www.indeed.com/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "16bd850eb5e090b8dde037b183d9af9b79e57df5b21e51f05b5f2d5829d0e03d", "bytes_read": 13695 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [] } }, "llms": { "requested_url": "https://indeed.com/llms.txt", "observed_at": "2026-09-20T06:24:35.639302+00:00", "status": 403, "final_url": "https://www.indeed.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "e1b3d45af96ed4314619bab4508e2bb8a0093b98ee50b370147afc1863c11d7a", "bytes_read": 27563 }, "llms_state": "unavailable", "control": null }, { "rank": 373, "domain": "issuu.com", "robots": { "requested_url": "https://issuu.com/robots.txt", "observed_at": "2026-09-20T06:24:35.510341+00:00", "status": 200, "final_url": "https://issuu.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "923ce4f8f4bb78876c0f3215744f1ddb8edef190b10a90f863f536117ff98496", "bytes_read": 4866 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://issuu.com/llms.txt", "observed_at": "2026-09-20T06:24:35.755309+00:00", "status": 404, "final_url": "https://issuu.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "29a9e4c60b70a72b3edc888ac999e5a89e43ceaf05452a71b3b0c62211439c06", "bytes_read": 50550 }, "llms_state": "not_detected", "control": null }, { "rank": 374, "domain": "cdnbuild.net", "robots": { "requested_url": "https://cdnbuild.net/robots.txt", "observed_at": "2026-09-20T06:24:35.759757+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cdnbuild.net/llms.txt", "observed_at": "2026-09-20T06:24:35.846926+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 375, "domain": "cookiedatabase.org", "robots": { "requested_url": "https://cookiedatabase.org/robots.txt", "observed_at": "2026-09-20T06:24:35.849506+00:00", "status": 403, "final_url": "https://cookiedatabase.org/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "ba704180240385372db07253cd226f5a3a73f01211930ed6e0337348932e2d28", "bytes_read": 5545 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cookiedatabase.org/llms.txt", "observed_at": "2026-09-20T06:24:36.137181+00:00", "status": 403, "final_url": "https://cookiedatabase.org/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "755fa2b2d56a30e2f547880cc0f9e5fb5f0c7d2bed14527ea68836e7d354a9ce", "bytes_read": 5518 }, "llms_state": "unavailable", "control": null }, { "rank": 376, "domain": "samsungcloudsolution.com", "robots": { "requested_url": "https://samsungcloudsolution.com/robots.txt", "observed_at": "2026-09-20T06:24:35.904462+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://samsungcloudsolution.com/llms.txt", "observed_at": "2026-09-20T06:24:35.976982+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 377, "domain": "wsj.com", "robots": { "requested_url": "https://wsj.com/robots.txt", "observed_at": "2026-09-20T06:24:35.979517+00:00", "status": 200, "final_url": "https://www.wsj.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "89210f23a6252c1f03f3615e35e81ab50714fcea36cb56e8ef7725fdb91de3aa", "bytes_read": 4298 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://wsj.com/llms.txt", "observed_at": "2026-09-20T06:24:36.528612+00:00", "status": 403, "final_url": "https://www.wsj.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6561675785e8369d2bbdc7edb7a5672d72580c64c02e97f92ac4e1bda7157424", "bytes_read": 447743 }, "llms_state": "unavailable", "control": null }, { "rank": 378, "domain": "mi.com", "robots": { "requested_url": "https://mi.com/robots.txt", "observed_at": "2026-09-20T06:24:36.053175+00:00", "status": 403, "final_url": "https://www.mi.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "d9ed9f6496d789de056e4ce54a370797e251113001b4b8aa17836d5e76549120", "bytes_read": 378 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://mi.com/llms.txt", "observed_at": "2026-09-20T06:24:37.574065+00:00", "status": 403, "final_url": "https://www.mi.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "267f04ada8ba83322724bca8cc376a9d8f1876a057b2d7ca6b3d4bf603dc8a2a", "bytes_read": 376 }, "llms_state": "unavailable", "control": null }, { "rank": 379, "domain": "bloomberg.com", "robots": { "requested_url": "https://bloomberg.com/robots.txt", "observed_at": "2026-09-20T06:24:36.236125+00:00", "status": 200, "final_url": "https://www.bloomberg.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "9e9fde22f5fb63a5a1c2037b611e4b67aa232a03d0046b73267afd2e53800b37", "bytes_read": 7048 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://bloomberg.com/llms.txt", "observed_at": "2026-09-20T06:24:36.839910+00:00", "status": 403, "final_url": "https://www.bloomberg.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "2b91608e88713d96e1b61e951d29e10f9bad814c420c2ad1db0cffdecbe32187", "bytes_read": 13856 }, "llms_state": "unavailable", "control": null }, { "rank": 380, "domain": "calendly.com", "robots": { "requested_url": "https://calendly.com/robots.txt", "observed_at": "2026-09-20T06:24:36.282598+00:00", "status": 200, "final_url": "https://calendly.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "0bc39da8943d0ba14995453d49ea18216195b036f97c135bd17a52aa9410ba2d", "bytes_read": 335 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://calendly.com/llms.txt", "observed_at": "2026-09-20T06:24:36.561739+00:00", "status": 200, "final_url": "https://calendly.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "4f128b124eb255750cd2514b297cedfece0678a233c24f8ed149df47134b2bcf", "bytes_read": 11913 }, "llms_state": "candidate", "control": { "requested_url": "https://calendly.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:36.868929+00:00", "status": 404, "final_url": "https://calendly.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "68eedde16098db49162ff0d6480bc247236b77fef2e50388dc4e035ec3a23295", "bytes_read": 208996 } }, { "rank": 381, "domain": "stanford.edu", "robots": { "requested_url": "https://stanford.edu/robots.txt", "observed_at": "2026-09-20T06:24:37.184303+00:00", "status": 200, "final_url": "https://www.stanford.edu/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "aadf777d4727200060490d80c1e1c942e45fbdc03ee179aaee40ef1de72dffbd", "bytes_read": 197 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://stanford.edu/llms.txt", "observed_at": "2026-09-20T06:24:38.003595+00:00", "status": 404, "final_url": "https://www.stanford.edu/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "344153742a13ef9dfbcf904bdd9fdfdc298a44a3acd127b7f1126a79f0a6689d", "bytes_read": 235912 }, "llms_state": "not_detected", "control": null }, { "rank": 382, "domain": "volcfcdndvs.com", "robots": { "requested_url": "https://volcfcdndvs.com/robots.txt", "observed_at": "2026-09-20T06:24:37.197023+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://volcfcdndvs.com/llms.txt", "observed_at": "2026-09-20T06:24:37.508757+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 383, "domain": "tiktokv.us", "robots": { "requested_url": "https://tiktokv.us/robots.txt", "observed_at": "2026-09-20T06:24:37.511083+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://tiktokv.us/llms.txt", "observed_at": "2026-09-20T06:24:37.573675+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 384, "domain": "gitlab.com", "robots": { "requested_url": "https://gitlab.com/robots.txt", "observed_at": "2026-09-20T06:24:37.576084+00:00", "status": 200, "final_url": "https://gitlab.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "9e1e837e03b726eedb04c30b565e69068ccbb96c4d12c7c090d9affcf54f5eb0", "bytes_read": 3230 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://gitlab.com/llms.txt", "observed_at": "2026-09-20T06:24:37.834405+00:00", "status": 403, "final_url": "https://gitlab.com/users/sign_in", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "73c6a9118fcd163b61f616f3eaae7e615a6b86c819f834f2544f4219a38fa6d0", "bytes_read": 5546 }, "llms_state": "unavailable", "control": null }, { "rank": 385, "domain": "ip-api.com", "robots": { "requested_url": "https://ip-api.com/robots.txt", "observed_at": "2026-09-20T06:24:37.588563+00:00", "status": 200, "final_url": "https://ip-api.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "1aba7979031f193e9507c8b6d560c4b3efe7e2c42a6b834670c26c10bd0b94c9", "bytes_read": 213 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://ip-api.com/llms.txt", "observed_at": "2026-09-20T06:24:37.919747+00:00", "status": 200, "final_url": "https://ip-api.com/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "61dc7c95fb5fb72ff523c69063a4d510d6724697560b5db4efe9e78592bf9846", "bytes_read": 11848 }, "llms_state": "not_detected", "control": null }, { "rank": 386, "domain": "virginm.net", "robots": { "requested_url": "https://virginm.net/robots.txt", "observed_at": "2026-09-20T06:24:37.836263+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://virginm.net/llms.txt", "observed_at": "2026-09-20T06:24:38.013785+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 387, "domain": "online-metrix.net", "robots": { "requested_url": "https://online-metrix.net/robots.txt", "observed_at": "2026-09-20T06:24:38.016289+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://online-metrix.net/llms.txt", "observed_at": "2026-09-20T06:24:38.107910+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 388, "domain": "plesk.com", "robots": { "requested_url": "https://plesk.com/robots.txt", "observed_at": "2026-09-20T06:24:38.110859+00:00", "status": 200, "final_url": "https://www.plesk.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "4be359500e94ab7012a02835ef4a6c40bac5f07fc02a86f40ea72f21ff81c3e8", "bytes_read": 101 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://plesk.com/llms.txt", "observed_at": "2026-09-20T06:24:38.588990+00:00", "status": 200, "final_url": "https://www.plesk.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "a0ce0842763cda9d25e3e8487b31d97521d6007a77a3b4fecf7fb190e977b41d", "bytes_read": 8871 }, "llms_state": "candidate", "control": { "requested_url": "https://plesk.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:38.877833+00:00", "status": 404, "final_url": "https://www.plesk.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "7b1fc1d8ac48937d82e52e4c58a9e3e259ed9a92b72ed9a8eb4ac3565804f5d3", "bytes_read": 299340 } }, { "rank": 389, "domain": "newrelic.com", "robots": { "requested_url": "https://newrelic.com/robots.txt", "observed_at": "2026-09-20T06:24:38.216720+00:00", "status": 200, "final_url": "https://newrelic.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "f6ee1db86519f30130314f77a9cea7aa1005357824b8f20b1ae21952b69f12e4", "bytes_read": 2556 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://newrelic.com/llms.txt", "observed_at": "2026-09-20T06:24:38.425799+00:00", "status": 200, "final_url": "https://newrelic.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "6c4dd104f35e1acf265955af74de61457fb15b700644e177e85e40174d860b30", "bytes_read": 43840 }, "llms_state": "candidate", "control": { "requested_url": "https://newrelic.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:39.634896+00:00", "status": 404, "final_url": "https://newrelic.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "246dc4cfa50791e7cd14569e3427e3e65cb16f0a23662cb428f1c84941d03cbb", "bytes_read": 404470 } }, { "rank": 390, "domain": "yandex.com", "robots": { "requested_url": "https://yandex.com/robots.txt", "observed_at": "2026-09-20T06:24:38.437573+00:00", "status": 200, "final_url": "https://yandex.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "7afa5219ab35a8321c93c5a690b4e5460bd585c4dc0ac00f331612002974877b", "bytes_read": 21167 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://yandex.com/llms.txt", "observed_at": "2026-09-20T06:24:39.373510+00:00", "status": 404, "final_url": "https://yandex.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "c2f4324dd9379107e32b499e969650f71698a29a576071acb05bd80bcdd5535e", "bytes_read": 12550 }, "llms_state": "not_detected", "control": null }, { "rank": 391, "domain": "paloaltonetworks.com", "robots": { "requested_url": "https://paloaltonetworks.com/robots.txt", "observed_at": "2026-09-20T06:24:38.612092+00:00", "status": 200, "final_url": "https://www.paloaltonetworks.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "329486a1c300341e2b2ad756655830d5be25f139a09a7d0a7425408b896ba157", "bytes_read": 7701 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://paloaltonetworks.com/llms.txt", "observed_at": "2026-09-20T06:24:39.308708+00:00", "status": 200, "final_url": "https://www.paloaltonetworks.com/llms.txt", "content_type": "text/plain;charset=utf-8", "truncated": false, "body_sha256": "14e55414db825c4385a8e3ade91481b28fcc6392c1a43f67ae945eaf97f42bd6", "bytes_read": 17903 }, "llms_state": "candidate", "control": { "requested_url": "https://paloaltonetworks.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:39.770558+00:00", "status": 404, "final_url": "https://www.paloaltonetworks.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "f5d2ede0348e4a2212c258ae0f48631b37a3b31f035d433d8c41bc9d167a63d6", "bytes_read": 431400 } }, { "rank": 392, "domain": "netangels.ru", "robots": { "requested_url": "https://netangels.ru/robots.txt", "observed_at": "2026-09-20T06:24:38.801298+00:00", "status": 200, "final_url": "https://www.netangels.ru/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "d96259f1abb73ab61cb6518a7c8aaef54b6e1f192c973bbae0b480e890fc3084", "bytes_read": 1992 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://netangels.ru/llms.txt", "observed_at": "2026-09-20T06:24:40.500345+00:00", "status": 404, "final_url": "https://www.netangels.ru/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5", "bytes_read": 9 }, "llms_state": "not_detected", "control": null }, { "rank": 393, "domain": "amazon.de", "robots": { "requested_url": "https://amazon.de/robots.txt", "observed_at": "2026-09-20T06:24:39.150926+00:00", "status": 200, "final_url": "https://www.amazon.de/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "d266f6a17e1194ccdabe6b88ef2120d35ff617009c9815d8fec0d62123dc68a2", "bytes_read": 7508 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://amazon.de/llms.txt", "observed_at": "2026-09-20T06:24:40.187077+00:00", "status": 404, "final_url": "https://www.amazon.de/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "71dcf5ae0567c14adf860cd310c4e319d35dfa211d7dbb1dc9d8fc74c23e2ea6", "bytes_read": 2328 }, "llms_state": "not_detected", "control": null }, { "rank": 394, "domain": "amazontrust.com", "robots": { "requested_url": "https://amazontrust.com/robots.txt", "observed_at": "2026-09-20T06:24:39.910144+00:00", "status": 404, "final_url": "https://www.amazontrust.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "3500253d7dbbd9a42fc46ee314bd4ed8a280369876ecc66298af48392a91b355", "bytes_read": 12629 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://amazontrust.com/llms.txt", "observed_at": "2026-09-20T06:24:40.506186+00:00", "status": null, "error": "TimeoutError" }, "llms_state": "unavailable", "control": null }, { "rank": 395, "domain": "tp-link.com", "robots": { "requested_url": "https://tp-link.com/robots.txt", "observed_at": "2026-09-20T06:24:39.995862+00:00", "status": 200, "final_url": "https://www.tp-link.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "ec49f49069ec2c95db417e84ef3ea212e62d348cb7bbfd8b3a9584646575820f", "bytes_read": 544 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://tp-link.com/llms.txt", "observed_at": "2026-09-20T06:24:40.475564+00:00", "status": 404, "final_url": "https://www.tp-link.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "c0d5464cf8e9e90f8cfa7f61e607242e53817b541e6d1cf6831413363f457b66", "bytes_read": 1867 }, "llms_state": "not_detected", "control": null }, { "rank": 396, "domain": "nasa.gov", "robots": { "requested_url": "https://nasa.gov/robots.txt", "observed_at": "2026-09-20T06:24:40.227950+00:00", "status": 200, "final_url": "https://www.nasa.gov/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "44d72c5ad1a94bb45f25caaba917c39b39b383af3a121b59a5a7e46fbe3ccb05", "bytes_read": 69 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://nasa.gov/llms.txt", "observed_at": "2026-09-20T06:24:40.663214+00:00", "status": 404, "final_url": "https://www.nasa.gov/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "ad013b815fd86025cab7646675ee72f9949efbf2295b3fa98c07263d1e9ec745", "bytes_read": 196046 }, "llms_state": "not_detected", "control": null }, { "rank": 397, "domain": "vkuser.net", "robots": { "requested_url": "https://vkuser.net/robots.txt", "observed_at": "2026-09-20T06:24:40.357575+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://vkuser.net/llms.txt", "observed_at": "2026-09-20T06:24:50.570012+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 398, "domain": "shopifysvc.com", "robots": { "requested_url": "https://shopifysvc.com/robots.txt", "observed_at": "2026-09-20T06:24:40.760515+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://shopifysvc.com/llms.txt", "observed_at": "2026-09-20T06:24:40.822961+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 399, "domain": "ibyteimg.com", "robots": { "requested_url": "https://ibyteimg.com/robots.txt", "observed_at": "2026-09-20T06:24:40.789959+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ibyteimg.com/llms.txt", "observed_at": "2026-09-20T06:24:40.860690+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 400, "domain": "gandi-ns.fr", "robots": { "requested_url": "https://gandi-ns.fr/robots.txt", "observed_at": "2026-09-20T06:24:40.825448+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://gandi-ns.fr/llms.txt", "observed_at": "2026-09-20T06:24:40.973815+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 401, "domain": "xcal.tv", "robots": { "requested_url": "https://xcal.tv/robots.txt", "observed_at": "2026-09-20T06:24:40.863087+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://xcal.tv/llms.txt", "observed_at": "2026-09-20T06:24:40.945122+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 402, "domain": "bsky.app", "robots": { "requested_url": "https://bsky.app/robots.txt", "observed_at": "2026-09-20T06:24:40.948595+00:00", "status": 200, "final_url": "https://bsky.app/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "17e761d28a21d862c01f69aa5e7e69908e64076f17303d059276bea3a8aea891", "bytes_read": 441 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://bsky.app/llms.txt", "observed_at": "2026-09-20T06:24:41.295527+00:00", "status": 404, "final_url": "https://bsky.app/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "not_detected", "control": null }, { "rank": 403, "domain": "demdex.net", "robots": { "requested_url": "https://demdex.net/robots.txt", "observed_at": "2026-09-20T06:24:40.976180+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://demdex.net/llms.txt", "observed_at": "2026-09-20T06:24:41.035723+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 404, "domain": "espn.com", "robots": { "requested_url": "https://espn.com/robots.txt", "observed_at": "2026-09-20T06:24:40.993926+00:00", "status": 200, "final_url": "https://www.espn.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "69e57dadcb6f535e960fcc201f1f8fcd998bc4423f0261186408274b1d8b86a6", "bytes_read": 3223 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://espn.com/llms.txt", "observed_at": "2026-09-20T06:24:41.422537+00:00", "status": 404, "final_url": "https://www.espn.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "36a9e7f1c95b82ffb99743e0c5c4ce95d83c9a430aac59f84ef3cbfab6145068", "bytes_read": 1 }, "llms_state": "not_detected", "control": null }, { "rank": 405, "domain": "vivoglobal.com", "robots": { "requested_url": "https://vivoglobal.com/robots.txt", "observed_at": "2026-09-20T06:24:41.039071+00:00", "status": 200, "final_url": "https://vivoglobal.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "16ceb5ee3e0dc13aa9adf31a3ebbe45a1d965b8c2b9f72eaf84e5911e140ed95", "bytes_read": 23 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://vivoglobal.com/llms.txt", "observed_at": "2026-09-20T06:24:42.264699+00:00", "status": 404, "final_url": "https://vivoglobal.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 }, "llms_state": "not_detected", "control": null }, { "rank": 406, "domain": "ipify.org", "robots": { "requested_url": "https://ipify.org/robots.txt", "observed_at": "2026-09-20T06:24:41.355307+00:00", "status": 200, "final_url": "https://www.ipify.org/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e5c4b84484ee4216e9373be99380320c25dd94805f99f0a805846f087636553f", "bytes_read": 24 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://ipify.org/llms.txt", "observed_at": "2026-09-20T06:24:42.130621+00:00", "status": 404, "final_url": "https://www.ipify.org/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "a4072cd888ab5f672879cb1440ef5ffd0580293007bb3300454c8e1a42776b5c", "bytes_read": 514 }, "llms_state": "not_detected", "control": null }, { "rank": 407, "domain": "onelink.me", "robots": { "requested_url": "https://onelink.me/robots.txt", "observed_at": "2026-09-20T06:24:41.430945+00:00", "status": 200, "final_url": "https://www.appsflyer.com/products/deep-linking/", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "9030d7dbe88a36528e4cc64a8927e2f0f8e29d0a8f00f3985958c177c12f4de5", "bytes_read": 424337 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://onelink.me/llms.txt", "observed_at": "2026-09-20T06:24:42.125215+00:00", "status": 200, "final_url": "https://www.appsflyer.com/products/deep-linking/", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "e4280f2e339a99351d0ee4037d2861fca7c471c710b6d43016ba887d529d1507", "bytes_read": 424337 }, "llms_state": "not_detected", "control": null }, { "rank": 408, "domain": "rocket-cdn.com", "robots": { "requested_url": "https://rocket-cdn.com/robots.txt", "observed_at": "2026-09-20T06:24:41.543823+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://rocket-cdn.com/llms.txt", "observed_at": "2026-09-20T06:24:41.847751+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 409, "domain": "kwai-pro.com", "robots": { "requested_url": "https://kwai-pro.com/robots.txt", "observed_at": "2026-09-20T06:24:41.569328+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://kwai-pro.com/llms.txt", "observed_at": "2026-09-20T06:24:41.632278+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 410, "domain": "quickconnect.to", "robots": { "requested_url": "https://quickconnect.to/robots.txt", "observed_at": "2026-09-20T06:24:41.634459+00:00", "status": 200, "final_url": "https://quickconnect.to/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", "bytes_read": 1 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://quickconnect.to/llms.txt", "observed_at": "2026-09-20T06:24:42.170534+00:00", "status": 200, "final_url": "http://llms.txt.quickconnect.to/https_first", "content_type": "text/html", "truncated": false, "body_sha256": "275e9dfac715bc52b080e7be64e327e16275da32e762198e1479c8c750e3d609", "bytes_read": 82086 }, "llms_state": "not_detected", "control": null }, { "rank": 411, "domain": "nflxvideo.net", "robots": { "requested_url": "https://nflxvideo.net/robots.txt", "observed_at": "2026-09-20T06:24:41.808189+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://nflxvideo.net/llms.txt", "observed_at": "2026-09-20T06:24:41.985660+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 412, "domain": "aliexpress.com", "robots": { "requested_url": "https://aliexpress.com/robots.txt", "observed_at": "2026-09-20T06:24:41.850074+00:00", "status": 200, "final_url": "https://www.aliexpress.com/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "c08fbfef7316346e2d395db0532badf2feec38a7a003078d3b90e6adc13cac85", "bytes_read": 1971 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://aliexpress.com/llms.txt", "observed_at": "2026-09-20T06:24:43.964714+00:00", "status": 200, "final_url": "https://www.aliexpress.com/s/error/404", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "330127181c4beb39a295902463d21767fada8187b26f4160127e34778ca1d36f", "bytes_read": 13874 }, "llms_state": "not_detected", "control": null }, { "rank": 413, "domain": "launchdarkly.com", "robots": { "requested_url": "https://launchdarkly.com/robots.txt", "observed_at": "2026-09-20T06:24:42.082185+00:00", "status": 200, "final_url": "https://launchdarkly.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "a9459689ee5b2e8fc20a6b0791eaf39b6371a52d7203cf4171fe834ec62fe5b4", "bytes_read": 251 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://launchdarkly.com/llms.txt", "observed_at": "2026-09-20T06:24:42.342693+00:00", "status": 404, "final_url": "https://launchdarkly.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "12af4097800610c47c64d2615aa3507f6a65ba2cb4e6fd29cecc11f019665b3c", "bytes_read": 77554 }, "llms_state": "not_detected", "control": null }, { "rank": 414, "domain": "globo.com", "robots": { "requested_url": "https://globo.com/robots.txt", "observed_at": "2026-09-20T06:24:42.502306+00:00", "status": 200, "final_url": "https://www.globo.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "afe0cd909d91b1403bb861811b8678438150e02caa8fdd2e884362f4f59b5c03", "bytes_read": 205 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://globo.com/llms.txt", "observed_at": "2026-09-20T06:24:44.047757+00:00", "status": 404, "final_url": "https://www.globo.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "41038beb75dcf6c5e6e8b5180f3e08982f9f32ea59a9b9c1edca7d3245f4e8cd", "bytes_read": 4167 }, "llms_state": "not_detected", "control": null }, { "rank": 415, "domain": "unsplash.com", "robots": { "requested_url": "https://unsplash.com/robots.txt", "observed_at": "2026-09-20T06:24:42.697149+00:00", "status": 200, "final_url": "https://unsplash.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "48c47b9141d837ed6feeb4921b2f56b2995e48e5fef58b59af614b5f3eb61276", "bytes_read": 8706 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://unsplash.com/llms.txt", "observed_at": "2026-09-20T06:24:42.983519+00:00", "status": 401, "final_url": "https://unsplash.com/.within.website?redir=%2Fllms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "cce0c9bc4989e37f19e036d581fbb29562562ad7402ee89ac914856cc1349b98", "bytes_read": 24885 }, "llms_state": "unavailable", "control": null }, { "rank": 416, "domain": "163.com", "robots": { "requested_url": "https://163.com/robots.txt", "observed_at": "2026-09-20T06:24:42.802726+00:00", "status": 200, "final_url": "http://www.163.com/robots.txt", "content_type": "text/plain; charset=gb2312", "truncated": false, "body_sha256": "f8a61c9d8c19bd1d32ca782eba7007d560b7487579d816cb0351b1945b13268e", "bytes_read": 225 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://163.com/llms.txt", "observed_at": "2026-09-20T06:24:45.094432+00:00", "status": 200, "final_url": "http://www.163.com/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "c66a92e30d856fc7ab4e10f1d7b406a846b5eebb61cdbce96240bffbdab3dad3", "bytes_read": 274836 }, "llms_state": "not_detected", "control": null }, { "rank": 417, "domain": "expireddomains.com", "robots": { "requested_url": "https://expireddomains.com/robots.txt", "observed_at": "2026-09-20T06:24:42.870236+00:00", "status": 404, "final_url": "https://expireddomains.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "948807b2ed9ded90e74032545c1fb6f31aa8e561dfa46544d6fdede95f61dce7", "bytes_read": 35868 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://expireddomains.com/llms.txt", "observed_at": "2026-09-20T06:24:43.276368+00:00", "status": 404, "final_url": "https://expireddomains.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "f7b15bfeeb00c981246f446ca41ab93cdfb3dbd0ad42873f3bee65ec38b7ef24", "bytes_read": 35868 }, "llms_state": "not_detected", "control": null }, { "rank": 418, "domain": "heytapdl.com", "robots": { "requested_url": "https://heytapdl.com/robots.txt", "observed_at": "2026-09-20T06:24:43.387458+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://heytapdl.com/llms.txt", "observed_at": "2026-09-20T06:24:43.467505+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 419, "domain": "walmart.com", "robots": { "requested_url": "https://walmart.com/robots.txt", "observed_at": "2026-09-20T06:24:43.392466+00:00", "status": 200, "final_url": "https://www.walmart.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "2a7f8d1550d91c0ef5262f4fca3099a89a117249ee59999b9e70db30de3d3eeb", "bytes_read": 3584 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://walmart.com/llms.txt", "observed_at": "2026-09-20T06:24:44.327111+00:00", "status": 200, "final_url": "https://www.walmart.com/blocked?url=L2xsbXMudHh0&uuid=f896225e-b4bb-11f1-b48a-c00c86f8e11e&vid=&g=b", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "35ba06376bc2ead89ab20e31981c96ee6a4b04a13c705f2312fb50bf1ee76c9d", "bytes_read": 15195 }, "llms_state": "not_detected", "control": null }, { "rank": 420, "domain": "ezvizlife.com", "robots": { "requested_url": "https://ezvizlife.com/robots.txt", "observed_at": "2026-09-20T06:24:43.432550+00:00", "status": 200, "final_url": "https://www.ezviz.com/cn", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "208d008643297596a559ab80478583f6fb0f40b0ac6a0b7c3668a12c79846632", "bytes_read": 63932 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ezvizlife.com/llms.txt", "observed_at": "2026-09-20T06:24:44.456308+00:00", "status": 200, "final_url": "https://www.ezviz.com/cn", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "208d008643297596a559ab80478583f6fb0f40b0ac6a0b7c3668a12c79846632", "bytes_read": 63932 }, "llms_state": "not_detected", "control": null }, { "rank": 421, "domain": "line.me", "robots": { "requested_url": "https://line.me/robots.txt", "observed_at": "2026-09-20T06:24:43.471414+00:00", "status": 200, "final_url": "https://line.me/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "238a5714cec095cbec743614bbc0458c5ed4f00ac43303cd61252d9380808a4c", "bytes_read": 454 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://line.me/llms.txt", "observed_at": "2026-09-20T06:24:44.309043+00:00", "status": 404, "final_url": "https://line.me/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "3cf0665eb2eb9a25b81dacd759e27878ccaa58410855883d00a2127a81183021", "bytes_read": 7835 }, "llms_state": "not_detected", "control": null }, { "rank": 422, "domain": "brave.com", "robots": { "requested_url": "https://brave.com/robots.txt", "observed_at": "2026-09-20T06:24:44.856709+00:00", "status": 200, "final_url": "https://brave.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "105ae9ec1bb33213b73be71b290c54eef3a15bc98c9780f6e4669d67ff68b4d2", "bytes_read": 53 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://brave.com/llms.txt", "observed_at": "2026-09-20T06:24:45.089110+00:00", "status": 404, "final_url": "https://brave.com/llms.txt/", "content_type": "text/html", "truncated": false, "body_sha256": "5325638be2cea2f784fa20d9dc6e8d9366036ccf251244fe2d5aa87bb7cc15ac", "bytes_read": 54499 }, "llms_state": "not_detected", "control": null }, { "rank": 423, "domain": "tiktokv.eu", "robots": { "requested_url": "https://tiktokv.eu/robots.txt", "observed_at": "2026-09-20T06:24:44.896476+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://tiktokv.eu/llms.txt", "observed_at": "2026-09-20T06:24:44.981657+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 424, "domain": "google.cn", "robots": { "requested_url": "https://google.cn/robots.txt", "observed_at": "2026-09-20T06:24:44.919553+00:00", "status": 404, "final_url": "https://google.cn/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2", "bytes_read": 1571 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://google.cn/llms.txt", "observed_at": "2026-09-20T06:24:45.224131+00:00", "status": 404, "final_url": "https://google.cn/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 425, "domain": "steamcommunity.com", "robots": { "requested_url": "https://steamcommunity.com/robots.txt", "observed_at": "2026-09-20T06:24:44.983410+00:00", "status": 200, "final_url": "https://steamcommunity.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "4e3f5ea02946c4677b77402a16fcbecfc02beb880742bc8abb641abe99e82d22", "bytes_read": 141 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://steamcommunity.com/llms.txt", "observed_at": "2026-09-20T06:24:45.264861+00:00", "status": 200, "final_url": "https://steamcommunity.com/", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "f1b76bcb910d25ae8ccb68eab66af7162d946f1caa365ab0cce5e9bd83d40240", "bytes_read": 47571 }, "llms_state": "not_detected", "control": null }, { "rank": 426, "domain": "amazonalexa.com", "robots": { "requested_url": "https://amazonalexa.com/robots.txt", "observed_at": "2026-09-20T06:24:45.021390+00:00", "status": 200, "final_url": "https://www.amazon.com:443/b?node=210710065011", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "370d22972e95d1aef0918f2007685cc941f0fcca7c1496d2ff975d5827d55e8e", "bytes_read": 138437 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://amazonalexa.com/llms.txt", "observed_at": "2026-09-20T06:24:45.781728+00:00", "status": 200, "final_url": "https://www.amazon.com:443/b?node=210710065011", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "3815e3a0b881e6440b9d6bdc80446544741981421505f221f59d15928ed0bc24", "bytes_read": 156390 }, "llms_state": "not_detected", "control": null }, { "rank": 427, "domain": "dailymotion.com", "robots": { "requested_url": "https://dailymotion.com/robots.txt", "observed_at": "2026-09-20T06:24:45.111010+00:00", "status": 200, "final_url": "https://www.dailymotion.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "6d8bdcdf3f738c7645802f35c4850ea1f33e7fc0f9e0d883411d2e042706575e", "bytes_read": 18276 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://dailymotion.com/llms.txt", "observed_at": "2026-09-20T06:24:46.283173+00:00", "status": 200, "final_url": "https://www.dailymotion.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "252ecdbe75b42fcbd43219e933db4ce05bfe4fa62f3f82585cbab8b0a40cf81b", "bytes_read": 6153 }, "llms_state": "not_detected", "control": null }, { "rank": 428, "domain": "beian.gov.cn", "robots": { "requested_url": "https://beian.gov.cn/robots.txt", "observed_at": "2026-09-20T06:24:45.470974+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://beian.gov.cn/llms.txt", "observed_at": "2026-09-20T06:24:45.790929+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 429, "domain": "tiktokcdn-eu.com", "robots": { "requested_url": "https://tiktokcdn-eu.com/robots.txt", "observed_at": "2026-09-20T06:24:45.756325+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://tiktokcdn-eu.com/llms.txt", "observed_at": "2026-09-20T06:24:45.840628+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 430, "domain": "temu.com", "robots": { "requested_url": "https://temu.com/robots.txt", "observed_at": "2026-09-20T06:24:45.793403+00:00", "status": 200, "final_url": "https://www.temu.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e6791399506405731bf4f737044195aee793aa81d5c17e12f7a79d82c0b3d8b8", "bytes_read": 8239 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/*" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://temu.com/llms.txt", "observed_at": "2026-09-20T06:24:46.487956+00:00", "status": 404, "final_url": "https://www.temu.com/llms.txt", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "not_detected", "control": null }, { "rank": 431, "domain": "supercell.com", "robots": { "requested_url": "https://supercell.com/robots.txt", "observed_at": "2026-09-20T06:24:45.842947+00:00", "status": 200, "final_url": "https://supercell.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "d6d762ba31c79cb16680576742f4cd0e85f629aadac6343fef4c748993da11a7", "bytes_read": 66 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://supercell.com/llms.txt", "observed_at": "2026-09-20T06:24:46.160350+00:00", "status": 404, "final_url": "https://supercell.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "6cf36d99542e45dc7b06c19b4b0b23ad0dd4ad7a89b177a8a044ec1ed0d7a880", "bytes_read": 18437 }, "llms_state": "not_detected", "control": null }, { "rank": 432, "domain": "chinamobile.com", "robots": { "requested_url": "https://chinamobile.com/robots.txt", "observed_at": "2026-09-20T06:24:46.023249+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://chinamobile.com/llms.txt", "observed_at": "2026-09-20T06:24:56.341587+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 433, "domain": "kaspersky-labs.com", "robots": { "requested_url": "https://kaspersky-labs.com/robots.txt", "observed_at": "2026-09-20T06:24:46.456557+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://kaspersky-labs.com/llms.txt", "observed_at": "2026-09-20T06:24:46.537147+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 434, "domain": "huawei.com", "robots": { "requested_url": "https://huawei.com/robots.txt", "observed_at": "2026-09-20T06:24:46.459545+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://huawei.com/llms.txt", "observed_at": "2026-09-20T06:24:47.398402+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 435, "domain": "claude.ai", "robots": { "requested_url": "https://claude.ai/robots.txt", "observed_at": "2026-09-20T06:24:46.539191+00:00", "status": 200, "final_url": "https://claude.ai/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "75ccbccac0e02fbc11965050f3152c1ef7435fff085f0eee27592a82059b8902", "bytes_read": 281 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://claude.ai/llms.txt", "observed_at": "2026-09-20T06:24:46.712959+00:00", "status": 200, "final_url": "https://claude.ai/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "7c4b40046f09e8d92c7cb9533332c2e1e9fa08c6ff6cfdcb7bbf8b39f86fe336", "bytes_read": 122494 }, "llms_state": "not_detected", "control": null }, { "rank": 436, "domain": "avcdn.net", "robots": { "requested_url": "https://avcdn.net/robots.txt", "observed_at": "2026-09-20T06:24:46.670055+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://avcdn.net/llms.txt", "observed_at": "2026-09-20T06:24:46.734200+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 437, "domain": "pangle.io", "robots": { "requested_url": "https://pangle.io/robots.txt", "observed_at": "2026-09-20T06:24:46.736824+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://pangle.io/llms.txt", "observed_at": "2026-09-20T06:24:46.815484+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 438, "domain": "wildberries.ru", "robots": { "requested_url": "https://wildberries.ru/robots.txt", "observed_at": "2026-09-20T06:24:46.804297+00:00", "status": 498, "final_url": "https://www.wildberries.ru/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "f0b1155d8fdce838424f050da29e481b5241717475b96c9f679160b78ad511a8", "bytes_read": 1136 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://wildberries.ru/llms.txt", "observed_at": "2026-09-20T06:24:48.338671+00:00", "status": 498, "final_url": "https://www.wildberries.ru/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "737cfb49cb4d5bb42d51c2f7f861643df13c400a390cabbf39ca3bcd1cf6bb88", "bytes_read": 1136 }, "llms_state": "unavailable", "control": null }, { "rank": 439, "domain": "intel.com", "robots": { "requested_url": "https://intel.com/robots.txt", "observed_at": "2026-09-20T06:24:46.817650+00:00", "status": 200, "final_url": "https://www.intel.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "930a465dbf2bd43fbcb048f3c092e13a036497efbaa86dee5bad4a8ec7219f8d", "bytes_read": 4211 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://intel.com/llms.txt", "observed_at": "2026-09-20T06:24:47.535466+00:00", "status": 200, "final_url": "https://www.intel.com/llms.txt", "content_type": "text/plain;charset=utf-8", "truncated": false, "body_sha256": "69845398118dd3d29170b557397b17dd8104d99225e4c10f48be60fc3b02fb96", "bytes_read": 12892 }, "llms_state": "candidate", "control": { "requested_url": "https://intel.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:48.060330+00:00", "status": 404, "final_url": "https://www.intel.com/content/www/us/en/404.html?ref=https://www.intel.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "0a3cdf5d223b004581d02f81313fff1d18403514728dcad1e883db1cb6111cbe", "bytes_read": 70155 } }, { "rank": 440, "domain": "taobao.com", "robots": { "requested_url": "https://taobao.com/robots.txt", "observed_at": "2026-09-20T06:24:46.898650+00:00", "status": 200, "final_url": "https://www.taobao.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "8a8ca7eb96a8e1a508acd13bed695655b1abd5a075d835d7902a3ba08aa56cd6", "bytes_read": 71 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ] ] } }, "llms": { "requested_url": "https://taobao.com/llms.txt", "observed_at": "2026-09-20T06:24:48.947512+00:00", "status": 200, "final_url": "https://error.taobao.com/tbpc/error.html?from=https://www.taobao.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "08e50bf268d4ca3fda5864aafa62e7d8f323fec18f51b2e87b6248347150fdfa", "bytes_read": 1996 }, "llms_state": "not_detected", "control": null }, { "rank": 441, "domain": "go.com", "robots": { "requested_url": "https://go.com/robots.txt", "observed_at": "2026-09-20T06:24:47.094349+00:00", "status": 200, "final_url": "https://www.go.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "14a7f27ad6aa72167f654474ca38730ea407cdaebfa259907028002b9937a417", "bytes_read": 212 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://go.com/llms.txt", "observed_at": "2026-09-20T06:24:47.745929+00:00", "status": 200, "final_url": "https://www.go.com/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "3c41e47c49c4ae7939bec5a791f6772afda025dd3e1e55ec6c6fb35abdd517ed", "bytes_read": 358895 }, "llms_state": "not_detected", "control": null }, { "rank": 442, "domain": "netease.com", "robots": { "requested_url": "https://netease.com/robots.txt", "observed_at": "2026-09-20T06:24:47.296316+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://netease.com/llms.txt", "observed_at": "2026-09-20T06:24:47.932206+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 443, "domain": "timeweb.ru", "robots": { "requested_url": "https://timeweb.ru/robots.txt", "observed_at": "2026-09-20T06:24:47.914457+00:00", "status": 200, "final_url": "https://timeweb.com/", "content_type": "text/html", "truncated": false, "body_sha256": "84c28e4c50c249caeef49e290219e3ed794175a41db1c978d171ceaaa6b3bf15", "bytes_read": 421495 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://timeweb.ru/llms.txt", "observed_at": "2026-09-20T06:24:49.949156+00:00", "status": 200, "final_url": "https://timeweb.com/", "content_type": "text/html", "truncated": false, "body_sha256": "0d1f1863623f74916fc81a001c0d32cad822ecbfc1271607e100ea96c1dc1167", "bytes_read": 421495 }, "llms_state": "not_detected", "control": null }, { "rank": 444, "domain": "kwaipros.com", "robots": { "requested_url": "https://kwaipros.com/robots.txt", "observed_at": "2026-09-20T06:24:47.975356+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://kwaipros.com/llms.txt", "observed_at": "2026-09-20T06:24:48.044601+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 445, "domain": "dropcatch.com", "robots": { "requested_url": "https://dropcatch.com/robots.txt", "observed_at": "2026-09-20T06:24:48.047233+00:00", "status": 200, "final_url": "https://www.dropcatch.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "ba0ba6a9a1a6324f2b3802eb4a49e492ddf27405516347ce3d31a331b465b580", "bytes_read": 104289 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://dropcatch.com/llms.txt", "observed_at": "2026-09-20T06:24:48.570420+00:00", "status": 200, "final_url": "https://www.dropcatch.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "ba0ba6a9a1a6324f2b3802eb4a49e492ddf27405516347ce3d31a331b465b580", "bytes_read": 104289 }, "llms_state": "not_detected", "control": null }, { "rank": 446, "domain": "byteglb.com", "robots": { "requested_url": "https://byteglb.com/robots.txt", "observed_at": "2026-09-20T06:24:48.409218+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://byteglb.com/llms.txt", "observed_at": "2026-09-20T06:24:48.508590+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 447, "domain": "awsglobalaccelerator.com", "robots": { "requested_url": "https://awsglobalaccelerator.com/robots.txt", "observed_at": "2026-09-20T06:24:48.429546+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://awsglobalaccelerator.com/llms.txt", "observed_at": "2026-09-20T06:24:48.493091+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 448, "domain": "dotaplabs.net", "robots": { "requested_url": "https://dotaplabs.net/robots.txt", "observed_at": "2026-09-20T06:24:48.442283+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://dotaplabs.net/llms.txt", "observed_at": "2026-09-20T06:24:48.673634+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 449, "domain": "skyhigh.cloud", "robots": { "requested_url": "https://skyhigh.cloud/robots.txt", "observed_at": "2026-09-20T06:24:48.495418+00:00", "status": 200, "final_url": "https://www.skyhighsecurity.com/", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "ccd2c0e892ded3a91b1eea5e140ca3bd6b24d569c638e5b55fe3d7cd7e89bf5b", "bytes_read": 316853 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://skyhigh.cloud/llms.txt", "observed_at": "2026-09-20T06:24:49.348169+00:00", "status": 200, "final_url": "https://www.skyhighsecurity.com/", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "ccd2c0e892ded3a91b1eea5e140ca3bd6b24d569c638e5b55fe3d7cd7e89bf5b", "bytes_read": 316853 }, "llms_state": "not_detected", "control": null }, { "rank": 450, "domain": "ipinfo.io", "robots": { "requested_url": "https://ipinfo.io/robots.txt", "observed_at": "2026-09-20T06:24:48.510707+00:00", "status": 200, "final_url": "https://ipinfo.io/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "dc28b45d5b491141be16d863b59e364d333eb0b988a58a325f75a0c1eefc8021", "bytes_read": 793 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://ipinfo.io/llms.txt", "observed_at": "2026-09-20T06:24:48.779916+00:00", "status": 404, "final_url": "https://ipinfo.io/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "a2e1fb64baceae165be241ed3930bf4640db68dfaa757877026e4a788ff00adf", "bytes_read": 34273 }, "llms_state": "not_detected", "control": null }, { "rank": 451, "domain": "wbx2.com", "robots": { "requested_url": "https://wbx2.com/robots.txt", "observed_at": "2026-09-20T06:24:48.675367+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://wbx2.com/llms.txt", "observed_at": "2026-09-20T06:25:28.737887+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 452, "domain": "substack.com", "robots": { "requested_url": "https://substack.com/robots.txt", "observed_at": "2026-09-20T06:24:48.948563+00:00", "status": 200, "final_url": "https://substack.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "e468186aecca926921a597e3b593f101d82741e04874f7f723007d47adb3b1e8", "bytes_read": 585 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://substack.com/llms.txt", "observed_at": "2026-09-20T06:24:49.189133+00:00", "status": 404, "final_url": "https://substack.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "a82db04a945d96913d3840b268d59efa6b71eb62e05482c55cf3a5227d7851e2", "bytes_read": 60311 }, "llms_state": "not_detected", "control": null }, { "rank": 453, "domain": "goodreads.com", "robots": { "requested_url": "https://goodreads.com/robots.txt", "observed_at": "2026-09-20T06:24:49.063714+00:00", "status": 200, "final_url": "https://www.goodreads.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "e1738b2f727969388be5a47f0186c2a50203292e26f4ec0186f4393d4694cab9", "bytes_read": 2647 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://goodreads.com/llms.txt", "observed_at": "2026-09-20T06:24:49.726881+00:00", "status": 404, "final_url": "https://www.goodreads.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "66a9332915c09f8479a278322bbacda576ea6c00c432097a797d6850854a2610", "bytes_read": 60674 }, "llms_state": "not_detected", "control": null }, { "rank": 454, "domain": "myqcloud.com", "robots": { "requested_url": "https://myqcloud.com/robots.txt", "observed_at": "2026-09-20T06:24:49.233171+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://myqcloud.com/llms.txt", "observed_at": "2026-09-20T06:24:49.377186+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 455, "domain": "prodregistryv2.org", "robots": { "requested_url": "https://prodregistryv2.org/robots.txt", "observed_at": "2026-09-20T06:24:49.379493+00:00", "status": 404, "final_url": "https://prodregistryv2.org/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "4ff444d1abe117bbe3055e97182351923ce4db715959b733900bb28f72fa0e29", "bytes_read": 18 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://prodregistryv2.org/llms.txt", "observed_at": "2026-09-20T06:24:49.669737+00:00", "status": 404, "final_url": "https://prodregistryv2.org/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "4ff444d1abe117bbe3055e97182351923ce4db715959b733900bb28f72fa0e29", "bytes_read": 18 }, "llms_state": "not_detected", "control": null }, { "rank": 456, "domain": "cloudns.net", "robots": { "requested_url": "https://cloudns.net/robots.txt", "observed_at": "2026-09-20T06:24:49.409908+00:00", "status": 200, "final_url": "https://www.cloudns.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "3b1ed861f26dd7b7c7340b0b782b965fbb90eee66e92b7462972e4c0c53bfbd5", "bytes_read": 25 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://cloudns.net/llms.txt", "observed_at": "2026-09-20T06:24:50.571332+00:00", "status": 200, "final_url": "https://www.cloudns.net/", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "4836e8d7051d5d4af9eac43f0f3ba625eade88e179753cbdfd400a6bc2993f01", "bytes_read": 173755 }, "llms_state": "not_detected", "control": null }, { "rank": 457, "domain": "yahoo.co.jp", "robots": { "requested_url": "https://yahoo.co.jp/robots.txt", "observed_at": "2026-09-20T06:24:49.477593+00:00", "status": 200, "final_url": "https://www.yahoo.co.jp:443/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "d0e013f7c6a36fdfe5bd5cc85968164511fccece54fe032e428e69cf965385fb", "bytes_read": 87 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://yahoo.co.jp/llms.txt", "observed_at": "2026-09-20T06:24:51.118678+00:00", "status": 200, "final_url": "https://www.yahoo.co.jp/", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "979f53bb89d3c6a25af6343d7219f4240d6afb03f3b20115885fc5502fa216a4", "bytes_read": 33258 }, "llms_state": "not_detected", "control": null }, { "rank": 458, "domain": "enacdn.net", "robots": { "requested_url": "https://enacdn.net/robots.txt", "observed_at": "2026-09-20T06:24:49.862914+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://enacdn.net/llms.txt", "observed_at": "2026-09-20T06:24:49.945678+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 459, "domain": "businessinsider.com", "robots": { "requested_url": "https://businessinsider.com/robots.txt", "observed_at": "2026-09-20T06:24:49.949640+00:00", "status": 200, "final_url": "https://www.businessinsider.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "89f51cf7e3b532fe80b2057cf13765fdc52bc55aafc263e876a37fe120873ddd", "bytes_read": 2231 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://businessinsider.com/llms.txt", "observed_at": "2026-09-20T06:24:50.455041+00:00", "status": 404, "final_url": "https://www.businessinsider.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "38a7b860ed4e6252a421b1b108e08faacb3f1181e878482ce752bfb79724df67", "bytes_read": 181914 }, "llms_state": "not_detected", "control": null }, { "rank": 460, "domain": "adobe.net", "robots": { "requested_url": "https://adobe.net/robots.txt", "observed_at": "2026-09-20T06:24:50.039470+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://adobe.net/llms.txt", "observed_at": "2026-09-20T06:24:50.421426+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 461, "domain": "tencent-cloud.net", "robots": { "requested_url": "https://tencent-cloud.net/robots.txt", "observed_at": "2026-09-20T06:24:50.258793+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://tencent-cloud.net/llms.txt", "observed_at": "2026-09-20T06:24:50.331965+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 462, "domain": "gcdn.co", "robots": { "requested_url": "https://gcdn.co/robots.txt", "observed_at": "2026-09-20T06:24:50.334322+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://gcdn.co/llms.txt", "observed_at": "2026-09-20T06:24:50.769778+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 463, "domain": "sina.com.cn", "robots": { "requested_url": "https://sina.com.cn/robots.txt", "observed_at": "2026-09-20T06:24:50.603484+00:00", "status": 200, "final_url": "https://www.sina.com.cn/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "4fc307b72ced6adbeda9abb8a0776988cd1517e711c797b155de11f0d84a58e3", "bytes_read": 528 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://sina.com.cn/llms.txt", "observed_at": "2026-09-20T06:24:52.086343+00:00", "status": 404, "final_url": "https://www.sina.com.cn/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "58f9100952a849d87108e23094cfb17dc24ea1ae998d8161228d2c12ea4c3170", "bytes_read": 1010 }, "llms_state": "not_detected", "control": null }, { "rank": 464, "domain": "edgecdn.ru", "robots": { "requested_url": "https://edgecdn.ru/robots.txt", "observed_at": "2026-09-20T06:24:50.691495+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://edgecdn.ru/llms.txt", "observed_at": "2026-09-20T06:24:51.240209+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 465, "domain": "pixabay.com", "robots": { "requested_url": "https://pixabay.com/robots.txt", "observed_at": "2026-09-20T06:24:50.849744+00:00", "status": 200, "final_url": "https://pixabay.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1b0d4f9b4b12b727626d632d774ce60a905c3f0c3ac73d58dbcc2f20d7d349cc", "bytes_read": 3248 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://pixabay.com/llms.txt", "observed_at": "2026-09-20T06:24:51.175923+00:00", "status": 403, "final_url": "https://pixabay.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "5e10491086b69f13dc9b0572da6ff416df568a8f459fe1170db7d9a937bbda86", "bytes_read": 5532 }, "llms_state": "unavailable", "control": null }, { "rank": 466, "domain": "wp.com", "robots": { "requested_url": "https://wp.com/robots.txt", "observed_at": "2026-09-20T06:24:51.053571+00:00", "status": 200, "final_url": "https://wordpress.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "462d4866929634bf5cc361d3af1d4b9cff0f744bca3a2f3bb7a6b6c67a1c496d", "bytes_read": 1387 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://wp.com/llms.txt", "observed_at": "2026-09-20T06:24:51.442350+00:00", "status": 200, "final_url": "https://wordpress.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "4c085ee948dd6fbdecf247242cd38b6ebde72b16fe78938438a9975503ff842b", "bytes_read": 7018 }, "llms_state": "candidate", "control": { "requested_url": "https://wp.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:51.785940+00:00", "status": 404, "final_url": "https://wordpress.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "c7311ee845efb3728587742d83a881bc514a3cf1139c5c4e37bc6d477351d407", "bytes_read": 144494 } }, { "rank": 467, "domain": "aol.com", "robots": { "requested_url": "https://aol.com/robots.txt", "observed_at": "2026-09-20T06:24:51.306939+00:00", "status": 200, "final_url": "https://www.aol.com:443/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "8d5f1df3e224414de17aaa66d5cdb18733b12082a149d8898ca55f0435b49b16", "bytes_read": 2207 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://aol.com/llms.txt", "observed_at": "2026-09-20T06:24:51.873846+00:00", "status": 404, "final_url": "https://www.aol.com:443/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "22b273def95c1c9c52afd38c5564604c7be6dc2ba1f1972c4cc5d5fe805e0628", "bytes_read": 99231 }, "llms_state": "not_detected", "control": null }, { "rank": 468, "domain": "avsxappcaptiveportal.com", "robots": { "requested_url": "https://avsxappcaptiveportal.com/robots.txt", "observed_at": "2026-09-20T06:24:51.662976+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://avsxappcaptiveportal.com/llms.txt", "observed_at": "2026-09-20T06:26:11.730694+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 469, "domain": "vercel-dns-016.com", "robots": { "requested_url": "https://vercel-dns-016.com/robots.txt", "observed_at": "2026-09-20T06:24:52.027864+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://vercel-dns-016.com/llms.txt", "observed_at": "2026-09-20T06:24:52.107650+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 470, "domain": "speedtest.net", "robots": { "requested_url": "https://speedtest.net/robots.txt", "observed_at": "2026-09-20T06:24:52.109602+00:00", "status": 200, "final_url": "https://www.speedtest.net/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "23506808181b89ef437a22d10574d97f9bbf371623fcbf50f3346dec395342f9", "bytes_read": 247 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://speedtest.net/llms.txt", "observed_at": "2026-09-20T06:24:52.528733+00:00", "status": 404, "final_url": "https://www.speedtest.net/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "ec33b7dcad809d5bd7e6961b7a3bec6d23aca0f6f0222641bfd55b35042df0d8", "bytes_read": 71979 }, "llms_state": "not_detected", "control": null }, { "rank": 471, "domain": "g.page", "robots": { "requested_url": "https://g.page/robots.txt", "observed_at": "2026-09-20T06:24:52.171185+00:00", "status": 200, "final_url": "https://www.google.com/search?q=robots.txt", "content_type": "text/html; charset=ISO-8859-1", "truncated": false, "body_sha256": "ff020845f08ccf3653811c198653631c9ad9332364f89eb231f374ca62f565b4", "bytes_read": 91909 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://g.page/llms.txt", "observed_at": "2026-09-20T06:24:53.278493+00:00", "status": 200, "final_url": "https://www.google.com/search?q=llms.txt", "content_type": "text/html; charset=ISO-8859-1", "truncated": false, "body_sha256": "f7b5e5a155a611995ae12f9116df13ce09b5bd076f285fd8189fb80a920c255f", "bytes_read": 91903 }, "llms_state": "not_detected", "control": null }, { "rank": 472, "domain": "cloudflareinsights.com", "robots": { "requested_url": "https://cloudflareinsights.com/robots.txt", "observed_at": "2026-09-20T06:24:52.354034+00:00", "status": 200, "final_url": "https://cloudflareinsights.com/robots.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "508d8607891d71e7d33fbec82b11d5aa16817122d5a93673a92ae411bf18d4d7", "bytes_read": 498 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cloudflareinsights.com/llms.txt", "observed_at": "2026-09-20T06:24:52.527191+00:00", "status": 200, "final_url": "https://cloudflareinsights.com/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "508d8607891d71e7d33fbec82b11d5aa16817122d5a93673a92ae411bf18d4d7", "bytes_read": 498 }, "llms_state": "not_detected", "control": null }, { "rank": 473, "domain": "duolingo.com", "robots": { "requested_url": "https://duolingo.com/robots.txt", "observed_at": "2026-09-20T06:24:52.407626+00:00", "status": 200, "final_url": "https://www.duolingo.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "13dbb017d9b20e5b8ccce6a1d4eb4eb9984d0684763dbc49a0cf17d3f91e74f7", "bytes_read": 826 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://duolingo.com/llms.txt", "observed_at": "2026-09-20T06:24:53.011389+00:00", "status": 200, "final_url": "https://www.duolingo.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47232765b3951c417f2d0c8344db21284926791981f271befbd3a36efb513f0a", "bytes_read": 11579 }, "llms_state": "not_detected", "control": null }, { "rank": 474, "domain": "eye4.cn", "robots": { "requested_url": "https://eye4.cn/robots.txt", "observed_at": "2026-09-20T06:24:52.576800+00:00", "status": 500, "final_url": "https://eye4.cn/robots.txt", "content_type": "text/html; charset=iso-8859-1", "truncated": false, "body_sha256": "4cac43d10ca4583ec38a2581d346033d0d5b6999eef1eeb3ed62f8fee475e617", "bytes_read": 592 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://eye4.cn/llms.txt", "observed_at": "2026-09-20T06:24:53.530360+00:00", "status": 403, "final_url": "https://eye4.cn/llms.txt", "content_type": "text/html; charset=iso-8859-1", "truncated": false, "body_sha256": "ce87365ad8c1a7fef7573b3722e24217f4bf301575b2831d72962735462da65a", "bytes_read": 257 }, "llms_state": "unavailable", "control": null }, { "rank": 475, "domain": "gamepass.com", "robots": { "requested_url": "https://gamepass.com/robots.txt", "observed_at": "2026-09-20T06:24:52.656815+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://gamepass.com/llms.txt", "observed_at": "2026-09-20T06:24:53.350734+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 476, "domain": "mikrotik.com", "robots": { "requested_url": "https://mikrotik.com/robots.txt", "observed_at": "2026-09-20T06:24:52.774241+00:00", "status": 200, "final_url": "https://mikrotik.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "52cc0410879dcc820131e15982e63b9ccd737c3ec423f0b49b0711aee3035983", "bytes_read": 66 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://mikrotik.com/llms.txt", "observed_at": "2026-09-20T06:24:53.579517+00:00", "status": 404, "final_url": "https://mikrotik.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "5099d27284c2257d2983450585cbd4bede6475519755508047e213d985cbc7c9", "bytes_read": 14 }, "llms_state": "not_detected", "control": null }, { "rank": 477, "domain": "sohu.com", "robots": { "requested_url": "https://sohu.com/robots.txt", "observed_at": "2026-09-20T06:24:52.821713+00:00", "status": 200, "final_url": "https://www.sohu.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "06a2166c549a518ed89bfd544c9f8ddaeab5ea2d655e1955988e5a227b702aab", "bytes_read": 3097 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://sohu.com/llms.txt", "observed_at": "2026-09-20T06:24:55.531699+00:00", "status": 200, "final_url": "https://www.sohu.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "cc01bbeea998987aee109523c2bebb9bd5621d877d009db924c5cf8c16c70ef8", "bytes_read": 8511 }, "llms_state": "not_detected", "control": null }, { "rank": 478, "domain": "nflximg.com", "robots": { "requested_url": "https://nflximg.com/robots.txt", "observed_at": "2026-09-20T06:24:53.124557+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://nflximg.com/llms.txt", "observed_at": "2026-09-20T06:25:03.210807+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 479, "domain": "arxiv.org", "robots": { "requested_url": "https://arxiv.org/robots.txt", "observed_at": "2026-09-20T06:24:53.464516+00:00", "status": 406, "final_url": "https://arxiv.org/robots.txt", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://arxiv.org/llms.txt", "observed_at": "2026-09-20T06:24:53.720993+00:00", "status": 406, "final_url": "https://arxiv.org/llms.txt", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "unavailable", "control": null }, { "rank": 480, "domain": "dual-s-msedge.net", "robots": { "requested_url": "https://dual-s-msedge.net/robots.txt", "observed_at": "2026-09-20T06:24:53.530892+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://dual-s-msedge.net/llms.txt", "observed_at": "2026-09-20T06:24:53.593392+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 481, "domain": "firetvcaptiveportal.com", "robots": { "requested_url": "https://firetvcaptiveportal.com/robots.txt", "observed_at": "2026-09-20T06:24:53.595152+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://firetvcaptiveportal.com/llms.txt", "observed_at": "2026-09-20T06:26:13.670620+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 482, "domain": "bdydns.com", "robots": { "requested_url": "https://bdydns.com/robots.txt", "observed_at": "2026-09-20T06:24:53.856599+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://bdydns.com/llms.txt", "observed_at": "2026-09-20T06:24:54.184430+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 483, "domain": "kwai.net", "robots": { "requested_url": "https://kwai.net/robots.txt", "observed_at": "2026-09-20T06:24:53.924836+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://kwai.net/llms.txt", "observed_at": "2026-09-20T06:25:03.997654+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 484, "domain": "cnbc.com", "robots": { "requested_url": "https://cnbc.com/robots.txt", "observed_at": "2026-09-20T06:24:54.042717+00:00", "status": 200, "final_url": "https://www.cnbc.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "010d7b0c49939bec886c60ee850729570b62dcd87a680a90f01a39ec9b0823fc", "bytes_read": 2477 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://cnbc.com/llms.txt", "observed_at": "2026-09-20T06:24:54.748727+00:00", "status": 404, "final_url": "https://www.cnbc.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "4aef3c9aa2d33943d5f33c1b2e14e040b977b69eb1c649f779fe2dd059eb443d", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 485, "domain": "statista.com", "robots": { "requested_url": "https://statista.com/robots.txt", "observed_at": "2026-09-20T06:24:54.186472+00:00", "status": 200, "final_url": "https://www.statista.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "57bfb95a1d305aca2c2a197a29c0431d5b32a8272dfc78655b81645442fba0c0", "bytes_read": 11030 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://statista.com/llms.txt", "observed_at": "2026-09-20T06:24:54.764905+00:00", "status": 404, "final_url": "https://www.statista.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "cc1b332a637131198c644b3ec9e15c42ea64ed9b17bc898bcb2005c65ab07c60", "bytes_read": 261104 }, "llms_state": "not_detected", "control": null }, { "rank": 486, "domain": "duckduckgo.com", "robots": { "requested_url": "https://duckduckgo.com/robots.txt", "observed_at": "2026-09-20T06:24:54.189748+00:00", "status": 200, "final_url": "https://duckduckgo.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "02fc159e8e3fefe13845427b27022341b6dc9e62a96e5605f59c2c0d03ee0baf", "bytes_read": 757 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://duckduckgo.com/llms.txt", "observed_at": "2026-09-20T06:24:54.489053+00:00", "status": 200, "final_url": "https://duckduckgo.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "baad32cd1b8a781dad3c80d356ea3dfb5591599dcc0dd21d7e93b3c54d354ebd", "bytes_read": 18949 }, "llms_state": "not_detected", "control": null }, { "rank": 487, "domain": "list-manage.com", "robots": { "requested_url": "https://list-manage.com/robots.txt", "observed_at": "2026-09-20T06:24:54.313740+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://list-manage.com/llms.txt", "observed_at": "2026-09-20T06:25:04.400581+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 488, "domain": "nease.net", "robots": { "requested_url": "https://nease.net/robots.txt", "observed_at": "2026-09-20T06:24:54.399612+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://nease.net/llms.txt", "observed_at": "2026-09-20T06:25:04.540670+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 489, "domain": "addtoany.com", "robots": { "requested_url": "https://addtoany.com/robots.txt", "observed_at": "2026-09-20T06:24:54.777021+00:00", "status": 200, "final_url": "https://www.addtoany.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "e55fbe65fbbf99bffdb3919f924a91ba85fada00f1277d1f16c317dcd9b6d592", "bytes_read": 308 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://addtoany.com/llms.txt", "observed_at": "2026-09-20T06:24:55.292675+00:00", "status": 404, "final_url": "https://www.addtoany.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "9f43189cc4ab4436f60cd3d97fd604580775d16d8fff26694958ff6a684b6386", "bytes_read": 2021 }, "llms_state": "not_detected", "control": null }, { "rank": 490, "domain": "ietf.org", "robots": { "requested_url": "https://ietf.org/robots.txt", "observed_at": "2026-09-20T06:24:54.840306+00:00", "status": 200, "final_url": "https://www.ietf.org/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "ab0325f3171f1f1e6e221b92ec6c86909ea782804d5cf35f1802542cd78ef377", "bytes_read": 51 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://ietf.org/llms.txt", "observed_at": "2026-09-20T06:24:55.391495+00:00", "status": 404, "final_url": "https://www.ietf.org/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "a6877e1d3c7570b8017d017f4606014920ded68303af811052ab6a1f6d0491d0", "bytes_read": 77030 }, "llms_state": "not_detected", "control": null }, { "rank": 491, "domain": "okta.com", "robots": { "requested_url": "https://okta.com/robots.txt", "observed_at": "2026-09-20T06:24:55.394554+00:00", "status": 200, "final_url": "https://www.okta.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "c1f9eb29d75800e04f12993a7c62259ed447a6afb5d4ee352318cd2469d5c533", "bytes_read": 2518 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://okta.com/llms.txt", "observed_at": "2026-09-20T06:24:56.013141+00:00", "status": 200, "final_url": "https://www.okta.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "7c08e1b74e78ba8fabb3f60d87042b6e552b7c33430fededbf9d648a87e3f072", "bytes_read": 221071 }, "llms_state": "candidate", "control": { "requested_url": "https://okta.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:56.723934+00:00", "status": 404, "final_url": "https://www.okta.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "3cc3f22fc04bafe1bb428593a21fe543376dceb83179ca03452813416c312c50", "bytes_read": 29843 } }, { "rank": 492, "domain": "sc-gw.com", "robots": { "requested_url": "https://sc-gw.com/robots.txt", "observed_at": "2026-09-20T06:24:55.792347+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://sc-gw.com/llms.txt", "observed_at": "2026-09-20T06:24:55.873048+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 493, "domain": "amazon.co.jp", "robots": { "requested_url": "https://amazon.co.jp/robots.txt", "observed_at": "2026-09-20T06:24:55.874657+00:00", "status": 200, "final_url": "https://www.amazon.co.jp/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "81e1db815061b2b2616a43021615888d17cc5594a91786527a4ea74e61b35356", "bytes_read": 10174 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://amazon.co.jp/llms.txt", "observed_at": "2026-09-20T06:24:56.754603+00:00", "status": 404, "final_url": "https://www.amazon.co.jp/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "f30b722f59c9232decb8643ffaf294eb665920ebe449c91e96635ed870ca9c45", "bytes_read": 2674 }, "llms_state": "not_detected", "control": null }, { "rank": 494, "domain": "adriver.ru", "robots": { "requested_url": "https://adriver.ru/robots.txt", "observed_at": "2026-09-20T06:24:56.030396+00:00", "status": 404, "final_url": "https://adriver.ru/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://adriver.ru/llms.txt", "observed_at": "2026-09-20T06:24:56.768120+00:00", "status": 404, "final_url": "https://www.adriver.ru/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "43ecfc030b63cd11a98423f8397f367d45d0539ff178597b8b143eb39982a3af", "bytes_read": 148250 }, "llms_state": "not_detected", "control": null }, { "rank": 495, "domain": "quora.com", "robots": { "requested_url": "https://quora.com/robots.txt", "observed_at": "2026-09-20T06:24:56.688432+00:00", "status": 200, "final_url": "https://www.quora.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "bec85c4dc0fc0526c67cacae1daa4e6644b5d6278f05104e8617c867f9187ce3", "bytes_read": 18813 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://quora.com/llms.txt", "observed_at": "2026-09-20T06:24:57.290738+00:00", "status": 404, "final_url": "https://www.quora.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "b84472123d6f293860249f20402ca5023de7f2f2e690e666c3333183ab417a21", "bytes_read": 72069 }, "llms_state": "not_detected", "control": null }, { "rank": 496, "domain": "mcafee.com", "robots": { "requested_url": "https://mcafee.com/robots.txt", "observed_at": "2026-09-20T06:24:56.937480+00:00", "status": null, "error": "TimeoutError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://mcafee.com/llms.txt", "observed_at": "2026-09-20T06:25:07.989822+00:00", "status": null, "error": "TimeoutError" }, "llms_state": "unavailable", "control": null }, { "rank": 497, "domain": "bluehost.com", "robots": { "requested_url": "https://bluehost.com/robots.txt", "observed_at": "2026-09-20T06:24:57.224857+00:00", "status": 403, "final_url": "https://www.bluehost.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "79b21e6a303e1e50064ac4298d18dda82890a28d8d2c1dc269e0f49d726a2234", "bytes_read": 447189 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://bluehost.com/llms.txt", "observed_at": "2026-09-20T06:24:57.783965+00:00", "status": 200, "final_url": "https://www.bluehost.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "16bdc31db42551350e6fd697781b4123e8bd796260a5e4e9235d6cfbbb6c9242", "bytes_read": 7467 }, "llms_state": "unknown_control", "control": { "requested_url": "https://bluehost.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:24:58.134613+00:00", "status": 403, "final_url": "https://www.bluehost.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "f6de3dc3e4f6fded8aee996cbdad5c204d83f178ac86f627fe214b8d47e3e0f2", "bytes_read": 447321 } }, { "rank": 498, "domain": "atlassian.net", "robots": { "requested_url": "https://atlassian.net/robots.txt", "observed_at": "2026-09-20T06:24:57.383385+00:00", "status": 200, "final_url": "https://atlassian.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1575e5f767c4f585199ca83255518464fe302cf33ed89fd505df623866bfeb0d", "bytes_read": 347 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://atlassian.net/llms.txt", "observed_at": "2026-09-20T06:24:57.598188+00:00", "status": 200, "final_url": "https://www.atlassian.com/software", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "9bc95f1f4d687a5013375bf6957424660f90f386d5b0eeb1a827fb79582984f9", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 499, "domain": "ivi.ru", "robots": { "requested_url": "https://ivi.ru/robots.txt", "observed_at": "2026-09-20T06:24:57.869272+00:00", "status": 200, "final_url": "https://www.ivi.ru/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "ad8638ea811283a7da62999302ce10973b4068ae31e3d7ef6d72a72546b6a74f", "bytes_read": 6098 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://ivi.ru/llms.txt", "observed_at": "2026-09-20T06:24:59.027696+00:00", "status": 404, "final_url": "https://www.ivi.ru/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 }, "llms_state": "not_detected", "control": null }, { "rank": 500, "domain": "amazon.co.za", "robots": { "requested_url": "https://amazon.co.za/robots.txt", "observed_at": "2026-09-20T06:24:58.510843+00:00", "status": 200, "final_url": "https://www.amazon.co.za/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "42ee22d9d7f0ab6a6065f57c5244e129f27b3a76cb8ddcd79bb08a4414a0e2fc", "bytes_read": 7257 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://amazon.co.za/llms.txt", "observed_at": "2026-09-20T06:24:59.755453+00:00", "status": 404, "final_url": "https://www.amazon.co.za/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "f480bfff3341258cecc4325c5e8e5622c079f2de7c10353a77eba68943e00f21", "bytes_read": 2186 }, "llms_state": "not_detected", "control": null }, { "rank": 501, "domain": "dynatrace.com", "robots": { "requested_url": "https://dynatrace.com/robots.txt", "observed_at": "2026-09-20T06:24:58.553427+00:00", "status": 200, "final_url": "https://www.dynatrace.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "690c910ff4bc157eabebf12fe408f90e182b76ad3fc494565248b4d144f9b732", "bytes_read": 319 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://dynatrace.com/llms.txt", "observed_at": "2026-09-20T06:24:59.605759+00:00", "status": 200, "final_url": "https://www.dynatrace.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "79355f40b0a12abeb991d19f7bd0926b6abf181c853c49eee8c47f4db4d891a5", "bytes_read": 182111 }, "llms_state": "candidate", "control": { "requested_url": "https://dynatrace.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:01.012133+00:00", "status": 404, "final_url": "https://www.dynatrace.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "bdf9adb1db0e896e8c9af186e7eeed23a2d00e13888acb526c5f7cb28858826e", "bytes_read": 2714 } }, { "rank": 502, "domain": "impervadns.net", "robots": { "requested_url": "https://impervadns.net/robots.txt", "observed_at": "2026-09-20T06:24:59.915234+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://impervadns.net/llms.txt", "observed_at": "2026-09-20T06:24:59.994421+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 503, "domain": "scribd.com", "robots": { "requested_url": "https://scribd.com/robots.txt", "observed_at": "2026-09-20T06:24:59.995939+00:00", "status": 200, "final_url": "https://www.scribd.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "3467b2a622d186e465574e61ca7ce31c76879fbdb7b954a7c5699655f0b41ed5", "bytes_read": 3546 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://scribd.com/llms.txt", "observed_at": "2026-09-20T06:25:00.457779+00:00", "status": 404, "final_url": "https://www.scribd.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "1dec24838bfc9f64a7032bc495d6f71d537ca90e9ce7d47be3ea4b7eaaec5294", "bytes_read": 41889 }, "llms_state": "not_detected", "control": null }, { "rank": 504, "domain": "markmonitor.com", "robots": { "requested_url": "https://markmonitor.com/robots.txt", "observed_at": "2026-09-20T06:25:00.573188+00:00", "status": 403, "final_url": "https://markmonitor.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "31ab80e0a9bf45b1c224823fd2a004e3611a51a5687d16655999970bc86f92e1", "bytes_read": 5564 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://markmonitor.com/llms.txt", "observed_at": "2026-09-20T06:25:00.869618+00:00", "status": 403, "final_url": "https://markmonitor.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "789a6a6188c372f7112d12e082f87f4cff6f9680f6b9b1069754a92671c5b2f2", "bytes_read": 5579 }, "llms_state": "unavailable", "control": null }, { "rank": 505, "domain": "npr.org", "robots": { "requested_url": "https://npr.org/robots.txt", "observed_at": "2026-09-20T06:25:00.650428+00:00", "status": null, "error": "TimeoutError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://npr.org/llms.txt", "observed_at": "2026-09-20T06:25:11.110911+00:00", "status": null, "error": "TimeoutError" }, "llms_state": "unavailable", "control": null }, { "rank": 506, "domain": "mediatek.com", "robots": { "requested_url": "https://mediatek.com/robots.txt", "observed_at": "2026-09-20T06:25:00.782163+00:00", "status": 200, "final_url": "https://www.mediatek.com/", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "90366063aee484f89fe8a7f5583cc571a7479880a9beb16489b728bfe2798b1f", "bytes_read": 215716 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://mediatek.com/llms.txt", "observed_at": "2026-09-20T06:25:02.355724+00:00", "status": 200, "final_url": "https://www.mediatek.com/", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "984e4b1879c2ec2baf125d0e0907a42abddb402eb1e3b725742643a4d0ef2d44", "bytes_read": 215716 }, "llms_state": "not_detected", "control": null }, { "rank": 507, "domain": "shein.com", "robots": { "requested_url": "https://shein.com/robots.txt", "observed_at": "2026-09-20T06:25:00.851024+00:00", "status": 200, "final_url": "https://www.shein.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "7a6d32e222e319a6425d08100b814b1ab110ad21d1b08b520faeaa158d973735", "bytes_read": 279 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://shein.com/llms.txt", "observed_at": "2026-09-20T06:25:01.712957+00:00", "status": 200, "final_url": "https://www.shein.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "19f2eb866e9b8eff8ceb9cee0eca9dd614a9ab7dbf31b77b205a8be876c0c90d", "bytes_read": 63855 }, "llms_state": "candidate", "control": { "requested_url": "https://shein.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:02.505211+00:00", "status": 404, "final_url": "https://www.shein.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "52cc79d2e896022aa935919eb7e6e9ffdc06672d0e4a336cd097782417573d13", "bytes_read": 390713 } }, { "rank": 508, "domain": "docker.com", "robots": { "requested_url": "https://docker.com/robots.txt", "observed_at": "2026-09-20T06:25:00.992952+00:00", "status": 200, "final_url": "https://www.docker.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "bd14e1bfcf77ea4dce9dc4e88b8160639a83a7d2dd30dd5dbc3248948ebfc0aa", "bytes_read": 1033 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://docker.com/llms.txt", "observed_at": "2026-09-20T06:25:01.522578+00:00", "status": 404, "final_url": "https://www.docker.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "1e298488aca3c505e299f41ef2d1b380370abf41c3537d740390156c3ae6939f", "bytes_read": 205348 }, "llms_state": "not_detected", "control": null }, { "rank": 509, "domain": "run.app", "robots": { "requested_url": "https://run.app/robots.txt", "observed_at": "2026-09-20T06:25:01.858923+00:00", "status": 404, "final_url": "https://run.app/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2", "bytes_read": 1571 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://run.app/llms.txt", "observed_at": "2026-09-20T06:25:02.187607+00:00", "status": 404, "final_url": "https://run.app/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 510, "domain": "gosuslugi.ru", "robots": { "requested_url": "https://gosuslugi.ru/robots.txt", "observed_at": "2026-09-20T06:25:02.050449+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://gosuslugi.ru/llms.txt", "observed_at": "2026-09-20T06:25:22.265736+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 511, "domain": "stackoverflow.com", "robots": { "requested_url": "https://stackoverflow.com/robots.txt", "observed_at": "2026-09-20T06:25:02.420907+00:00", "status": 418, "final_url": "https://stackoverflow.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "da7453156d9b2849749e7455839e7a226e9688147bfa2d67f8f34248582ca26d", "bytes_read": 113 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://stackoverflow.com/llms.txt", "observed_at": "2026-09-20T06:25:02.613257+00:00", "status": 404, "final_url": "https://stackoverflow.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "d4f197960c7c00d8e84b14b45b48d1034c42993d728a8419c47cf648ff86e7a9", "bytes_read": 80463 }, "llms_state": "not_detected", "control": null }, { "rank": 512, "domain": "google.de", "robots": { "requested_url": "https://google.de/robots.txt", "observed_at": "2026-09-20T06:25:02.833339+00:00", "status": 200, "final_url": "https://www.google.de/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1aba570a1b32deb66941e0c29d4f37be71ee58aa1a624aa20754b3639538cf2c", "bytes_read": 6624 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://google.de/llms.txt", "observed_at": "2026-09-20T06:25:03.282118+00:00", "status": 404, "final_url": "https://google.de/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 513, "domain": "behance.net", "robots": { "requested_url": "https://behance.net/robots.txt", "observed_at": "2026-09-20T06:25:03.391790+00:00", "status": 200, "final_url": "https://www.behance.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1a55abe2475b1430444111a3ffceedde40c2782e40c2fda4a42ca95e2e02ba2b", "bytes_read": 1317 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://behance.net/llms.txt", "observed_at": "2026-09-20T06:25:03.790421+00:00", "status": 404, "final_url": "https://www.behance.net/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "e81e75ed86c272066cb902edea36f5c207e5a9e6f3af1e1325837d9dd3b18760", "bytes_read": 370293 }, "llms_state": "not_detected", "control": null }, { "rank": 514, "domain": "skybridge.click", "robots": { "requested_url": "https://skybridge.click/robots.txt", "observed_at": "2026-09-20T06:25:03.501230+00:00", "status": 200, "final_url": "https://skybridge.click/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "331ea9090db0c9f6f597bd9840fd5b171830f6e0b3ba1cb24dfa91f0c95aedc1", "bytes_read": 26 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://skybridge.click/llms.txt", "observed_at": "2026-09-20T06:25:04.465035+00:00", "status": 404, "final_url": "https://skybridge.click/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "d9391437c7986bc637393d374362a088dbe1182278b8ef3e7555f1c663fd11bc", "bytes_read": 11939 }, "llms_state": "not_detected", "control": null }, { "rank": 515, "domain": "foxnews.com", "robots": { "requested_url": "https://foxnews.com/robots.txt", "observed_at": "2026-09-20T06:25:03.515422+00:00", "status": 200, "final_url": "https://www.foxnews.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "4abc1bd3b4d0b87c64de7eba0eafe63dd90415f25d12133d88d7225cacb9dc07", "bytes_read": 301 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://foxnews.com/llms.txt", "observed_at": "2026-09-20T06:25:03.967140+00:00", "status": 200, "final_url": "https://www.foxnews.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "4a078f7b2ef22b42dd54845ca31b4eb1782e096c3eab0e4ada5733ad49c92327", "bytes_read": 2191 }, "llms_state": "candidate", "control": { "requested_url": "https://foxnews.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:04.296165+00:00", "status": 404, "final_url": "https://www.foxnews.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 } }, { "rank": 516, "domain": "eventbrite.com", "robots": { "requested_url": "https://eventbrite.com/robots.txt", "observed_at": "2026-09-20T06:25:04.388683+00:00", "status": 200, "final_url": "https://eventbrite.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "b7fa62d995e2b649df4b7a9ec9fea2bc0c7461276cb58e892562d4193c498222", "bytes_read": 7559 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://eventbrite.com/llms.txt", "observed_at": "2026-09-20T06:25:04.614585+00:00", "status": 404, "final_url": "https://www.eventbrite.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "13b1375364d4386999389cccea83225ad7b146cd84458f26c11718b237343170", "bytes_read": 79138 }, "llms_state": "not_detected", "control": null }, { "rank": 517, "domain": "libp2p.direct", "robots": { "requested_url": "https://libp2p.direct/robots.txt", "observed_at": "2026-09-20T06:25:04.638486+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://libp2p.direct/llms.txt", "observed_at": "2026-09-20T06:25:04.721404+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 518, "domain": "indiatimes.com", "robots": { "requested_url": "https://indiatimes.com/robots.txt", "observed_at": "2026-09-20T06:25:04.722969+00:00", "status": 200, "final_url": "https://www.indiatimes.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "f7912d9f8fe7de23512df6ea0452e1e8de600d48f94fde00244347981c894900", "bytes_read": 1143 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://indiatimes.com/llms.txt", "observed_at": "2026-09-20T06:25:05.623074+00:00", "status": 404, "final_url": "https://www.indiatimes.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "d8410106886f257ab5df9a78220c90bf8fb338b9adcb139f76fc8148d176c73f", "bytes_read": 202535 }, "llms_state": "not_detected", "control": null }, { "rank": 519, "domain": "amzn.to", "robots": { "requested_url": "https://amzn.to/robots.txt", "observed_at": "2026-09-20T06:25:05.314202+00:00", "status": 200, "final_url": "https://amzn.to/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "aeddaf37d71afe500676c334c63a84340d99c91651b9d41bc0c4b2f9437d6cd8", "bytes_read": 123 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://amzn.to/llms.txt", "observed_at": "2026-09-20T06:25:05.584733+00:00", "status": 200, "final_url": "https://www.amazon.com/", "content_type": "", "truncated": true, "body_sha256": "fad1bc02d84d09e41afa8f7ad6a19df3e049fa9745dfea069df38128812bea42", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 520, "domain": "mysql.com", "robots": { "requested_url": "https://mysql.com/robots.txt", "observed_at": "2026-09-20T06:25:05.332119+00:00", "status": 200, "final_url": "https://www.mysql.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "6cc119828bf339ccda83fb07889bda871954c926531e4aa15a43e5051d7c4f81", "bytes_read": 184 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://mysql.com/llms.txt", "observed_at": "2026-09-20T06:25:06.676019+00:00", "status": 404, "final_url": "https://www.mysql.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "a521a61e8806877e822ad6a60e864aad09f856ba588925e61dd67bc0af3b8183", "bytes_read": 19079 }, "llms_state": "not_detected", "control": null }, { "rank": 521, "domain": "ampproject.org", "robots": { "requested_url": "https://ampproject.org/robots.txt", "observed_at": "2026-09-20T06:25:06.344564+00:00", "status": 404, "final_url": "https://amp.dev/robots.txt/?referrer=ampproject.org", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "ef57bc7a593b59e852cfda52e28a36b193b7f05136c183e20bc08f8f134a6fff", "bytes_read": 77664 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ampproject.org/llms.txt", "observed_at": "2026-09-20T06:25:07.316260+00:00", "status": 404, "final_url": "https://amp.dev/llms.txt/?referrer=ampproject.org", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "ef57bc7a593b59e852cfda52e28a36b193b7f05136c183e20bc08f8f134a6fff", "bytes_read": 77664 }, "llms_state": "not_detected", "control": null }, { "rank": 522, "domain": "ngenix.net", "robots": { "requested_url": "https://ngenix.net/robots.txt", "observed_at": "2026-09-20T06:25:06.445226+00:00", "status": 503, "final_url": "https://ngenix.net/robots.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "8158cdda105b3b3207ae239f3a2acc4aef1456825f02955d688aa4855e493e3d", "bytes_read": 7262 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ngenix.net/llms.txt", "observed_at": "2026-09-20T06:25:07.379466+00:00", "status": 503, "final_url": "https://ngenix.net/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "70ea8b1d227b8e3e1d8542864b57af31467cff9ec70fb33a8dfdad07de42a0b0", "bytes_read": 7262 }, "llms_state": "unavailable", "control": null }, { "rank": 523, "domain": "exp-tas.com", "robots": { "requested_url": "https://exp-tas.com/robots.txt", "observed_at": "2026-09-20T06:25:06.596326+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://exp-tas.com/llms.txt", "observed_at": "2026-09-20T06:25:06.667842+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 524, "domain": "ipv4only.arpa", "robots": { "requested_url": "https://ipv4only.arpa/robots.txt", "observed_at": "2026-09-20T06:25:06.669602+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ipv4only.arpa/llms.txt", "observed_at": "2026-09-20T06:25:26.741376+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 525, "domain": "whecloud.com", "robots": { "requested_url": "https://whecloud.com/robots.txt", "observed_at": "2026-09-20T06:25:07.674914+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://whecloud.com/llms.txt", "observed_at": "2026-09-20T06:25:07.739872+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 526, "domain": "tplinkcloud.com", "robots": { "requested_url": "https://tplinkcloud.com/robots.txt", "observed_at": "2026-09-20T06:25:07.741791+00:00", "status": 200, "final_url": "https://www.tplinkcloud.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "c10c202bc280a4b8b3b9eec0205c1117cbe97cdc4e01d4efeb9002e7eda96c3d", "bytes_read": 70 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://tplinkcloud.com/llms.txt", "observed_at": "2026-09-20T06:25:08.557921+00:00", "status": 404, "final_url": "https://www.tplinkcloud.com/llms.txt", "content_type": "text/html; charset=iso-8859-1", "truncated": false, "body_sha256": "80c3fe2ae1062abf56456f52518bd670f9ec3917b7f85e152b347ac6b6faf880", "bytes_read": 196 }, "llms_state": "not_detected", "control": null }, { "rank": 527, "domain": "elasticbeanstalk.com", "robots": { "requested_url": "https://elasticbeanstalk.com/robots.txt", "observed_at": "2026-09-20T06:25:07.989386+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://elasticbeanstalk.com/llms.txt", "observed_at": "2026-09-20T06:25:08.406972+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 528, "domain": "adobedtm.com", "robots": { "requested_url": "https://adobedtm.com/robots.txt", "observed_at": "2026-09-20T06:25:08.054794+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://adobedtm.com/llms.txt", "observed_at": "2026-09-20T06:25:08.189900+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 529, "domain": "discord.media", "robots": { "requested_url": "https://discord.media/robots.txt", "observed_at": "2026-09-20T06:25:08.191881+00:00", "status": 520, "final_url": "https://discord.media/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "b59e493586b4118c563aaa751e04a7f7bc1c9741d241e0c1d3ba9405c323c570", "bytes_read": 16 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://discord.media/llms.txt", "observed_at": "2026-09-20T06:25:08.516657+00:00", "status": 520, "final_url": "https://discord.media/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "b59e493586b4118c563aaa751e04a7f7bc1c9741d241e0c1d3ba9405c323c570", "bytes_read": 16 }, "llms_state": "unavailable", "control": null }, { "rank": 530, "domain": "icloud-content.com", "robots": { "requested_url": "https://icloud-content.com/robots.txt", "observed_at": "2026-09-20T06:25:08.696637+00:00", "status": 200, "final_url": "https://developer.apple.com/icloud/", "content_type": "text/html", "truncated": false, "body_sha256": "007fcc165f464b2e27742690d9567b079d3c680e3359e342a5d005e058bc9059", "bytes_read": 109848 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://icloud-content.com/llms.txt", "observed_at": "2026-09-20T06:25:09.831028+00:00", "status": 200, "final_url": "https://developer.apple.com/icloud/", "content_type": "text/html", "truncated": false, "body_sha256": "007fcc165f464b2e27742690d9567b079d3c680e3359e342a5d005e058bc9059", "bytes_read": 109848 }, "llms_state": "not_detected", "control": null }, { "rank": 531, "domain": "slideshare.net", "robots": { "requested_url": "https://slideshare.net/robots.txt", "observed_at": "2026-09-20T06:25:08.737023+00:00", "status": 200, "final_url": "https://www.slideshare.net/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "28a89a844d552847c3d257ecc7226bcedcbafa8dcaa7ecb3860604a0e083e34e", "bytes_read": 1443 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://slideshare.net/llms.txt", "observed_at": "2026-09-20T06:25:09.301807+00:00", "status": 200, "final_url": "https://www.slideshare.net/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "8376ffb30f785e88b34efbeb814812833d8851476bbd59b4674af24c24962090", "bytes_read": 371446 }, "llms_state": "not_detected", "control": null }, { "rank": 532, "domain": "trendmicro.com", "robots": { "requested_url": "https://trendmicro.com/robots.txt", "observed_at": "2026-09-20T06:25:09.257764+00:00", "status": 200, "final_url": "https://www.trendmicro.com/robots.txt", "content_type": "text/plain;charset=utf-8", "truncated": false, "body_sha256": "c1043957d079629a6e2996b7cdcbfbe2ef9cf090d6ca1f3e4525cde83f67f23f", "bytes_read": 2211 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [] } }, "llms": { "requested_url": "https://trendmicro.com/llms.txt", "observed_at": "2026-09-20T06:25:10.903772+00:00", "status": 200, "final_url": "https://www.trendmicro.com/llms.txt", "content_type": "text/plain;charset=utf-8", "truncated": false, "body_sha256": "fe6cd2ca2c85a14f98be527c052126a42c113014ee66cf216a6c1151d0486bac", "bytes_read": 22999 }, "llms_state": "candidate", "control": { "requested_url": "https://trendmicro.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:12.135245+00:00", "status": 404, "final_url": "https://www.trendmicro.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "c50e63f4d8a3c96bd1c73c4091636215f4c66e5d0e59a710cb64bf392af51976", "bytes_read": 73100 } }, { "rank": 533, "domain": "ft.com", "robots": { "requested_url": "https://ft.com/robots.txt", "observed_at": "2026-09-20T06:25:09.477223+00:00", "status": 200, "final_url": "https://www.ft.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "57ce1e58ae57b17699c48037deda0f549296f5c0bfe44e92cf7af6deb6767b4b", "bytes_read": 1556 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://ft.com/llms.txt", "observed_at": "2026-09-20T06:25:10.221022+00:00", "status": 404, "final_url": "https://www.ft.com/llms.txt", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "not_detected", "control": null }, { "rank": 534, "domain": "noaa.gov", "robots": { "requested_url": "https://noaa.gov/robots.txt", "observed_at": "2026-09-20T06:25:10.089798+00:00", "status": 403, "final_url": "https://www.noaa.gov/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "d28fdb2765927442a4998ec8865fc0fcd9b4c90ded05ff51ecb945a3c8e2081a", "bytes_read": 919 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://noaa.gov/llms.txt", "observed_at": "2026-09-20T06:25:11.155018+00:00", "status": 403, "final_url": "https://www.noaa.gov/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "fa9e9ddd9f75372522e4b02db7133361fa8445a1224c4368028a12d284f63a4f", "bytes_read": 919 }, "llms_state": "unavailable", "control": null }, { "rank": 535, "domain": "capcut.com", "robots": { "requested_url": "https://capcut.com/robots.txt", "observed_at": "2026-09-20T06:25:10.608993+00:00", "status": 200, "final_url": "https://www.capcut.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "0164812d10b8d6dcf43c60a42dbbd519332dbe1a95b0b551282f90c574e57ab4", "bytes_read": 1630 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://capcut.com/llms.txt", "observed_at": "2026-09-20T06:25:11.750317+00:00", "status": 200, "final_url": "https://www.capcut.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "c3a7161f16bc231d4f0fa4973adb227a64bc47916e7259572c722c2b86f45145", "bytes_read": 8429 }, "llms_state": "unknown_control", "control": { "requested_url": "https://capcut.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:12.217765+00:00", "status": 502, "final_url": "https://www.capcut.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "243748a253d19197de9e9f3de0895591932ec638655af0de6f0d066213376795", "bytes_read": 252 } }, { "rank": 536, "domain": "share.google", "robots": { "requested_url": "https://share.google/robots.txt", "observed_at": "2026-09-20T06:25:10.975183+00:00", "status": 200, "final_url": "https://share.google/error", "content_type": "text/html", "truncated": false, "body_sha256": "7d64b26c603ea0250a03f6be2dd87d8243f928ebd7dfe770e3f57ecb6222af39", "bytes_read": 1675 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://share.google/llms.txt", "observed_at": "2026-09-20T06:25:11.707392+00:00", "status": 200, "final_url": "https://share.google/error", "content_type": "text/html", "truncated": false, "body_sha256": "6ba50332d487248e96d42d156b76b783b0faf295ef1e575aef6833f5e6931610", "bytes_read": 1675 }, "llms_state": "not_detected", "control": null }, { "rank": 537, "domain": "shop.app", "robots": { "requested_url": "https://shop.app/robots.txt", "observed_at": "2026-09-20T06:25:11.321080+00:00", "status": 200, "final_url": "https://shop.app/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "7da0d40ada4ed5fe57f49271fe1bd545955d0b5b99963b1f9b8837d8697a5f5c", "bytes_read": 2770 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://shop.app/llms.txt", "observed_at": "2026-09-20T06:25:11.745244+00:00", "status": 200, "final_url": "https://shop.app/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "72f1b680f8416562cacb7b42e7f6e65045d051e14d4fec44048e2e365b56a8c5", "bytes_read": 3833 }, "llms_state": "candidate", "control": { "requested_url": "https://shop.app/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:11.931353+00:00", "status": 404, "final_url": "https://shop.app/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "5c04091b318da12161ceeaa8ddd4490f8a69e05e5726e266d489fc582b4853af", "bytes_read": 379560 } }, { "rank": 538, "domain": "fandom.com", "robots": { "requested_url": "https://fandom.com/robots.txt", "observed_at": "2026-09-20T06:25:11.759454+00:00", "status": 403, "final_url": "https://www.fandom.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "fb2774a257e4a80cce03e63db5f96e89b68a544fc32065cb745b8f9fbe302433", "bytes_read": 5541 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://fandom.com/llms.txt", "observed_at": "2026-09-20T06:25:12.383899+00:00", "status": 403, "final_url": "https://www.fandom.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "caefbd6a091c3f84918e389e8bf00cc469e094da9254c6b693f0f898a8260eca", "bytes_read": 5535 }, "llms_state": "unavailable", "control": null }, { "rank": 539, "domain": "ack.de", "robots": { "requested_url": "https://ack.de/robots.txt", "observed_at": "2026-09-20T06:25:12.300825+00:00", "status": 200, "final_url": "https://www.ack.de/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "16ceb5ee3e0dc13aa9adf31a3ebbe45a1d965b8c2b9f72eaf84e5911e140ed95", "bytes_read": 23 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://ack.de/llms.txt", "observed_at": "2026-09-20T06:25:14.004625+00:00", "status": 200, "final_url": "https://www.ack.de/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "93bc76d06151a645c1a71a4d88821f065d3672fc8574b3beebd66f6a538cb0a9", "bytes_read": 3663 }, "llms_state": "not_detected", "control": null }, { "rank": 540, "domain": "on.aws", "robots": { "requested_url": "https://on.aws/robots.txt", "observed_at": "2026-09-20T06:25:12.672282+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://on.aws/llms.txt", "observed_at": "2026-09-20T06:25:12.842526+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 541, "domain": "visualstudio.com", "robots": { "requested_url": "https://visualstudio.com/robots.txt", "observed_at": "2026-09-20T06:25:12.729367+00:00", "status": 200, "final_url": "https://visualstudio.microsoft.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e8402cbb9b3542be7aa69c0b6560b95f4585d4e75033a1b083dbb3ff59ee4f4c", "bytes_read": 234 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://visualstudio.com/llms.txt", "observed_at": "2026-09-20T06:25:13.938669+00:00", "status": 404, "final_url": "https://visualstudio.microsoft.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "d8531a1ac331414d357ac014ac8813b93c2c0ea1756542ad8bacb6d352ed4891", "bytes_read": 60 }, "llms_state": "not_detected", "control": null }, { "rank": 542, "domain": "google.com.hk", "robots": { "requested_url": "https://google.com.hk/robots.txt", "observed_at": "2026-09-20T06:25:12.748835+00:00", "status": 200, "final_url": "https://www.google.com.hk/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1aba570a1b32deb66941e0c29d4f37be71ee58aa1a624aa20754b3639538cf2c", "bytes_read": 6624 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://google.com.hk/llms.txt", "observed_at": "2026-09-20T06:25:13.340522+00:00", "status": 404, "final_url": "https://google.com.hk/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 543, "domain": "oup.com", "robots": { "requested_url": "https://oup.com/robots.txt", "observed_at": "2026-09-20T06:25:12.844401+00:00", "status": 202, "final_url": "http://global.oup.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://oup.com/llms.txt", "observed_at": "2026-09-20T06:25:13.879823+00:00", "status": 202, "final_url": "http://global.oup.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "unavailable", "control": null }, { "rank": 544, "domain": "openx.net", "robots": { "requested_url": "https://openx.net/robots.txt", "observed_at": "2026-09-20T06:25:13.213513+00:00", "status": 404, "final_url": "https://openx.net/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5", "bytes_read": 9 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://openx.net/llms.txt", "observed_at": "2026-09-20T06:25:13.448072+00:00", "status": 404, "final_url": "https://openx.net/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5", "bytes_read": 9 }, "llms_state": "not_detected", "control": null }, { "rank": 545, "domain": "squarespace.com", "robots": { "requested_url": "https://squarespace.com/robots.txt", "observed_at": "2026-09-20T06:25:13.350855+00:00", "status": 200, "final_url": "https://www.squarespace.com/robots.txt", "content_type": "text/plain;charset=utf-8", "truncated": false, "body_sha256": "2280dba517d015842f2e97870bf9c17952ac25fec56b9685637501e839d8ca3b", "bytes_read": 2379 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [] } }, "llms": { "requested_url": "https://squarespace.com/llms.txt", "observed_at": "2026-09-20T06:25:14.061752+00:00", "status": 200, "final_url": "https://www.squarespace.com/assets/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "3c5cafaee640f3ff85af72b34294f5fb4bf3fbc75bfed6f37635c8cca99e9199", "bytes_read": 2088 }, "llms_state": "candidate", "control": { "requested_url": "https://squarespace.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:14.809668+00:00", "status": 404, "final_url": "https://www.squarespace.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "f1040405413addb6532e0930b5bd3ae4ae4ba38b86e8fd40b70eb603ccfa02d0", "bytes_read": 351834 } }, { "rank": 546, "domain": "firefox.com", "robots": { "requested_url": "https://firefox.com/robots.txt", "observed_at": "2026-09-20T06:25:13.585046+00:00", "status": 200, "final_url": "https://www.firefox.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "edac9086299b02b0022f72001e619ef8c1b53dc5e3d900e1a24dcccfb76a2054", "bytes_read": 78 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://firefox.com/llms.txt", "observed_at": "2026-09-20T06:25:14.357185+00:00", "status": 404, "final_url": "https://www.firefox.com/en-US/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "f08a704f1e2499e520d937ed140696b1ff10762cc93be9c4945e9fb11875a74a", "bytes_read": 8996 }, "llms_state": "not_detected", "control": null }, { "rank": 547, "domain": "ui-dns.com", "robots": { "requested_url": "https://ui-dns.com/robots.txt", "observed_at": "2026-09-20T06:25:13.589685+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ui-dns.com/llms.txt", "observed_at": "2026-09-20T06:25:13.726019+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 548, "domain": "adobedc.net", "robots": { "requested_url": "https://adobedc.net/robots.txt", "observed_at": "2026-09-20T06:25:13.728371+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://adobedc.net/llms.txt", "observed_at": "2026-09-20T06:25:13.831430+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 549, "domain": "jquery.com", "robots": { "requested_url": "https://jquery.com/robots.txt", "observed_at": "2026-09-20T06:25:13.833152+00:00", "status": 200, "final_url": "https://jquery.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "f827412c8b340e376171b45b40b44d8f9c9cc0b4c1020c0087048b8499f99566", "bytes_read": 111 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://jquery.com/llms.txt", "observed_at": "2026-09-20T06:25:14.204877+00:00", "status": 404, "final_url": "https://jquery.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "27d5151ef70de128e7e5ff7af85087f84939ae166df41bb80e331470afb395df", "bytes_read": 18865 }, "llms_state": "not_detected", "control": null }, { "rank": 550, "domain": "azure-dns.com", "robots": { "requested_url": "https://azure-dns.com/robots.txt", "observed_at": "2026-09-20T06:25:14.001078+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://azure-dns.com/llms.txt", "observed_at": "2026-09-20T06:25:14.100513+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 551, "domain": "fast.com", "robots": { "requested_url": "https://fast.com/robots.txt", "observed_at": "2026-09-20T06:25:14.102709+00:00", "status": 200, "final_url": "https://fast.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "0d2cc73bb9311486c8eaa9f3a595d2215feb0ee3ea585ffb890ab05b04b1e114", "bytes_read": 41 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://fast.com/llms.txt", "observed_at": "2026-09-20T06:25:14.419973+00:00", "status": 200, "final_url": "https://fast.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "b6471f8ddc8d98c81679220c67a7971b2fbc42d35e09ef8c9ad3f464eaac305e", "bytes_read": 4006 }, "llms_state": "not_detected", "control": null }, { "rank": 552, "domain": "bitrix24.ru", "robots": { "requested_url": "https://bitrix24.ru/robots.txt", "observed_at": "2026-09-20T06:25:14.403022+00:00", "status": 200, "final_url": "https://www.bitrix24.ru/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e1e88b45bfd15cdccc3aeeb0d24c8428b296e8b8bf572bb41c0221ab1aa206ed", "bytes_read": 3594 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://bitrix24.ru/llms.txt", "observed_at": "2026-09-20T06:25:15.597645+00:00", "status": 200, "final_url": "https://www.bitrix24.ru/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "c913e4808a92e65072bed826c030de132be366e862c072d4d7844591f70b73e5", "bytes_read": 11378 }, "llms_state": "candidate", "control": { "requested_url": "https://bitrix24.ru/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:16.707524+00:00", "status": 404, "final_url": "https://www.bitrix24.ru/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 } }, { "rank": 553, "domain": "wired.com", "robots": { "requested_url": "https://wired.com/robots.txt", "observed_at": "2026-09-20T06:25:14.505456+00:00", "status": 200, "final_url": "https://www.wired.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "70018eb70e50517cff0d65ef6f68839955205e722ada5d64993fc9a9e8401daa", "bytes_read": 2224 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://wired.com/llms.txt", "observed_at": "2026-09-20T06:25:15.044065+00:00", "status": 404, "final_url": "https://www.wired.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "2d1cdf10a2e36aee526d7c2af76dd0513478ba8d04fc14719b70650b43190d6a", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 554, "domain": "netflix.net", "robots": { "requested_url": "https://netflix.net/robots.txt", "observed_at": "2026-09-20T06:25:14.543241+00:00", "status": 200, "final_url": "https://www.netflix.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1e14fa0284e9757b52b48dde38b390effae68afdd72447088e72f18f49b1aa09", "bytes_read": 3790 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://netflix.net/llms.txt", "observed_at": "2026-09-20T06:25:15.410534+00:00", "status": 200, "final_url": "https://www.netflix.com/NotFound?prev=https%3A%2F%2Fwww.netflix.com%2Fllms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "37f33987e9b4df3aa05072db191b854cf432217d8d638a1b0fc27ca9b3e3ff5c", "bytes_read": 23136 }, "llms_state": "not_detected", "control": null }, { "rank": 555, "domain": "squarespacedns.com", "robots": { "requested_url": "https://squarespacedns.com/robots.txt", "observed_at": "2026-09-20T06:25:14.831822+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://squarespacedns.com/llms.txt", "observed_at": "2026-09-20T06:25:14.915859+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 556, "domain": "myhuaweicloud.com", "robots": { "requested_url": "https://myhuaweicloud.com/robots.txt", "observed_at": "2026-09-20T06:25:14.917419+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://myhuaweicloud.com/llms.txt", "observed_at": "2026-09-20T06:25:15.065142+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 557, "domain": "vidaahub.com", "robots": { "requested_url": "https://vidaahub.com/robots.txt", "observed_at": "2026-09-20T06:25:14.941199+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://vidaahub.com/llms.txt", "observed_at": "2026-09-20T06:25:15.033684+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 558, "domain": "spo-msedge.net", "robots": { "requested_url": "https://spo-msedge.net/robots.txt", "observed_at": "2026-09-20T06:25:15.037208+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://spo-msedge.net/llms.txt", "observed_at": "2026-09-20T06:25:15.125482+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 559, "domain": "tandfonline.com", "robots": { "requested_url": "https://tandfonline.com/robots.txt", "observed_at": "2026-09-20T06:25:15.067372+00:00", "status": 200, "final_url": "https://www.tandfonline.com/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "70adb5768df9af8c11135d5e8b723a5cede06f805c3d255e7e09018386927ef8", "bytes_read": 1379 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://tandfonline.com/llms.txt", "observed_at": "2026-09-20T06:25:15.997085+00:00", "status": 403, "final_url": "https://tandfonline.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "9c51edecd9f9ef3ae20128fb9f3e5f053223129e2193b3b414e86dce9f38f252", "bytes_read": 5579 }, "llms_state": "unavailable", "control": null }, { "rank": 560, "domain": "blogger.com", "robots": { "requested_url": "https://blogger.com/robots.txt", "observed_at": "2026-09-20T06:25:15.127653+00:00", "status": 200, "final_url": "https://www.blogger.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "2076a51a5ea6b870209770053655a5e2499bddd4ae6a6a40b5453b9898168a34", "bytes_read": 572 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://blogger.com/llms.txt", "observed_at": "2026-09-20T06:25:15.725925+00:00", "status": 404, "final_url": "https://www.blogger.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "103bd81325370dd77244d0f6e2d92e8eb4852627d6a63deb7bf05a840409210a", "bytes_read": 168 }, "llms_state": "not_detected", "control": null }, { "rank": 561, "domain": "grammarly.com", "robots": { "requested_url": "https://grammarly.com/robots.txt", "observed_at": "2026-09-20T06:25:15.156634+00:00", "status": 200, "final_url": "https://www.grammarly.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "cce8a2723a7fde148f1d9e0840dde392d0d615654dba2b1c3fd0694d0c075973", "bytes_read": 289 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://grammarly.com/llms.txt", "observed_at": "2026-09-20T06:25:16.316581+00:00", "status": 404, "final_url": "https://www.grammarly.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6641c7016184892156f2a7825656113f8db0985d39f8d6f76fef540742d251d0", "bytes_read": 450566 }, "llms_state": "not_detected", "control": null }, { "rank": 562, "domain": "palmplaystore.com", "robots": { "requested_url": "https://palmplaystore.com/robots.txt", "observed_at": "2026-09-20T06:25:15.304722+00:00", "status": 404, "final_url": "https://palmplaystore.com/robots.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "166eb5da894c8712dd2ec08e89d305732a978828aff19ba7d4f3f6028d53ee8a", "bytes_read": 377326 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://palmplaystore.com/llms.txt", "observed_at": "2026-09-20T06:25:17.170055+00:00", "status": 404, "final_url": "https://palmplaystore.com/llms.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "f03d67b5dddd67b6ab4b923e723e4c5cc2da91ee053eb07ee243b2b0184028cc", "bytes_read": 377312 }, "llms_state": "not_detected", "control": null }, { "rank": 563, "domain": "nstld.com", "robots": { "requested_url": "https://nstld.com/robots.txt", "observed_at": "2026-09-20T06:25:16.155411+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://nstld.com/llms.txt", "observed_at": "2026-09-20T06:25:16.264866+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 564, "domain": "uol.com.br", "robots": { "requested_url": "https://uol.com.br/robots.txt", "observed_at": "2026-09-20T06:25:16.211367+00:00", "status": 200, "final_url": "https://www.uol.com.br/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "4f4948a9777c3d4f68c329ba5be470f43a3a89a0eb8349ad63809891b0c31ed1", "bytes_read": 2515 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://uol.com.br/llms.txt", "observed_at": "2026-09-20T06:25:17.709027+00:00", "status": 404, "final_url": "https://www.uol.com.br/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "35313bc3db2ecfda4fd4b15c47236f78d762da3ca3b19e4b6202f20bae347856", "bytes_read": 506 }, "llms_state": "not_detected", "control": null }, { "rank": 565, "domain": "oraclecloud.com", "robots": { "requested_url": "https://oraclecloud.com/robots.txt", "observed_at": "2026-09-20T06:25:16.266537+00:00", "status": 403, "final_url": "https://www.oracle.com/cloud/", "content_type": "text/html", "truncated": false, "body_sha256": "a1ab36cfcec9632b7c4777eaf8639148d44a1125dbd27353c6b01be3b1d923b3", "bytes_read": 1450 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://oraclecloud.com/llms.txt", "observed_at": "2026-09-20T06:25:17.661808+00:00", "status": 403, "final_url": "https://www.oracle.com/cloud/", "content_type": "text/html", "truncated": false, "body_sha256": "1c8e5116a44a02b2d00679f9f66b3739df441565f456cf37a3af47de16826988", "bytes_read": 1450 }, "llms_state": "unavailable", "control": null }, { "rank": 566, "domain": "live.net", "robots": { "requested_url": "https://live.net/robots.txt", "observed_at": "2026-09-20T06:25:16.282531+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://live.net/llms.txt", "observed_at": "2026-09-20T06:25:16.415663+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 567, "domain": "360yield.com", "robots": { "requested_url": "https://360yield.com/robots.txt", "observed_at": "2026-09-20T06:25:16.356174+00:00", "status": 200, "final_url": "https://ssp.improvedigital.com:443/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "7c55f46de40de31de6cd1ddebe7663ace5183933647b3600c2a328c135299b13", "bytes_read": 106 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://360yield.com/llms.txt", "observed_at": "2026-09-20T06:25:17.522210+00:00", "status": 404, "final_url": "https://ssp.improvedigital.com:443/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "d78204a118a20c28469cec553cd2ee4d6194b0125f7087bd094b87d83d59a675", "bytes_read": 1189 }, "llms_state": "not_detected", "control": null }, { "rank": 568, "domain": "svc.ms", "robots": { "requested_url": "https://svc.ms/robots.txt", "observed_at": "2026-09-20T06:25:16.422760+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://svc.ms/llms.txt", "observed_at": "2026-09-20T06:25:16.495773+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 569, "domain": "eset.com", "robots": { "requested_url": "https://eset.com/robots.txt", "observed_at": "2026-09-20T06:25:16.513356+00:00", "status": 200, "final_url": "https://www.eset.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "e4c41ec26d6623f527880b7a04a6477bfd58a487f554380bdb62861f3b85f5e4", "bytes_read": 6699 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://eset.com/llms.txt", "observed_at": "2026-09-20T06:25:17.880482+00:00", "status": 404, "final_url": "https://www.eset.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "84184d2d5ef006ec7f48a0e7751d96ff76854c967a23c8d582c336aa08e6ea28", "bytes_read": 17418 }, "llms_state": "not_detected", "control": null }, { "rank": 570, "domain": "bitdefender.net", "robots": { "requested_url": "https://bitdefender.net/robots.txt", "observed_at": "2026-09-20T06:25:16.532996+00:00", "status": 200, "final_url": "https://www.bitdefender.com/en-us/", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "e27ce569542fa25eeee99cf34394c96786be7797b9e1ee483966232675a3a433", "bytes_read": 250728 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://bitdefender.net/llms.txt", "observed_at": "2026-09-20T06:25:17.749764+00:00", "status": 200, "final_url": "https://www.bitdefender.com/en-us/", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "e27ce569542fa25eeee99cf34394c96786be7797b9e1ee483966232675a3a433", "bytes_read": 250728 }, "llms_state": "not_detected", "control": null }, { "rank": 571, "domain": "digitalocean.com", "robots": { "requested_url": "https://digitalocean.com/robots.txt", "observed_at": "2026-09-20T06:25:16.747029+00:00", "status": 200, "final_url": "https://www.digitalocean.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "fa88d03d74f7d9a29da6780431c0eda495aad6e9d16eb080d7b07d966aad28a8", "bytes_read": 172 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://digitalocean.com/llms.txt", "observed_at": "2026-09-20T06:25:17.294617+00:00", "status": 404, "final_url": "https://www.digitalocean.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "e8315bcc2a7019eae488b8d071bbf7b092a439f07b254067ec5aea92275a981a", "bytes_read": 137019 }, "llms_state": "not_detected", "control": null }, { "rank": 572, "domain": "time.com", "robots": { "requested_url": "https://time.com/robots.txt", "observed_at": "2026-09-20T06:25:16.951727+00:00", "status": 200, "final_url": "https://time.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "74c4c40ead0ed6f1c7a4e05bdd07da92609c73d255818af1ecbc1b527147fc41", "bytes_read": 2326 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://time.com/llms.txt", "observed_at": "2026-09-20T06:25:17.350510+00:00", "status": 200, "final_url": "https://time.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "33b2b869b9498b20f104002731adc96644345ce5f8d4cd4823c4e9f54499231f", "bytes_read": 716 }, "llms_state": "not_detected", "control": null }, { "rank": 573, "domain": "rakuten.co.jp", "robots": { "requested_url": "https://rakuten.co.jp/robots.txt", "observed_at": "2026-09-20T06:25:17.469556+00:00", "status": 200, "final_url": "https://www.rakuten.co.jp/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "10c0ca813e13339bc5ef5c67077a04c41cfb21dd44bff629644d2e03e9b760f4", "bytes_read": 490 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://rakuten.co.jp/llms.txt", "observed_at": "2026-09-20T06:25:23.939566+00:00", "status": 404, "final_url": "https://www.rakuten.co.jp/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "892c2ad1ccc44e97d7a2059caa79badea1228f09d805ba288e802a2ec37abca9", "bytes_read": 3412 }, "llms_state": "not_detected", "control": null }, { "rank": 574, "domain": "hicloud.com", "robots": { "requested_url": "https://hicloud.com/robots.txt", "observed_at": "2026-09-20T06:25:17.539073+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://hicloud.com/llms.txt", "observed_at": "2026-09-20T06:25:17.672467+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 575, "domain": "qlivecdn.com", "robots": { "requested_url": "https://qlivecdn.com/robots.txt", "observed_at": "2026-09-20T06:25:17.674122+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://qlivecdn.com/llms.txt", "observed_at": "2026-09-20T06:25:17.734496+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 576, "domain": "deviantart.com", "robots": { "requested_url": "https://deviantart.com/robots.txt", "observed_at": "2026-09-20T06:25:17.736053+00:00", "status": 200, "final_url": "https://www.deviantart.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "c9d8f9d47084f57742b25a21c4607aff7b47377c81e22dea1345fb9239c7fc60", "bytes_read": 284 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://deviantart.com/llms.txt", "observed_at": "2026-09-20T06:25:18.337350+00:00", "status": 404, "final_url": "https://www.deviantart.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "a58f8876e57090c679c0385bf02f198f571e5c19da034bbe76baadf275028cec", "bytes_read": 27911 }, "llms_state": "not_detected", "control": null }, { "rank": 577, "domain": "cornell.edu", "robots": { "requested_url": "https://cornell.edu/robots.txt", "observed_at": "2026-09-20T06:25:17.750160+00:00", "status": 200, "final_url": "https://www.cornell.edu/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "baada4216bf732573f67f021a8ee420f1f8b53d202d0fa546913556ea39298ee", "bytes_read": 1046 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://cornell.edu/llms.txt", "observed_at": "2026-09-20T06:25:18.760120+00:00", "status": 404, "final_url": "https://www.cornell.edu/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "9012a2aafda9d2b6ae9b282b32f128d5d0f0d44934170f7af7830b382c25b353", "bytes_read": 40794 }, "llms_state": "not_detected", "control": null }, { "rank": 578, "domain": "t-msedge.net", "robots": { "requested_url": "https://t-msedge.net/robots.txt", "observed_at": "2026-09-20T06:25:18.289012+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://t-msedge.net/llms.txt", "observed_at": "2026-09-20T06:25:18.369638+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 579, "domain": "yahoodns.net", "robots": { "requested_url": "https://yahoodns.net/robots.txt", "observed_at": "2026-09-20T06:25:18.332691+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://yahoodns.net/llms.txt", "observed_at": "2026-09-20T06:25:18.438115+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 580, "domain": "aboutads.info", "robots": { "requested_url": "https://aboutads.info/robots.txt", "observed_at": "2026-09-20T06:25:18.371557+00:00", "status": 200, "final_url": "https://youradchoices.com/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "4bf0164562d16329014b90aefe98c4b5b20cc45cde7238099276519a91f28453", "bytes_read": 34950 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://aboutads.info/llms.txt", "observed_at": "2026-09-20T06:25:19.086349+00:00", "status": 200, "final_url": "https://youradchoices.com/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "15ae0679987cd7ff4c8416a5d64d6e829bd61e7e484339e17f0ccbc5832257e6", "bytes_read": 34950 }, "llms_state": "not_detected", "control": null }, { "rank": 581, "domain": "translate.goog", "robots": { "requested_url": "https://translate.goog/robots.txt", "observed_at": "2026-09-20T06:25:18.432542+00:00", "status": 404, "final_url": "https://translate.goog/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2", "bytes_read": 1571 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://translate.goog/llms.txt", "observed_at": "2026-09-20T06:25:18.764884+00:00", "status": 404, "final_url": "https://translate.goog/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 582, "domain": "cookielaw.org", "robots": { "requested_url": "https://cookielaw.org/robots.txt", "observed_at": "2026-09-20T06:25:18.439831+00:00", "status": 421, "final_url": "https://www.cookielaw.org/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "d00a56e2585497b2173949a9320bdf26a745666bd3cfeb314d12bcd91363161c", "bytes_read": 291 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cookielaw.org/llms.txt", "observed_at": "2026-09-20T06:25:18.836355+00:00", "status": 421, "final_url": "https://www.cookielaw.org/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "d00a56e2585497b2173949a9320bdf26a745666bd3cfeb314d12bcd91363161c", "bytes_read": 291 }, "llms_state": "unavailable", "control": null }, { "rank": 583, "domain": "sberbank.ru", "robots": { "requested_url": "https://sberbank.ru/robots.txt", "observed_at": "2026-09-20T06:25:18.501535+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://sberbank.ru/llms.txt", "observed_at": "2026-09-20T06:25:19.139287+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 584, "domain": "teamviewer.com", "robots": { "requested_url": "https://teamviewer.com/robots.txt", "observed_at": "2026-09-20T06:25:18.651442+00:00", "status": 200, "final_url": "https://www.teamviewer.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "160bad0ace888f14d455b916cd79472986eb1a34515d8fba0f51d44754eab264", "bytes_read": 581 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://teamviewer.com/llms.txt", "observed_at": "2026-09-20T06:25:19.394328+00:00", "status": 404, "final_url": "https://www.teamviewer.com/llms.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "4c4cd058e9890fef20fd17e994a4106218950a710609b8c0d4bc90b5d913565e", "bytes_read": 245264 }, "llms_state": "not_detected", "control": null }, { "rank": 585, "domain": "usatoday.com", "robots": { "requested_url": "https://usatoday.com/robots.txt", "observed_at": "2026-09-20T06:25:18.685669+00:00", "status": 200, "final_url": "https://www.usatoday.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "a86ba97de592ff071921cb4f274b69bace2fb7ab068c2e1a3a82e29dcc077657", "bytes_read": 16130 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://usatoday.com/llms.txt", "observed_at": "2026-09-20T06:25:19.229310+00:00", "status": 404, "final_url": "https://www.usatoday.com/llms.txt/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "95a89bb816b83be475134cb3780829882e2d1a3b826c74724d1463a2afd5a78d", "bytes_read": 134393 }, "llms_state": "not_detected", "control": null }, { "rank": 586, "domain": "free.fr", "robots": { "requested_url": "https://free.fr/robots.txt", "observed_at": "2026-09-20T06:25:18.770020+00:00", "status": 200, "final_url": "https://www.free.fr/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "f2c4b777426cd62d3f57e0bb39fe5943051f3f8599facd163e94c133031c1b10", "bytes_read": 1051 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://free.fr/llms.txt", "observed_at": "2026-09-20T06:25:19.901820+00:00", "status": 404, "final_url": "https://www.free.fr/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "770e6df6d9161a398c5773e3b1dc62f2894a4029a8c03a9ff827e46d1d1926e5", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 587, "domain": "target.com", "robots": { "requested_url": "https://target.com/robots.txt", "observed_at": "2026-09-20T06:25:18.801399+00:00", "status": 200, "final_url": "https://www.target.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "3f8a9e45a59d43ab1b0e4534ae71241fd9e6cbbd3c37b5426bc9efd6eb320bd7", "bytes_read": 3226 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://target.com/llms.txt", "observed_at": "2026-09-20T06:25:19.292164+00:00", "status": 200, "final_url": "https://www.target.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "401b6c28fa673c7993916e8a6ce4ac33e67aba61e27b8f0cd67ecb7b09d02c4d", "bytes_read": 6158 }, "llms_state": "candidate", "control": { "requested_url": "https://target.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:19.762820+00:00", "status": 404, "final_url": "https://www.target.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "db98bd60478472c6d0c8504a14804577f5bb427a89f01918174fe367ab3520d1", "bytes_read": 2176 } }, { "rank": 588, "domain": "vercel-dns-3.com", "robots": { "requested_url": "https://vercel-dns-3.com/robots.txt", "observed_at": "2026-09-20T06:25:19.016036+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://vercel-dns-3.com/llms.txt", "observed_at": "2026-09-20T06:25:19.097748+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 589, "domain": "scdn.co", "robots": { "requested_url": "https://scdn.co/robots.txt", "observed_at": "2026-09-20T06:25:19.069857+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://scdn.co/llms.txt", "observed_at": "2026-09-20T06:25:19.129556+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 590, "domain": "my.com", "robots": { "requested_url": "https://my.com/robots.txt", "observed_at": "2026-09-20T06:25:19.099336+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://my.com/llms.txt", "observed_at": "2026-09-20T06:25:19.312382+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 591, "domain": "tripadvisor.com", "robots": { "requested_url": "https://tripadvisor.com/robots.txt", "observed_at": "2026-09-20T06:25:19.131633+00:00", "status": 200, "final_url": "https://www.tripadvisor.com/robots.txt", "content_type": "text/plain;charset=utf-8", "truncated": false, "body_sha256": "2472be0731cf9ff1ff23e59b1159c667e8cfe73c6756af50bcc8e2681fc3c164", "bytes_read": 24267 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://tripadvisor.com/llms.txt", "observed_at": "2026-09-20T06:25:19.899683+00:00", "status": 403, "final_url": "https://www.tripadvisor.com/llms.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "1896494bc966a3a733abcfa67c95b9ebaea1e7e59c4083f888e2a97a06c42c1f", "bytes_read": 775 }, "llms_state": "unavailable", "control": null }, { "rank": 592, "domain": "3lift.com", "robots": { "requested_url": "https://3lift.com/robots.txt", "observed_at": "2026-09-20T06:25:19.313869+00:00", "status": 403, "final_url": "https://3lift.com/robots.txt", "content_type": "application/xml", "truncated": false, "body_sha256": "a824bc7739e226e1b40ea0f8c4e4f4c6f796fc3b4abfa6e9abe3bd119a30d938", "bytes_read": 111 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://3lift.com/llms.txt", "observed_at": "2026-09-20T06:25:19.591761+00:00", "status": 403, "final_url": "https://3lift.com/llms.txt", "content_type": "application/xml", "truncated": false, "body_sha256": "a824bc7739e226e1b40ea0f8c4e4f4c6f796fc3b4abfa6e9abe3bd119a30d938", "bytes_read": 111 }, "llms_state": "unavailable", "control": null }, { "rank": 593, "domain": "rlcdn.com", "robots": { "requested_url": "https://rlcdn.com/robots.txt", "observed_at": "2026-09-20T06:25:19.495440+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://rlcdn.com/llms.txt", "observed_at": "2026-09-20T06:25:19.578174+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 594, "domain": "klaviyo.com", "robots": { "requested_url": "https://klaviyo.com/robots.txt", "observed_at": "2026-09-20T06:25:19.548688+00:00", "status": 200, "final_url": "https://www.klaviyo.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "76016404bc4d6253b273d66f681383765e88ae1207e54621e1e546dc27af4669", "bytes_read": 3530 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [] } }, "llms": { "requested_url": "https://klaviyo.com/llms.txt", "observed_at": "2026-09-20T06:25:20.131425+00:00", "status": 200, "final_url": "https://www.klaviyo.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "c600de018aeb48a7ca1339a4f64b58b0df7dd37a6fa09872254b47fb8c23c0df", "bytes_read": 20277 }, "llms_state": "candidate", "control": { "requested_url": "https://klaviyo.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:20.650774+00:00", "status": 404, "final_url": "https://www.klaviyo.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "137b08e090c53eecb25ca4e5b9f699f45fc24c5dfe45626c99c8a838fe7db8d6", "bytes_read": 3185 } }, { "rank": 595, "domain": "hotjar.com", "robots": { "requested_url": "https://hotjar.com/robots.txt", "observed_at": "2026-09-20T06:25:19.565489+00:00", "status": 200, "final_url": "https://www.hotjar.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "95f1088bd49ac64b279552705fd2d444e047bc186efa58bbfb7ff7a8c1b0ee91", "bytes_read": 510 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://hotjar.com/llms.txt", "observed_at": "2026-09-20T06:25:20.289464+00:00", "status": 404, "final_url": "https://www.hotjar.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "8ab6c6b71d3710b7f2279189588e499d9da7e65c4b1bf87e72229ae9e3d624c0", "bytes_read": 146316 }, "llms_state": "not_detected", "control": null }, { "rank": 596, "domain": "dreamhost.com", "robots": { "requested_url": "https://dreamhost.com/robots.txt", "observed_at": "2026-09-20T06:25:19.580343+00:00", "status": 200, "final_url": "https://www.dreamhost.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "d165d1aaaffc86a5424982e2d1eead4b58d3055bee0a23873b078be3bcd75fc6", "bytes_read": 480 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://dreamhost.com/llms.txt", "observed_at": "2026-09-20T06:25:20.427119+00:00", "status": 200, "final_url": "https://www.dreamhost.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "f27b43fab010abfd3c1d31d5a51d67b954c9b0cbf1dedb8895c38d2c59ebdeed", "bytes_read": 18678 }, "llms_state": "candidate", "control": { "requested_url": "https://dreamhost.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:20.865230+00:00", "status": 200, "final_url": "https://www.dreamhost.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "bd064d56bdd0569628b28c1d857ac315f994558ff3bca0640b6137c0c0e23637", "bytes_read": 479858 } }, { "rank": 597, "domain": "imgsmail.ru", "robots": { "requested_url": "https://imgsmail.ru/robots.txt", "observed_at": "2026-09-20T06:25:19.788836+00:00", "status": 200, "final_url": "https://imgsmail.ru/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "3cd731c8f8d1b1d5a27b95cb79a002ae85a58d750b15109d05d493b98309a213", "bytes_read": 49 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://imgsmail.ru/llms.txt", "observed_at": "2026-09-20T06:25:20.734277+00:00", "status": 200, "final_url": "https://account.mail.ru/login?autologin_exp=v22&autologin_project=home&errorCode=11300&errorText=invalid+user&page=https%3A%2F%2Fmail.ru%2F%3Fautologin%3D1789885522&state=ba01b2c215a448dd85924c3798cf5473", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "f612f9acc9492d6d924fb784deac1979db719e911d8bb73d772123b1ddab159d", "bytes_read": 936 }, "llms_state": "not_detected", "control": null }, { "rank": 598, "domain": "bx-msedge.net", "robots": { "requested_url": "https://bx-msedge.net/robots.txt", "observed_at": "2026-09-20T06:25:20.102478+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://bx-msedge.net/llms.txt", "observed_at": "2026-09-20T06:25:20.186102+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 599, "domain": "h264.io", "robots": { "requested_url": "https://h264.io/robots.txt", "observed_at": "2026-09-20T06:25:20.117847+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://h264.io/llms.txt", "observed_at": "2026-09-20T06:25:20.177197+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 600, "domain": "ca.gov", "robots": { "requested_url": "https://ca.gov/robots.txt", "observed_at": "2026-09-20T06:25:20.178710+00:00", "status": 200, "final_url": "https://www.ca.gov/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "21b838218e2b07fa3ce867116306776ca3c8c967a6b4d6b04c879a16e58f40ad", "bytes_read": 379 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://ca.gov/llms.txt", "observed_at": "2026-09-20T06:25:21.249285+00:00", "status": 404, "final_url": "https://www.ca.gov/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "3e8fd07395db3ef610f38c23cbd4c0fd8a8905c5efbc130a6348481dae2bb8e5", "bytes_read": 18050 }, "llms_state": "not_detected", "control": null }, { "rank": 601, "domain": "openstreetmap.org", "robots": { "requested_url": "https://openstreetmap.org/robots.txt", "observed_at": "2026-09-20T06:25:20.187752+00:00", "status": 200, "final_url": "https://www.openstreetmap.org/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "8e312e9b5108ed49d10f639928d4650c540482bc35dd8eb526ef0ba0e28a04a4", "bytes_read": 886 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://openstreetmap.org/llms.txt", "observed_at": "2026-09-20T06:25:21.063752+00:00", "status": 404, "final_url": "https://www.openstreetmap.org/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "not_detected", "control": null }, { "rank": 602, "domain": "braze.com", "robots": { "requested_url": "https://braze.com/robots.txt", "observed_at": "2026-09-20T06:25:20.229066+00:00", "status": 200, "final_url": "https://braze.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "763e7163a376658c17b1cb3b7adba437425021cf6a30ac54c2db65dc62ede3d5", "bytes_read": 129 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://braze.com/llms.txt", "observed_at": "2026-09-20T06:25:20.542962+00:00", "status": 200, "final_url": "https://braze.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "d98714355cd73f7621753252592b391665cc5f05a285bc0ed9390a077fca2d17", "bytes_read": 94743 }, "llms_state": "unknown_control", "control": { "requested_url": "https://braze.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:20.902841+00:00", "status": 500, "final_url": "https://braze.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "2d1c022253ad78db54c1f1158f39490d71affddca69aec098ab6fcbea60c4fe6", "bytes_read": 2119 } }, { "rank": 603, "domain": "cdn20.com", "robots": { "requested_url": "https://cdn20.com/robots.txt", "observed_at": "2026-09-20T06:25:20.629137+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cdn20.com/llms.txt", "observed_at": "2026-09-20T06:25:20.716055+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 604, "domain": "avito.ru", "robots": { "requested_url": "https://avito.ru/robots.txt", "observed_at": "2026-09-20T06:25:20.717561+00:00", "status": 200, "final_url": "https://www.avito.ru/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "dea0300f7e9854b10b96ab5af7a5011277f014e4cf31db3f75c85329c7a05045", "bytes_read": 14700 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://avito.ru/llms.txt", "observed_at": "2026-09-20T06:25:21.492387+00:00", "status": 404, "final_url": "https://www.avito.ru/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "69b1c739b1f1c747525cb1645926f6fa5c5b54051efe38dc5287dbc913ec816b", "bytes_read": 29907 }, "llms_state": "not_detected", "control": null }, { "rank": 605, "domain": "playstation.com", "robots": { "requested_url": "https://playstation.com/robots.txt", "observed_at": "2026-09-20T06:25:21.147130+00:00", "status": 200, "final_url": "https://www.playstation.com:443/robots.txt", "content_type": "text/plain;charset=iso-8859-1", "truncated": false, "body_sha256": "3f3efcb84aa347ed7aef117d19fed670cc05ed97261d536edaac8719600f8e68", "bytes_read": 655 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://playstation.com/llms.txt", "observed_at": "2026-09-20T06:25:22.018909+00:00", "status": 200, "final_url": "https://www.playstation.com:443/llms.txt", "content_type": "text/plain; charset=ISO-8859-1", "truncated": false, "body_sha256": "390a1dd80e52a35f35fe50b3160c181775ed58ebd84dad96785b81c9177debdb", "bytes_read": 3842 }, "llms_state": "not_detected", "control": null }, { "rank": 606, "domain": "xiaomi.net", "robots": { "requested_url": "https://xiaomi.net/robots.txt", "observed_at": "2026-09-20T06:25:21.205981+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://xiaomi.net/llms.txt", "observed_at": "2026-09-20T06:25:21.842149+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 607, "domain": "ieee.org", "robots": { "requested_url": "https://ieee.org/robots.txt", "observed_at": "2026-09-20T06:25:21.309162+00:00", "status": 202, "final_url": "https://www.ieee.org/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ieee.org/llms.txt", "observed_at": "2026-09-20T06:25:21.876848+00:00", "status": 202, "final_url": "https://www.ieee.org/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "unavailable", "control": null }, { "rank": 608, "domain": "berkeley.edu", "robots": { "requested_url": "https://berkeley.edu/robots.txt", "observed_at": "2026-09-20T06:25:21.333937+00:00", "status": 200, "final_url": "https://www.berkeley.edu/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "43a81a7e209a162a8532b6ab0e3d887b108fe914c366e2396673cfd21b715bd1", "bytes_read": 190 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://berkeley.edu/llms.txt", "observed_at": "2026-09-20T06:25:22.049609+00:00", "status": 404, "final_url": "https://www.berkeley.edu/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "d8531a1ac331414d357ac014ac8813b93c2c0ea1756542ad8bacb6d352ed4891", "bytes_read": 60 }, "llms_state": "not_detected", "control": null }, { "rank": 609, "domain": "force.com", "robots": { "requested_url": "https://force.com/robots.txt", "observed_at": "2026-09-20T06:25:21.525000+00:00", "status": 403, "final_url": "https://www.force.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "d332782824c59f8bdc2e972878fcc7ae117f5edc718d99837aa69f554d4b0359", "bytes_read": 280 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://force.com/llms.txt", "observed_at": "2026-09-20T06:25:22.806296+00:00", "status": 403, "final_url": "https://www.force.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "fa3b73d549d4f94d8bc2a650e322d6532c30ed9306b4f1590fd47ed0e8cc3c9e", "bytes_read": 280 }, "llms_state": "unavailable", "control": null }, { "rank": 610, "domain": "mynetname.net", "robots": { "requested_url": "https://mynetname.net/robots.txt", "observed_at": "2026-09-20T06:25:21.614804+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://mynetname.net/llms.txt", "observed_at": "2026-09-20T06:25:21.799477+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 611, "domain": "anthropic.com", "robots": { "requested_url": "https://anthropic.com/robots.txt", "observed_at": "2026-09-20T06:25:21.690296+00:00", "status": 200, "final_url": "https://www.anthropic.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7", "bytes_read": 71 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://anthropic.com/llms.txt", "observed_at": "2026-09-20T06:25:22.205194+00:00", "status": 404, "final_url": "https://www.anthropic.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "99681282412b6801e97234b0bae72149396123085d3ba5bad01c38092ff35fcd", "bytes_read": 59884 }, "llms_state": "not_detected", "control": null }, { "rank": 612, "domain": "easebar.com", "robots": { "requested_url": "https://easebar.com/robots.txt", "observed_at": "2026-09-20T06:25:21.801033+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://easebar.com/llms.txt", "observed_at": "2026-09-20T06:25:21.863373+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 613, "domain": "wswebcdn.com", "robots": { "requested_url": "https://wswebcdn.com/robots.txt", "observed_at": "2026-09-20T06:25:21.865009+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://wswebcdn.com/llms.txt", "observed_at": "2026-09-20T06:25:21.925613+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 614, "domain": "footprintdns.com", "robots": { "requested_url": "https://footprintdns.com/robots.txt", "observed_at": "2026-09-20T06:25:21.927542+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://footprintdns.com/llms.txt", "observed_at": "2026-09-20T06:25:21.998845+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 615, "domain": "threads.com", "robots": { "requested_url": "https://threads.com/robots.txt", "observed_at": "2026-09-20T06:25:22.001218+00:00", "status": 200, "final_url": "https://www.threads.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "c9fff5b1a746ac235e1fef8f17baf68a736f8df81fdc277e8d196f3555703209", "bytes_read": 8149 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://threads.com/llms.txt", "observed_at": "2026-09-20T06:25:22.727585+00:00", "status": 200, "final_url": "https://www.threads.com/login/?next=https%3A%2F%2Fwww.threads.com%2Fllms.txt%2F", "content_type": "text/html; charset=\"utf-8\"", "truncated": false, "body_sha256": "c7bafd511f6ffad020edd5fc03a12a4f7eb5385e67e3315c53b2b3d7dcb61e1d", "bytes_read": 318500 }, "llms_state": "not_detected", "control": null }, { "rank": 616, "domain": "cdnvideo.ru", "robots": { "requested_url": "https://cdnvideo.ru/robots.txt", "observed_at": "2026-09-20T06:25:22.086648+00:00", "status": 200, "final_url": "https://cdnvideo.ru/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "6b7b423840217701f60398a7b2777b335b7b6a59f9292c34c1b7781cd191d6a3", "bytes_read": 5119 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://cdnvideo.ru/llms.txt", "observed_at": "2026-09-20T06:25:23.238635+00:00", "status": 404, "final_url": "https://cdnvideo.ru/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 }, "llms_state": "not_detected", "control": null }, { "rank": 617, "domain": "2gis.com", "robots": { "requested_url": "https://2gis.com/robots.txt", "observed_at": "2026-09-20T06:25:22.213046+00:00", "status": 200, "final_url": "https://2gis.ae/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "1edc20e35d86d2c19e48fb4a979a53d6e638c8a7a4df6f8e9fc4935fff77e1ec", "bytes_read": 1193 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://2gis.com/llms.txt", "observed_at": "2026-09-20T06:25:23.665131+00:00", "status": 404, "final_url": "https://2gis.ae/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "616d9bde18fb9268277d8536e0b81306f97180943c0f7f15c3cba8db28046cdb", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 618, "domain": "mdpi.com", "robots": { "requested_url": "https://mdpi.com/robots.txt", "observed_at": "2026-09-20T06:25:22.302688+00:00", "status": 403, "final_url": "https://mdpi.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "6b5f4e262301513e776f4ff6a8e136a82e97a60bc2001788621a0050367d58e6", "bytes_read": 372 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://mdpi.com/llms.txt", "observed_at": "2026-09-20T06:25:22.649649+00:00", "status": 403, "final_url": "https://mdpi.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "d7462fcf7b457ec29974985c82c05d2dc3bb2e860a44a25bdd052d33d0c0c3f3", "bytes_read": 368 }, "llms_state": "unavailable", "control": null }, { "rank": 619, "domain": "telegraph.co.uk", "robots": { "requested_url": "https://telegraph.co.uk/robots.txt", "observed_at": "2026-09-20T06:25:22.396821+00:00", "status": 200, "final_url": "https://www.telegraph.co.uk/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "9e246848905e8b4886bef649bcb4797e79bde8497a6d8545e925ab60b17f04d7", "bytes_read": 5210 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://telegraph.co.uk/llms.txt", "observed_at": "2026-09-20T06:25:23.338045+00:00", "status": 404, "final_url": "https://www.telegraph.co.uk/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "65db1171ca5974102653cef3cfdb5a3a121786d0d948c97deb03df37cc97b04b", "bytes_read": 146236 }, "llms_state": "not_detected", "control": null }, { "rank": 620, "domain": "pushy.io", "robots": { "requested_url": "https://pushy.io/robots.txt", "observed_at": "2026-09-20T06:25:22.538125+00:00", "status": 404, "final_url": "https://pushy.me/robots.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "8b1308074e798c4a2f5cbe4ef0507e80944d89a807ae29831213ea0008c472a5", "bytes_read": 11964 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://pushy.io/llms.txt", "observed_at": "2026-09-20T06:25:23.256159+00:00", "status": 404, "final_url": "https://pushy.me/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "8b1308074e798c4a2f5cbe4ef0507e80944d89a807ae29831213ea0008c472a5", "bytes_read": 11964 }, "llms_state": "not_detected", "control": null }, { "rank": 621, "domain": "onetrust.com", "robots": { "requested_url": "https://onetrust.com/robots.txt", "observed_at": "2026-09-20T06:25:22.905892+00:00", "status": 200, "final_url": "https://www.onetrust.com/robots.txt", "content_type": "text/plain; charset=ISO-8859-1", "truncated": false, "body_sha256": "8e19810fd4bacc2ff6b26179f16e1d6b910d55b490ae31ff19651738b755f790", "bytes_read": 396 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://onetrust.com/llms.txt", "observed_at": "2026-09-20T06:25:23.382469+00:00", "status": 200, "final_url": "https://www.onetrust.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "43d5772fdfe10d28e4c387b541211f9a62108cc124413fb0eb1a421aff30dec2", "bytes_read": 6859 }, "llms_state": "candidate", "control": { "requested_url": "https://onetrust.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:23.666562+00:00", "status": 404, "final_url": "https://www.onetrust.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "cab93f390237437c624e9792e10270c739281986be4b76f3cf37f948edc16bda", "bytes_read": 131798 } }, { "rank": 622, "domain": "atlassian.com", "robots": { "requested_url": "https://atlassian.com/robots.txt", "observed_at": "2026-09-20T06:25:22.942383+00:00", "status": 200, "final_url": "https://www.atlassian.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "c0a6d137d17b75f9f916b616a77e87fde588b0adb4338127b03b55ea77ba3841", "bytes_read": 1292 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://atlassian.com/llms.txt", "observed_at": "2026-09-20T06:25:23.321833+00:00", "status": 200, "final_url": "https://www.atlassian.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "cd8609813ba417ea3040411a22551a91b7663f552724332c97758649bba82ca2", "bytes_read": 26607 }, "llms_state": "candidate", "control": { "requested_url": "https://atlassian.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:23.638892+00:00", "status": 404, "final_url": "https://www.atlassian.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "40913137ead25f254e0a1bff6794cf37475fba83f3073287b76b7ba2b9a603e6", "bytes_read": 144594 } }, { "rank": 623, "domain": "nintendo.com", "robots": { "requested_url": "https://nintendo.com/robots.txt", "observed_at": "2026-09-20T06:25:23.054601+00:00", "status": 200, "final_url": "https://www.nintendo.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "9f7c4501d6487be15b78c3e5bb84f2355b95705def720c1d43d17f0fc92d9ccc", "bytes_read": 5596 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://nintendo.com/llms.txt", "observed_at": "2026-09-20T06:25:23.984247+00:00", "status": 404, "final_url": "https://www.nintendo.com/us/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "47cc44efdff3cc72da458a3de7289e7abddfef119a71804fd67561455ad2bde5", "bytes_read": 217819 }, "llms_state": "not_detected", "control": null }, { "rank": 624, "domain": "samsungapps.com", "robots": { "requested_url": "https://samsungapps.com/robots.txt", "observed_at": "2026-09-20T06:25:23.515728+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://samsungapps.com/llms.txt", "observed_at": "2026-09-20T06:25:33.782811+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 625, "domain": "browser-intake-datadoghq.com", "robots": { "requested_url": "https://browser-intake-datadoghq.com/robots.txt", "observed_at": "2026-09-20T06:25:23.560069+00:00", "status": 403, "final_url": "https://browser-intake-datadoghq.com/robots.txt", "content_type": "application/json", "truncated": false, "body_sha256": "069a75c303f05e0bb3ec6eefd594b832d47bd4c54d13b5937c74915c355b8c02", "bytes_read": 56 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://browser-intake-datadoghq.com/llms.txt", "observed_at": "2026-09-20T06:25:23.798382+00:00", "status": 403, "final_url": "https://browser-intake-datadoghq.com/llms.txt", "content_type": "application/json", "truncated": false, "body_sha256": "069a75c303f05e0bb3ec6eefd594b832d47bd4c54d13b5937c74915c355b8c02", "bytes_read": 56 }, "llms_state": "unavailable", "control": null }, { "rank": 626, "domain": "amazon.fr", "robots": { "requested_url": "https://amazon.fr/robots.txt", "observed_at": "2026-09-20T06:25:23.711040+00:00", "status": 200, "final_url": "https://www.amazon.fr/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "7bc69859053e614be026bfb89f05ebf9700177b43a25b33fad361df726a5a2d1", "bytes_read": 7161 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://amazon.fr/llms.txt", "observed_at": "2026-09-20T06:25:24.982626+00:00", "status": 404, "final_url": "https://www.amazon.fr/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "3fb1feae258a4a1f4501e4ec03c1d01a1cd71d332e53e0577015923758acbb9b", "bytes_read": 2344 }, "llms_state": "not_detected", "control": null }, { "rank": 627, "domain": "e2ro.com", "robots": { "requested_url": "https://e2ro.com/robots.txt", "observed_at": "2026-09-20T06:25:23.711861+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://e2ro.com/llms.txt", "observed_at": "2026-09-20T06:25:23.794659+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 628, "domain": "alicdn.com", "robots": { "requested_url": "https://alicdn.com/robots.txt", "observed_at": "2026-09-20T06:25:23.796768+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://alicdn.com/llms.txt", "observed_at": "2026-09-20T06:25:43.878779+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 629, "domain": "accuweather.com", "robots": { "requested_url": "https://accuweather.com/robots.txt", "observed_at": "2026-09-20T06:25:23.964010+00:00", "status": 403, "final_url": "https://accuweather.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "53f035b4626e1faa4e5fa4b9a73738ce005d647d5076f3a0f136ad985ed7a4a8", "bytes_read": 379 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://accuweather.com/llms.txt", "observed_at": "2026-09-20T06:25:24.222991+00:00", "status": 403, "final_url": "https://accuweather.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "02fe204264f0a28efaba5c7f412a0160fb046cf2494b956a11d418bb830014d2", "bytes_read": 377 }, "llms_state": "unavailable", "control": null }, { "rank": 630, "domain": "supertms.com", "robots": { "requested_url": "https://supertms.com/robots.txt", "observed_at": "2026-09-20T06:25:24.048594+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://supertms.com/llms.txt", "observed_at": "2026-09-20T06:25:24.382974+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 631, "domain": "wix.com", "robots": { "requested_url": "https://wix.com/robots.txt", "observed_at": "2026-09-20T06:25:24.079082+00:00", "status": 200, "final_url": "https://www.wix.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "fef1f6ed42a555f0eb1efa45b48d3d00c98d8eb0452bb6d2a1487d6ba8771d6a", "bytes_read": 2797 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://wix.com/llms.txt", "observed_at": "2026-09-20T06:25:24.743849+00:00", "status": 200, "final_url": "https://www.wix.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "ccfa8dc4eb331c658b95d69579dcb70aff2785b51af56ec94fd63e4455c3d529", "bytes_read": 36670 }, "llms_state": "candidate", "control": { "requested_url": "https://wix.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:25.219597+00:00", "status": 404, "final_url": "https://www.wix.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "4ee9c5c5d996e031ef3d67ff44b02df5d32d36e2c46ac94de7a9e27e5dffe090", "bytes_read": 2505 } }, { "rank": 632, "domain": "nike.com", "robots": { "requested_url": "https://nike.com/robots.txt", "observed_at": "2026-09-20T06:25:24.094233+00:00", "status": 200, "final_url": "https://www.nike.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "54ba534bff9e8b4cf89f30c9432c8aa4b8e2b652cb1a9170b25ac8d8ad356ee6", "bytes_read": 3503 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://nike.com/llms.txt", "observed_at": "2026-09-20T06:25:24.848980+00:00", "status": 404, "final_url": "https://www.nike.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": true, "body_sha256": "03a37bbfdf4926a13b9a10ec36851dfb67794e00aa23ca17e3bf42381c3cc694", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 633, "domain": "unesco.org", "robots": { "requested_url": "https://unesco.org/robots.txt", "observed_at": "2026-09-20T06:25:24.384913+00:00", "status": 200, "final_url": "https://www.unesco.org/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "cd7568b0aac29cfb545e9de672017954512c9bce5f96b314fb9a93023096faf7", "bytes_read": 67029 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://unesco.org/llms.txt", "observed_at": "2026-09-20T06:25:26.942902+00:00", "status": 200, "final_url": "https://www.unesco.org/llms.txt", "content_type": "text/markdown; charset=utf-8", "truncated": false, "body_sha256": "bd842f88c64d9c77aa58edc75980c00b4f7e2a015d61b6f4796b0c24cc14e757", "bytes_read": 9896 }, "llms_state": "unknown_control", "control": { "requested_url": "https://unesco.org/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:28.008447+00:00", "status": 404, "final_url": "https://www.unesco.org/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=UTF-8", "truncated": true, "body_sha256": "d75f1c87fb29d6765d0e99bf567d8060c73a75f55d7241f27124aba04929dd72", "bytes_read": 512001 } }, { "rank": 634, "domain": "mega.co.nz", "robots": { "requested_url": "https://mega.co.nz/robots.txt", "observed_at": "2026-09-20T06:25:24.399071+00:00", "status": 200, "final_url": "https://mega.co.nz/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://mega.co.nz/llms.txt", "observed_at": "2026-09-20T06:25:24.913206+00:00", "status": 404, "final_url": "https://mega.co.nz/llms.txt", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "not_detected", "control": null }, { "rank": 635, "domain": "doctolib.fr", "robots": { "requested_url": "https://doctolib.fr/robots.txt", "observed_at": "2026-09-20T06:25:24.447506+00:00", "status": 200, "final_url": "https://www.doctolib.fr/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "a8133a03d78e4f32b1839d702eb04421efc6a69e2712634623c6f69de0d61bf2", "bytes_read": 1626 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/*$" ], [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://doctolib.fr/llms.txt", "observed_at": "2026-09-20T06:25:25.749118+00:00", "status": 404, "final_url": "https://www.doctolib.fr/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "not_detected", "control": null }, { "rank": 636, "domain": "ikea.com", "robots": { "requested_url": "https://ikea.com/robots.txt", "observed_at": "2026-09-20T06:25:24.944658+00:00", "status": 200, "final_url": "https://www.ikea.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "53df86f20aea5f91125e8ed0d4ae2c69aaf94fd671a276b41354a5a62b4a9338", "bytes_read": 4054 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://ikea.com/llms.txt", "observed_at": "2026-09-20T06:25:25.444615+00:00", "status": 404, "final_url": "https://www.ikea.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5", "bytes_read": 9 }, "llms_state": "not_detected", "control": null }, { "rank": 637, "domain": "comcast.com", "robots": { "requested_url": "https://comcast.com/robots.txt", "observed_at": "2026-09-20T06:25:25.348694+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://comcast.com/llms.txt", "observed_at": "2026-09-20T06:25:35.415505+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 638, "domain": "sharethrough.com", "robots": { "requested_url": "https://sharethrough.com/robots.txt", "observed_at": "2026-09-20T06:25:25.750844+00:00", "status": 200, "final_url": "https://www.sharethrough.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "ef6a308607832bd2b5d09c7f44e54a7b952ef9089e9ca31f6c04bc64f7501fd6", "bytes_read": 1198 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://sharethrough.com/llms.txt", "observed_at": "2026-09-20T06:25:26.193098+00:00", "status": 404, "final_url": "https://www.sharethrough.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "6f1ae19b9cd827e8c7921991dd747c39c022a4ccd73e3714ca679b5aea2def5f", "bytes_read": 51505 }, "llms_state": "not_detected", "control": null }, { "rank": 639, "domain": "dyndns.org", "robots": { "requested_url": "https://dyndns.org/robots.txt", "observed_at": "2026-09-20T06:25:25.785985+00:00", "status": 200, "final_url": "https://account.dyn.com/", "content_type": "text/html; charset=ISO-8859-1", "truncated": false, "body_sha256": "5ba556cd39123213ae40efa7dc1b9d572d92484076363be8140779b9c808495b", "bytes_read": 28440 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://dyndns.org/llms.txt", "observed_at": "2026-09-20T06:25:26.770091+00:00", "status": 200, "final_url": "https://account.dyn.com/", "content_type": "text/html; charset=ISO-8859-1", "truncated": false, "body_sha256": "f23694c823a76f72b83c1db95203387a7363efbf8686e021d0f8a149874bef51", "bytes_read": 28440 }, "llms_state": "not_detected", "control": null }, { "rank": 640, "domain": "branch.io", "robots": { "requested_url": "https://branch.io/robots.txt", "observed_at": "2026-09-20T06:25:25.845039+00:00", "status": 200, "final_url": "https://www.branch.io/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "6d82fbc52f19b980ae2791440108e456e66cb8f862ae28f8e55d796225ca4a25", "bytes_read": 1783 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://branch.io/llms.txt", "observed_at": "2026-09-20T06:25:26.475509+00:00", "status": 200, "final_url": "https://www.branch.io/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "a70debbd9d149427ddaba488c7eb9a64f6ec9fdb4e46ab35346135eae213e23c", "bytes_read": 6608 }, "llms_state": "candidate", "control": { "requested_url": "https://branch.io/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:27.068815+00:00", "status": 404, "final_url": "https://www.branch.io/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 } }, { "rank": 641, "domain": "vercel-dns-017.com", "robots": { "requested_url": "https://vercel-dns-017.com/robots.txt", "observed_at": "2026-09-20T06:25:25.925356+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://vercel-dns-017.com/llms.txt", "observed_at": "2026-09-20T06:25:26.008806+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 642, "domain": "msidentity.com", "robots": { "requested_url": "https://msidentity.com/robots.txt", "observed_at": "2026-09-20T06:25:26.002609+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://msidentity.com/llms.txt", "observed_at": "2026-09-20T06:25:26.063073+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 643, "domain": "bytedns1.com", "robots": { "requested_url": "https://bytedns1.com/robots.txt", "observed_at": "2026-09-20T06:25:26.010350+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://bytedns1.com/llms.txt", "observed_at": "2026-09-20T06:25:26.141898+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 644, "domain": "tencent.com", "robots": { "requested_url": "https://tencent.com/robots.txt", "observed_at": "2026-09-20T06:25:26.066254+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://tencent.com/llms.txt", "observed_at": "2026-09-20T06:25:29.242822+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 645, "domain": "patreon.com", "robots": { "requested_url": "https://patreon.com/robots.txt", "observed_at": "2026-09-20T06:25:26.143875+00:00", "status": 200, "final_url": "https://patreon.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "2e16dd7f2df38af17199751642ca0a7405e1fa69010813c1b54e46b2777be8ae", "bytes_read": 759 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://patreon.com/llms.txt", "observed_at": "2026-09-20T06:25:26.505876+00:00", "status": 404, "final_url": "https://www.patreon.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "5065c20412df1b3602d4d2cc665330a6c5caae2c7b6e85bf1ede624d5414e09d", "bytes_read": 348672 }, "llms_state": "not_detected", "control": null }, { "rank": 646, "domain": "xhamster.com", "robots": { "requested_url": "https://xhamster.com/robots.txt", "observed_at": "2026-09-20T06:25:26.711061+00:00", "status": 200, "final_url": "https://xhamster.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "8ebd6b73326801117db8965aec8abfaefcf2e9b369c72bda940485a9701be0c1", "bytes_read": 4504 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://xhamster.com/llms.txt", "observed_at": "2026-09-20T06:25:27.048713+00:00", "status": 404, "final_url": "https://xhamster.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 }, "llms_state": "not_detected", "control": null }, { "rank": 647, "domain": "ovh.net", "robots": { "requested_url": "https://ovh.net/robots.txt", "observed_at": "2026-09-20T06:25:26.859244+00:00", "status": null, "error": "RemoteDisconnected" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ovh.net/llms.txt", "observed_at": "2026-09-20T06:25:28.096393+00:00", "status": null, "error": "RemoteDisconnected" }, "llms_state": "unavailable", "control": null }, { "rank": 648, "domain": "jotform.com", "robots": { "requested_url": "https://jotform.com/robots.txt", "observed_at": "2026-09-20T06:25:27.290175+00:00", "status": 200, "final_url": "https://www.jotform.com/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "4daf73dfd61da0de6150504190910928716bd0c074122e8b7a19c035332e56d5", "bytes_read": 12165 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [] } }, "llms": { "requested_url": "https://jotform.com/llms.txt", "observed_at": "2026-09-20T06:25:27.970081+00:00", "status": 404, "final_url": "https://www.jotform.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "ee56e9cd18a61a13473ff171de482385e6d4d280bdb758a004519ecf5ff45ac2", "bytes_read": 174475 }, "llms_state": "not_detected", "control": null }, { "rank": 649, "domain": "withgoogle.com", "robots": { "requested_url": "https://withgoogle.com/robots.txt", "observed_at": "2026-09-20T06:25:27.382058+00:00", "status": 200, "final_url": "https://www.google.com/", "content_type": "text/html; charset=ISO-8859-1", "truncated": false, "body_sha256": "d7ccd1cb431a3a38ae23478bfe80e01efbe288ad43fc70fa383d69b61cf9ad4d", "bytes_read": 84140 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://withgoogle.com/llms.txt", "observed_at": "2026-09-20T06:25:27.854473+00:00", "status": 200, "final_url": "https://www.google.com/", "content_type": "text/html; charset=ISO-8859-1", "truncated": false, "body_sha256": "d8038b51b085876971072114faaf964cc9b48acc5232c24b9db611169f068a29", "bytes_read": 83954 }, "llms_state": "not_detected", "control": null }, { "rank": 650, "domain": "loc.gov", "robots": { "requested_url": "https://loc.gov/robots.txt", "observed_at": "2026-09-20T06:25:27.623551+00:00", "status": 200, "final_url": "https://www.loc.gov/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "994f1660f526307cafb387d72d060c4691731bc3dea614961dc3e0cc2a5c9121", "bytes_read": 482 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://loc.gov/llms.txt", "observed_at": "2026-09-20T06:25:28.069865+00:00", "status": 404, "final_url": "https://loc.gov/llms.txt", "content_type": "text/html; charset=iso-8859-1", "truncated": false, "body_sha256": "0da006306ca67b9d88367bc1a63929b073ed36fb046b9b61695f934bfa544b89", "bytes_read": 206 }, "llms_state": "not_detected", "control": null }, { "rank": 651, "domain": "amazon.ca", "robots": { "requested_url": "https://amazon.ca/robots.txt", "observed_at": "2026-09-20T06:25:27.623931+00:00", "status": 200, "final_url": "https://www.amazon.ca/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "40667a44c5f7d4329115bd486b14b57cebe8b02062a953b2fca1e1de5a5358fe", "bytes_read": 6179 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://amazon.ca/llms.txt", "observed_at": "2026-09-20T06:25:28.262418+00:00", "status": 503, "final_url": "https://www.amazon.ca/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "5f9a1da86835f13b37ed9acb3510143cefb303787947bb4858f16d26c7c7502d", "bytes_read": 1888 }, "llms_state": "unavailable", "control": null }, { "rank": 652, "domain": "shopee.com.br", "robots": { "requested_url": "https://shopee.com.br/robots.txt", "observed_at": "2026-09-20T06:25:28.218871+00:00", "status": 200, "final_url": "https://shopee.com.br/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "c4368b5d7004635a1ab2ab4e0f08b898ec289cb90ae67b155b75f9a0aac0f432", "bytes_read": 2782 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://shopee.com.br/llms.txt", "observed_at": "2026-09-20T06:25:28.493633+00:00", "status": 200, "final_url": "https://shopee.com.br/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "0f8c9543140300194ae211d7522a13821ebfc52364619374cafe326d166a5fcd", "bytes_read": 198343 }, "llms_state": "not_detected", "control": null }, { "rank": 653, "domain": "vercel.app", "robots": { "requested_url": "https://vercel.app/robots.txt", "observed_at": "2026-09-20T06:25:28.394454+00:00", "status": 200, "final_url": "https://vercel.com/", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "3525103e107ebe71323705a3301746d9951f82026ea642fd396d943f6decd5c8", "bytes_read": 512001 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://vercel.app/llms.txt", "observed_at": "2026-09-20T06:25:28.925925+00:00", "status": 200, "final_url": "https://vercel.com/", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "3525103e107ebe71323705a3301746d9951f82026ea642fd396d943f6decd5c8", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 654, "domain": "mhverifier.ru", "robots": { "requested_url": "https://mhverifier.ru/robots.txt", "observed_at": "2026-09-20T06:25:28.609489+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://mhverifier.ru/llms.txt", "observed_at": "2026-09-20T06:25:29.212751+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 655, "domain": "techcrunch.com", "robots": { "requested_url": "https://techcrunch.com/robots.txt", "observed_at": "2026-09-20T06:25:28.720148+00:00", "status": 200, "final_url": "https://techcrunch.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "5fbefc8fdb1ded6f6e24100293afdf08e45ac81a21a51b7a65b27e484c6a03e9", "bytes_read": 710 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://techcrunch.com/llms.txt", "observed_at": "2026-09-20T06:25:29.711584+00:00", "status": 404, "final_url": "https://techcrunch.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "07489a8b683221b880d865f6de6ffe387ee4039767ad204df8b07b843e58ac7b", "bytes_read": 207010 }, "llms_state": "not_detected", "control": null }, { "rank": 656, "domain": "ups.com", "robots": { "requested_url": "https://ups.com/robots.txt", "observed_at": "2026-09-20T06:25:28.868261+00:00", "status": 200, "final_url": "https://www.ups.com/robots.txt", "content_type": "application/blank", "truncated": false, "body_sha256": "a4a34a30b19faf1967bdeec65b29a3555b4e1ff62dd051c55567c4df50480173", "bytes_read": 188 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ups.com/llms.txt", "observed_at": "2026-09-20T06:25:29.625894+00:00", "status": 200, "final_url": "https://www.ups.com/llms.txt", "content_type": "application/blank", "truncated": false, "body_sha256": "fc529e07645dc19585b4ba52a38308792e159649ba950e7995db42b56a54437e", "bytes_read": 186 }, "llms_state": "not_detected", "control": null }, { "rank": 657, "domain": "googletagservices.com", "robots": { "requested_url": "https://googletagservices.com/robots.txt", "observed_at": "2026-09-20T06:25:28.948271+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://googletagservices.com/llms.txt", "observed_at": "2026-09-20T06:25:29.057069+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 658, "domain": "sagepub.com", "robots": { "requested_url": "https://sagepub.com/robots.txt", "observed_at": "2026-09-20T06:25:29.059183+00:00", "status": 200, "final_url": "https://www.sagepub.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "75fd5d5deb5e5a70de8d5b00633eefbdfef6ae1a5c0ee3bfb44b0454a33386b1", "bytes_read": 5472 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://sagepub.com/llms.txt", "observed_at": "2026-09-20T06:25:29.578399+00:00", "status": 200, "final_url": "https://www.sagepub.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "c96a72101a85771fb767b39a8905dce42092ef5da9913d35469c8e7f8577fe82", "bytes_read": 21767 }, "llms_state": "not_detected", "control": null }, { "rank": 659, "domain": "bidswitch.net", "robots": { "requested_url": "https://bidswitch.net/robots.txt", "observed_at": "2026-09-20T06:25:29.479663+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://bidswitch.net/llms.txt", "observed_at": "2026-09-20T06:25:29.523181+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 660, "domain": "allegro.pl", "robots": { "requested_url": "https://allegro.pl/robots.txt", "observed_at": "2026-09-20T06:25:29.524934+00:00", "status": 200, "final_url": "https://allegro.pl/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "eeaf2458ba37bcabd8ddb3a552c9ff7fc5e899f15fffa3a0d00c7a9cb5d2b1dd", "bytes_read": 1118 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://allegro.pl/llms.txt", "observed_at": "2026-09-20T06:25:30.127180+00:00", "status": 200, "final_url": "https://allegro.pl/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "9826c1142ce7b069d9df4071676d4cb0dab56c68dfdf34e0628f3ce93b3e07cb", "bytes_read": 12680 }, "llms_state": "unknown_control", "control": { "requested_url": "https://allegro.pl/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:30.707508+00:00", "status": 404, "final_url": "https://allegro.pl/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "6cf566bd71a859e90e76cc60455fdc2f004b798b3aecba58ef85eeda969fded6", "bytes_read": 512001 } }, { "rank": 661, "domain": "intercom.io", "robots": { "requested_url": "https://intercom.io/robots.txt", "observed_at": "2026-09-20T06:25:29.582483+00:00", "status": 200, "final_url": "https://www.intercom.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "513e453566a35a7d2923cc6cad78a8444f25a525765d2212886c5c6785afbf71", "bytes_read": 261 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://intercom.io/llms.txt", "observed_at": "2026-09-20T06:25:30.120230+00:00", "status": 200, "final_url": "https://www.intercom.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "921a2728bd1e92624e6544fa29f93e50929e557cf870ca656063810cc2a9ed67", "bytes_read": 8183 }, "llms_state": "unknown_control", "control": { "requested_url": "https://intercom.io/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:30.969383+00:00", "status": 404, "final_url": "https://www.intercom.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "a888e6d4578195bcd63d5a72a2c12878563c2e805fc59dfee3d3915b05104867", "bytes_read": 512001 } }, { "rank": 662, "domain": "yelp.com", "robots": { "requested_url": "https://yelp.com/robots.txt", "observed_at": "2026-09-20T06:25:29.941725+00:00", "status": 200, "final_url": "https://www.yelp.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "60c2dd139b30dea2ba79575abe4a04ef182ef90fe6ed5491bf07deb7f75893b3", "bytes_read": 12697 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://yelp.com/llms.txt", "observed_at": "2026-09-20T06:25:30.501665+00:00", "status": 404, "final_url": "https://www.yelp.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "b5197c5a9dc0f4cf0cc5a14091cfe31ff560bdfdaeaf6b71d83cba0677f05359", "bytes_read": 5348 }, "llms_state": "not_detected", "control": null }, { "rank": 663, "domain": "redhat.com", "robots": { "requested_url": "https://redhat.com/robots.txt", "observed_at": "2026-09-20T06:25:30.126022+00:00", "status": 200, "final_url": "https://www.redhat.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "d7e51e6de52f13126c29fc08ab1817aca5c4c2fcc8439de05016420dcd984798", "bytes_read": 3337 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://redhat.com/llms.txt", "observed_at": "2026-09-20T06:25:30.764468+00:00", "status": 200, "final_url": "https://www.redhat.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "98be965f653943d2dc844f6465b728228a829584aa64da0a642a72dbdc01f083", "bytes_read": 4366 }, "llms_state": "candidate", "control": { "requested_url": "https://redhat.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:31.053738+00:00", "status": 404, "final_url": "https://www.redhat.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "a13e17e0d883b2788b92577fc3e14f3e62b5962d7f64fffb059999800a7a52c4", "bytes_read": 220 } }, { "rank": 664, "domain": "resolver.arpa", "robots": { "requested_url": "https://resolver.arpa/robots.txt", "observed_at": "2026-09-20T06:25:30.392042+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://resolver.arpa/llms.txt", "observed_at": "2026-09-20T06:25:30.424165+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 665, "domain": "share-dns.com", "robots": { "requested_url": "https://share-dns.com/robots.txt", "observed_at": "2026-09-20T06:25:30.425714+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://share-dns.com/llms.txt", "observed_at": "2026-09-20T06:25:30.507959+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 666, "domain": "telephony.goog", "robots": { "requested_url": "https://telephony.goog/robots.txt", "observed_at": "2026-09-20T06:25:30.510212+00:00", "status": null, "error": "TimeoutError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://telephony.goog/llms.txt", "observed_at": "2026-09-20T06:25:40.912805+00:00", "status": null, "error": "TimeoutError" }, "llms_state": "unavailable", "control": null }, { "rank": 667, "domain": "service.gov.uk", "robots": { "requested_url": "https://service.gov.uk/robots.txt", "observed_at": "2026-09-20T06:25:30.672086+00:00", "status": 200, "final_url": "https://www.gov.uk/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "cf56c10b1bb7c84aa158ac4ee3b177aed73841fe1243d4572fe8545cca1cbc36", "bytes_read": 85520 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://service.gov.uk/llms.txt", "observed_at": "2026-09-20T06:25:31.738278+00:00", "status": 200, "final_url": "https://www.gov.uk/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "cf56c10b1bb7c84aa158ac4ee3b177aed73841fe1243d4572fe8545cca1cbc36", "bytes_read": 85520 }, "llms_state": "not_detected", "control": null }, { "rank": 668, "domain": "stackadapt.com", "robots": { "requested_url": "https://stackadapt.com/robots.txt", "observed_at": "2026-09-20T06:25:30.847764+00:00", "status": 200, "final_url": "https://www.stackadapt.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "6ef77e266863775bafae5e0bb28a53307b06d2fd4e69b94ec6ae635ac8089115", "bytes_read": 623 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://stackadapt.com/llms.txt", "observed_at": "2026-09-20T06:25:31.376062+00:00", "status": 200, "final_url": "https://www.stackadapt.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "62020865d776952146549cbdeaf6dbff79463b74be9cc427815e97989e922eda", "bytes_read": 13804 }, "llms_state": "candidate", "control": { "requested_url": "https://stackadapt.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:31.836081+00:00", "status": 404, "final_url": "https://www.stackadapt.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "303f26bdaaf3e2360b1663e7852eac716a4d0bcfa001d56de0eb266958588aca", "bytes_read": 270960 } }, { "rank": 669, "domain": "britannica.com", "robots": { "requested_url": "https://britannica.com/robots.txt", "observed_at": "2026-09-20T06:25:31.286084+00:00", "status": 200, "final_url": "https://www.britannica.com:443/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "df7371ee298dc17e8fde72ca690850cb7207d91d784b7d0380d89a70ed7b2fcb", "bytes_read": 834 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://britannica.com/llms.txt", "observed_at": "2026-09-20T06:25:31.812264+00:00", "status": 404, "final_url": "https://www.britannica.com:443/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "7e694f6ecca20a7ba721cb1cf7c19420af959a0419a2e806a7936aea3e7740cb", "bytes_read": 53407 }, "llms_state": "not_detected", "control": null }, { "rank": 670, "domain": "hstgr.net", "robots": { "requested_url": "https://hstgr.net/robots.txt", "observed_at": "2026-09-20T06:25:31.534028+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://hstgr.net/llms.txt", "observed_at": "2026-09-20T06:25:31.747129+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 671, "domain": "vedsalb.com", "robots": { "requested_url": "https://vedsalb.com/robots.txt", "observed_at": "2026-09-20T06:25:31.599056+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://vedsalb.com/llms.txt", "observed_at": "2026-09-20T06:25:31.655690+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 672, "domain": "elpais.com", "robots": { "requested_url": "https://elpais.com/robots.txt", "observed_at": "2026-09-20T06:25:31.657251+00:00", "status": 200, "final_url": "https://elpais.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "24c2d0a38586c6ae57ea2030d5451034720725031c8273a8090e881b6cb2bda6", "bytes_read": 7738 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://elpais.com/llms.txt", "observed_at": "2026-09-20T06:25:31.913690+00:00", "status": 404, "final_url": "https://elpais.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "d2649620af60ed8d71135a02a267246cecf5711d2b6d93a5d1a61f3313631498", "bytes_read": 73139 }, "llms_state": "not_detected", "control": null }, { "rank": 673, "domain": "cmediahub.ru", "robots": { "requested_url": "https://cmediahub.ru/robots.txt", "observed_at": "2026-09-20T06:25:31.845485+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cmediahub.ru/llms.txt", "observed_at": "2026-09-20T06:25:32.024012+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 674, "domain": "uk.com", "robots": { "requested_url": "https://uk.com/robots.txt", "observed_at": "2026-09-20T06:25:32.002274+00:00", "status": 200, "final_url": "https://uk.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://uk.com/llms.txt", "observed_at": "2026-09-20T06:25:32.359523+00:00", "status": 404, "final_url": "https://uk.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "87b3ddc9ea88b56f1160bcd034402d3d5053d7b81b8b8801ce153308335c8529", "bytes_read": 8087 }, "llms_state": "not_detected", "control": null }, { "rank": 675, "domain": "primevideo.com", "robots": { "requested_url": "https://primevideo.com/robots.txt", "observed_at": "2026-09-20T06:25:32.025647+00:00", "status": 200, "final_url": "https://www.primevideo.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e924a08cd574df9664040688a18d722c6f08c1f8a7352ea8748dcda4795d04c8", "bytes_read": 4348 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://primevideo.com/llms.txt", "observed_at": "2026-09-20T06:25:32.458636+00:00", "status": 404, "final_url": "https://www.primevideo.com/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "05440d44c35780e9fe03ae2d1aada60b5934291f8084d9ac99c8839c2a1aaa4b", "bytes_read": 6649 }, "llms_state": "not_detected", "control": null }, { "rank": 676, "domain": "bunnyinfra.net", "robots": { "requested_url": "https://bunnyinfra.net/robots.txt", "observed_at": "2026-09-20T06:25:32.201407+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://bunnyinfra.net/llms.txt", "observed_at": "2026-09-20T06:25:32.252137+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 677, "domain": "ks-cdn.com", "robots": { "requested_url": "https://ks-cdn.com/robots.txt", "observed_at": "2026-09-20T06:25:32.254240+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ks-cdn.com/llms.txt", "observed_at": "2026-09-20T06:25:32.515359+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 678, "domain": "linode.com", "robots": { "requested_url": "https://linode.com/robots.txt", "observed_at": "2026-09-20T06:25:32.259186+00:00", "status": 200, "final_url": "https://www.linode.com/robots.txt", "content_type": "text/plain;charset=iso-8859-1", "truncated": false, "body_sha256": "78512e7964119cd972dca87ec957f0d1709a492cb873637c7b2337da692b86f2", "bytes_read": 499 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://linode.com/llms.txt", "observed_at": "2026-09-20T06:25:38.995032+00:00", "status": 404, "final_url": "https://www.linode.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "a30c654ad89329146693a4d42f25b9e4210302f307b34cfdcd309651afb809c4", "bytes_read": 98421 }, "llms_state": "not_detected", "control": null }, { "rank": 679, "domain": "surveymonkey.com", "robots": { "requested_url": "https://surveymonkey.com/robots.txt", "observed_at": "2026-09-20T06:25:32.517079+00:00", "status": 200, "final_url": "https://www.surveymonkey.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "cdb4a5de29db1e98b30415120f4968b5eb7b7251d5e70b9bb8836ea76935a086", "bytes_read": 713 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://surveymonkey.com/llms.txt", "observed_at": "2026-09-20T06:25:33.364849+00:00", "status": 200, "final_url": "https://www.surveymonkey.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "44a0fc5e17adf74bcf93501f90f72a6cb20d74e012bf57127a5b1d86ecd02efa", "bytes_read": 13693 }, "llms_state": "candidate", "control": { "requested_url": "https://surveymonkey.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:33.910719+00:00", "status": 404, "final_url": "https://www.surveymonkey.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "b7b15d89bd9a644c18059799df8946bea1f8f0690367c038fa043de97a5be998", "bytes_read": 4234 } }, { "rank": 680, "domain": "anydesk.com", "robots": { "requested_url": "https://anydesk.com/robots.txt", "observed_at": "2026-09-20T06:25:32.519025+00:00", "status": 403, "final_url": "https://anydesk.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "67f194a1c9e8d1bc3bdf3f919cb4a9b0d8c2634469d43193dc71f71af2cdc863", "bytes_read": 5517 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://anydesk.com/llms.txt", "observed_at": "2026-09-20T06:25:32.706131+00:00", "status": 403, "final_url": "https://anydesk.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "ab315ed95e36b0e7091f15f6199ffba1a54cf18ba0abe6a8bd206664815cbf0c", "bytes_read": 5511 }, "llms_state": "unavailable", "control": null }, { "rank": 681, "domain": "360.cn", "robots": { "requested_url": "https://360.cn/robots.txt", "observed_at": "2026-09-20T06:25:32.677246+00:00", "status": 200, "final_url": "https://360.cn/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "fb486bfead955e9484dc3f7de40fc886a873d4382d84ad7e8ccfe991022c3d0d", "bytes_read": 502 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://360.cn/llms.txt", "observed_at": "2026-09-20T06:25:34.160168+00:00", "status": 404, "final_url": "http://www.360.cn/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "3c398592d6cbc2c1086b6f6596a5971cbebd7262657cd42635e035f710f4213e", "bytes_read": 6780 }, "llms_state": "not_detected", "control": null }, { "rank": 682, "domain": "viber.com", "robots": { "requested_url": "https://viber.com/robots.txt", "observed_at": "2026-09-20T06:25:32.685836+00:00", "status": 200, "final_url": "https://www.viber.com:443/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "35f152efd24080a322c9661a0e194fc9e63f5011a7263fb4565fcf762797445d", "bytes_read": 181 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://viber.com/llms.txt", "observed_at": "2026-09-20T06:25:33.265851+00:00", "status": 200, "final_url": "https://chats.viber.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "a3c559eeb1ec19e0c05aaa7f85953a037834bc636a7c1820e38cdc779565b7f0", "bytes_read": 67149 }, "llms_state": "not_detected", "control": null }, { "rank": 683, "domain": "cambridge.org", "robots": { "requested_url": "https://cambridge.org/robots.txt", "observed_at": "2026-09-20T06:25:32.748982+00:00", "status": 200, "final_url": "https://cambridge.org/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "dddc775cf11f6d536d1ff56e0f6a65e6824964b8ad393da7113ab9552b31441f", "bytes_read": 2785 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://cambridge.org/llms.txt", "observed_at": "2026-09-20T06:25:32.979986+00:00", "status": null, "error": "TimeoutError" }, "llms_state": "unavailable", "control": null }, { "rank": 684, "domain": "geobasket.ru", "robots": { "requested_url": "https://geobasket.ru/robots.txt", "observed_at": "2026-09-20T06:25:32.837845+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://geobasket.ru/llms.txt", "observed_at": "2026-09-20T06:25:33.019627+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 685, "domain": "perplexity.ai", "robots": { "requested_url": "https://perplexity.ai/robots.txt", "observed_at": "2026-09-20T06:25:33.021286+00:00", "status": 200, "final_url": "https://www.perplexity.ai/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "33f5967c54d4934e2250adb72f47d9187855f36d5816c074cb550ee3e7116256", "bytes_read": 448 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://perplexity.ai/llms.txt", "observed_at": "2026-09-20T06:25:33.454912+00:00", "status": 404, "final_url": "https://www.perplexity.ai/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "c8c28468e13d462d67e658b80b1a9fc84384676ad1c9e10e1ac0859d3139ea47", "bytes_read": 12980 }, "llms_state": "not_detected", "control": null }, { "rank": 686, "domain": "pvp.net", "robots": { "requested_url": "https://pvp.net/robots.txt", "observed_at": "2026-09-20T06:25:33.724855+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://pvp.net/llms.txt", "observed_at": "2026-09-20T06:25:33.779814+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 687, "domain": "consultant.ru", "robots": { "requested_url": "https://consultant.ru/robots.txt", "observed_at": "2026-09-20T06:25:33.781676+00:00", "status": 200, "final_url": "https://www.consultant.ru/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "9d22354d3bd5641e4e1094079245379f50ffb6e4488832ffa4f4b4fdf2053944", "bytes_read": 1176 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://consultant.ru/llms.txt", "observed_at": "2026-09-20T06:25:34.956469+00:00", "status": 404, "final_url": "https://www.consultant.ru/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 }, "llms_state": "not_detected", "control": null }, { "rank": 688, "domain": "ioref.io", "robots": { "requested_url": "https://ioref.io/robots.txt", "observed_at": "2026-09-20T06:25:34.374821+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ioref.io/llms.txt", "observed_at": "2026-09-20T06:25:34.584174+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 689, "domain": "hihonorcloud.com", "robots": { "requested_url": "https://hihonorcloud.com/robots.txt", "observed_at": "2026-09-20T06:25:34.413656+00:00", "status": 200, "final_url": "https://hihonorcloud.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "331ea9090db0c9f6f597bd9840fd5b171830f6e0b3ba1cb24dfa91f0c95aedc1", "bytes_read": 26 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://hihonorcloud.com/llms.txt", "observed_at": "2026-09-20T06:25:35.610494+00:00", "status": 200, "final_url": "https://hihonorcloud.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "d3de40af5499cd03f4f8d80a90fc686408146a9583d3fa8d2ddff4afb4988387", "bytes_read": 2080 }, "llms_state": "not_detected", "control": null }, { "rank": 690, "domain": "optimizely.com", "robots": { "requested_url": "https://optimizely.com/robots.txt", "observed_at": "2026-09-20T06:25:34.588894+00:00", "status": 200, "final_url": "https://www.optimizely.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "3170a5a807b7f6d1dc85b00572afaee7f3224bcdea1e8848764010d371370afd", "bytes_read": 207 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://optimizely.com/llms.txt", "observed_at": "2026-09-20T06:25:35.164843+00:00", "status": 200, "final_url": "https://www.optimizely.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "f68612e65deb8aceaf5c21b743f9d383d46ab0304d04ab5411f973b8e5317bb2", "bytes_read": 10751 }, "llms_state": "candidate", "control": { "requested_url": "https://optimizely.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:35.638746+00:00", "status": 404, "final_url": "https://www.optimizely.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "71ac97ec0922b187922f1828db203f0e6d0c0cee1e4b0716b519c76c705479b3", "bytes_read": 52201 } }, { "rank": 691, "domain": "amazon.in", "robots": { "requested_url": "https://amazon.in/robots.txt", "observed_at": "2026-09-20T06:25:36.075073+00:00", "status": 200, "final_url": "https://www.amazon.in/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "7af2f6fe12a2ee4c88227d3a5d79f6690974bcf60422fcfb460cedf6bee5941f", "bytes_read": 9320 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://amazon.in/llms.txt", "observed_at": "2026-09-20T06:25:37.041248+00:00", "status": 404, "final_url": "https://www.amazon.in/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "72e3245551723b12f5d1bada7e9a469f4c49594e2b8df22ce147f8b37ce10eb6", "bytes_read": 2186 }, "llms_state": "not_detected", "control": null }, { "rank": 692, "domain": "dailymail.co.uk", "robots": { "requested_url": "https://dailymail.co.uk/robots.txt", "observed_at": "2026-09-20T06:25:36.281214+00:00", "status": 200, "final_url": "https://www.dailymail.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "7d3972a2058d918c91ea7b8565cb337346c03bc5930786fe80c3a100a8fe4b44", "bytes_read": 5452 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://dailymail.co.uk/llms.txt", "observed_at": "2026-09-20T06:25:36.770846+00:00", "status": 200, "final_url": "https://www.dailymail.com/home/index.html", "content_type": "text/html;charset=UTF-8", "truncated": true, "body_sha256": "4e42c449e0a3e02a0b4b03db0a4ac9405b1b9500b50322bd2e940023b59bd204", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 693, "domain": "outbrain.com", "robots": { "requested_url": "https://outbrain.com/robots.txt", "observed_at": "2026-09-20T06:25:36.721316+00:00", "status": 200, "final_url": "https://www.outbrain.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "8404cf565cf0e06053e02f263214fa83dd0afeed141fdb1d338465e8f6d3bc93", "bytes_read": 2736 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://outbrain.com/llms.txt", "observed_at": "2026-09-20T06:25:37.673801+00:00", "status": 406, "final_url": "https://www.outbrain.com/llms.txt", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "unavailable", "control": null }, { "rank": 694, "domain": "dotomi.com", "robots": { "requested_url": "https://dotomi.com/robots.txt", "observed_at": "2026-09-20T06:25:36.790615+00:00", "status": 200, "final_url": "https://www.epsilon.com/us?redirect=dotomi", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "de7f3741eaab49d91e5a785145c3f04339a6623e30778c10e586c16fc8464907", "bytes_read": 324286 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://dotomi.com/llms.txt", "observed_at": "2026-09-20T06:25:37.512871+00:00", "status": 200, "final_url": "https://www.epsilon.com/us?redirect=dotomi", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "de7f3741eaab49d91e5a785145c3f04339a6623e30778c10e586c16fc8464907", "bytes_read": 324286 }, "llms_state": "not_detected", "control": null }, { "rank": 695, "domain": "typekit.net", "robots": { "requested_url": "https://typekit.net/robots.txt", "observed_at": "2026-09-20T06:25:37.281598+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://typekit.net/llms.txt", "observed_at": "2026-09-20T06:25:57.320152+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 696, "domain": "googlezip.net", "robots": { "requested_url": "https://googlezip.net/robots.txt", "observed_at": "2026-09-20T06:25:37.784732+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://googlezip.net/llms.txt", "observed_at": "2026-09-20T06:25:37.837434+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 697, "domain": "wpguardian.io", "robots": { "requested_url": "https://wpguardian.io/robots.txt", "observed_at": "2026-09-20T06:25:37.838972+00:00", "status": 200, "final_url": "https://wpguardian.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "5a17bba063e4546358bd8c400658ff72cc543c1d0a1c48d154e7d3a1b1d7f3cf", "bytes_read": 172 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://wpguardian.io/llms.txt", "observed_at": "2026-09-20T06:25:38.607042+00:00", "status": 404, "final_url": "https://wpguardian.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "05c7a2248bc29604bc29b5e537f7589b227b78741544754563abb6260de2bacf", "bytes_read": 151403 }, "llms_state": "not_detected", "control": null }, { "rank": 698, "domain": "verisign.com", "robots": { "requested_url": "https://verisign.com/robots.txt", "observed_at": "2026-09-20T06:25:38.008617+00:00", "status": 200, "final_url": "https://www.verisign.com/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "2fa10e2d0d48a6f2f8b611d3e2816854a89755736bb5df6797100abbcf69ec93", "bytes_read": 62 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://verisign.com/llms.txt", "observed_at": "2026-09-20T06:25:38.501875+00:00", "status": 404, "final_url": "https://www.verisign.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "5e453f6e235cb31f2f5cffa5e6ad6e19c07501ef2e4fe81d73214f4b855e3680", "bytes_read": 101166 }, "llms_state": "not_detected", "control": null }, { "rank": 699, "domain": "vkontakte.ru", "robots": { "requested_url": "https://vkontakte.ru/robots.txt", "observed_at": "2026-09-20T06:25:38.138250+00:00", "status": 200, "final_url": "https://vkontakte.ru/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "d2efe58217458a8d8e6a5351dd63bb36442bd2b4f5cab94ce359f83338513888", "bytes_read": 55 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://vkontakte.ru/llms.txt", "observed_at": "2026-09-20T06:25:38.726455+00:00", "status": 404, "final_url": "https://vk.ru/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "02df22b1683f5dfbe9b9b06e4a1332d2be7acc48714baa1c504edacfbaa97f39", "bytes_read": 13120 }, "llms_state": "not_detected", "control": null }, { "rank": 700, "domain": "twimg.com", "robots": { "requested_url": "https://twimg.com/robots.txt", "observed_at": "2026-09-20T06:25:39.115821+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://twimg.com/llms.txt", "observed_at": "2026-09-20T06:25:39.149605+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 701, "domain": "seznam.cz", "robots": { "requested_url": "https://seznam.cz/robots.txt", "observed_at": "2026-09-20T06:25:39.152111+00:00", "status": 200, "final_url": "https://www.seznam.cz/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "bc759a9ffea145d451e17a716008a0fb49b1ea11c70daa32a3a31ae32946c856", "bytes_read": 219 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://seznam.cz/llms.txt", "observed_at": "2026-09-20T06:25:40.253611+00:00", "status": 404, "final_url": "https://www.seznam.cz/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "bc171ab88bb9d5cf781629b8b4790dc6712d4109af2a310cc83b6aa4aa5794f4", "bytes_read": 3143 }, "llms_state": "not_detected", "control": null }, { "rank": 702, "domain": "wps.com", "robots": { "requested_url": "https://wps.com/robots.txt", "observed_at": "2026-09-20T06:25:39.326013+00:00", "status": 200, "final_url": "https://www.wps.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "163ac61fd8fd59d9495ac3d94f8c19f799ff067aaf32a65df29f2d941a2822d3", "bytes_read": 666 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://wps.com/llms.txt", "observed_at": "2026-09-20T06:25:41.002065+00:00", "status": 200, "final_url": "https://www.wps.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "8a4ef0584f62057f72fd3fc85acf49497487552dc99d662bf9d9fc901962d071", "bytes_read": 6576 }, "llms_state": "candidate", "control": { "requested_url": "https://wps.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:42.737929+00:00", "status": 404, "final_url": "https://www.wps.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "c27a029f370c9a940ff528e452bff7f43610c2190c1a526efa8272f3be8b18e1", "bytes_read": 13192 } }, { "rank": 703, "domain": "imgur.com", "robots": { "requested_url": "https://imgur.com/robots.txt", "observed_at": "2026-09-20T06:25:39.639798+00:00", "status": 200, "final_url": "https://imgur.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "dfc6720c95e4eb0766557b6ee996c067b90c054cf02739b5b822187c2aab4489", "bytes_read": 352 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://imgur.com/llms.txt", "observed_at": "2026-09-20T06:25:39.901603+00:00", "status": 200, "final_url": "https://imgur.com/", "content_type": "text/html", "truncated": false, "body_sha256": "bc8fb1bc766271795d7e47a9e7e55d8ed165259780697f345945205203c537f8", "bytes_read": 6675 }, "llms_state": "not_detected", "control": null }, { "rank": 704, "domain": "alipaydns.com", "robots": { "requested_url": "https://alipaydns.com/robots.txt", "observed_at": "2026-09-20T06:25:40.243446+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://alipaydns.com/llms.txt", "observed_at": "2026-09-20T06:25:42.016635+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 705, "domain": "kick.com", "robots": { "requested_url": "https://kick.com/robots.txt", "observed_at": "2026-09-20T06:25:40.569712+00:00", "status": 200, "final_url": "https://kick.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "4a669022df53f6cd8250fe15a8a38ebe4bb467b2d191f631416bc8906aad2321", "bytes_read": 201 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://kick.com/llms.txt", "observed_at": "2026-09-20T06:25:40.912404+00:00", "status": 404, "final_url": "https://kick.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "99014764bc5d2d6e7fef5e736157f321077c697ab4a28fb9c1e25ff8464a9fdb", "bytes_read": 16348 }, "llms_state": "not_detected", "control": null }, { "rank": 706, "domain": "cbsnews.com", "robots": { "requested_url": "https://cbsnews.com/robots.txt", "observed_at": "2026-09-20T06:25:41.147233+00:00", "status": 200, "final_url": "https://www.cbsnews.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "3183f29f832838e64ddcdfe0fa253efe5003d650c636ba40efa6cfcfe4489570", "bytes_read": 950 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://cbsnews.com/llms.txt", "observed_at": "2026-09-20T06:25:41.548529+00:00", "status": 404, "final_url": "https://www.cbsnews.com/llms.txt/", "content_type": "text/html; charset=UTF-8", "truncated": true, "body_sha256": "e5182ca4810ca2aeb132de18ad5592f3b5dbdba1af2d3142ee905915852c3b7d", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 707, "domain": "lemonde.fr", "robots": { "requested_url": "https://lemonde.fr/robots.txt", "observed_at": "2026-09-20T06:25:41.242918+00:00", "status": 200, "final_url": "https://www.lemonde.fr/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "843d2d9af6837d4c052a7d7b88c2a76463b4ac9880d191dbd8800b14c67c85ca", "bytes_read": 13226 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://lemonde.fr/llms.txt", "observed_at": "2026-09-20T06:25:42.124831+00:00", "status": 200, "final_url": "https://www.lemonde.fr/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "32ed63159c77e21ee19ca1b9aa3213ccf0218eb59539560b132a8e68ef0e18ea", "bytes_read": 3038 }, "llms_state": "not_detected", "control": null }, { "rank": 708, "domain": "wp.pl", "robots": { "requested_url": "https://wp.pl/robots.txt", "observed_at": "2026-09-20T06:25:42.270437+00:00", "status": 200, "final_url": "https://www.wp.pl/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "5ab124cdf6f6080b0a0a3ad63bf43d11d4b774ae78e27e49f1831eba8c83020c", "bytes_read": 640 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://wp.pl/llms.txt", "observed_at": "2026-09-20T06:25:43.521254+00:00", "status": 404, "final_url": "https://www.wp.pl/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 }, "llms_state": "not_detected", "control": null }, { "rank": 709, "domain": "dailymail.com", "robots": { "requested_url": "https://dailymail.com/robots.txt", "observed_at": "2026-09-20T06:25:42.461712+00:00", "status": 200, "final_url": "https://www.dailymail.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "7d3972a2058d918c91ea7b8565cb337346c03bc5930786fe80c3a100a8fe4b44", "bytes_read": 5452 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://dailymail.com/llms.txt", "observed_at": "2026-09-20T06:25:42.818919+00:00", "status": 200, "final_url": "https://www.dailymail.com/home/index.html", "content_type": "text/html;charset=UTF-8", "truncated": true, "body_sha256": "2e7ff899e3a3e025e1a2dbdb29035d6b724be197bc5913077799ae08217ab31d", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 710, "domain": "fontawesome.com", "robots": { "requested_url": "https://fontawesome.com/robots.txt", "observed_at": "2026-09-20T06:25:42.819550+00:00", "status": 200, "final_url": "https://fontawesome.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "34976766938050265e30304f604d457d37504e9060869b905741de23d1d2cbc1", "bytes_read": 358 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://fontawesome.com/llms.txt", "observed_at": "2026-09-20T06:25:43.018242+00:00", "status": 404, "final_url": "https://fontawesome.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "198ebbe415dd3cfca4bf4ba90b3f43ff1d4f71bd4784b61764191ad086cecf14", "bytes_read": 14757 }, "llms_state": "not_detected", "control": null }, { "rank": 711, "domain": "sophos.com", "robots": { "requested_url": "https://sophos.com/robots.txt", "observed_at": "2026-09-20T06:25:43.088049+00:00", "status": 200, "final_url": "https://www.sophos.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "368ccb25ca88bfa6fb5f8c9b908f726f93101ae976099ea6e4bffd16913d336c", "bytes_read": 1719 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://sophos.com/llms.txt", "observed_at": "2026-09-20T06:25:43.955047+00:00", "status": 200, "final_url": "https://www.sophos.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "d9b04a74f9861a8beea73c8f77dd3e45779197d42c77296390b2dfe544e76750", "bytes_read": 25391 }, "llms_state": "candidate", "control": { "requested_url": "https://sophos.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:44.401396+00:00", "status": 404, "final_url": "https://www.sophos.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "b52d8820c8b09407f306411e2f94c70126fc34f8b967a3dd61deaaae5f983073", "bytes_read": 6625 } }, { "rank": 712, "domain": "samsungacr.com", "robots": { "requested_url": "https://samsungacr.com/robots.txt", "observed_at": "2026-09-20T06:25:43.216421+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://samsungacr.com/llms.txt", "observed_at": "2026-09-20T06:25:43.380994+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 713, "domain": "id5-sync.com", "robots": { "requested_url": "https://id5-sync.com/robots.txt", "observed_at": "2026-09-20T06:25:43.382558+00:00", "status": 200, "final_url": "https://console.id5.io/login/auth", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "e41eea93c7f7e3a26b389b446b519d5b559138160f658ca85102d23a734ab1af", "bytes_read": 6841 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://id5-sync.com/llms.txt", "observed_at": "2026-09-20T06:25:46.051635+00:00", "status": 200, "final_url": "https://console.id5.io/login/auth", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "e41eea93c7f7e3a26b389b446b519d5b559138160f658ca85102d23a734ab1af", "bytes_read": 6841 }, "llms_state": "not_detected", "control": null }, { "rank": 714, "domain": "cdnhwc1.com", "robots": { "requested_url": "https://cdnhwc1.com/robots.txt", "observed_at": "2026-09-20T06:25:43.553195+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cdnhwc1.com/llms.txt", "observed_at": "2026-09-20T06:25:43.857933+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 715, "domain": "iso.org", "robots": { "requested_url": "https://iso.org/robots.txt", "observed_at": "2026-09-20T06:25:43.786070+00:00", "status": 403, "final_url": "https://www.iso.org/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "17de9e970d58e49f92ce950d75ccfe7c88ad4e7dcecd21efd8689319c920bf87", "bytes_read": 5538 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://iso.org/llms.txt", "observed_at": "2026-09-20T06:25:44.324969+00:00", "status": 403, "final_url": "https://iso.org/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "761adf39e6bf25ac04fba25cefc064ddb1ae9a3ade608a680363828ed96c42c0", "bytes_read": 5592 }, "llms_state": "unavailable", "control": null }, { "rank": 716, "domain": "akamaitech.net", "robots": { "requested_url": "https://akamaitech.net/robots.txt", "observed_at": "2026-09-20T06:25:43.859429+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://akamaitech.net/llms.txt", "observed_at": "2026-09-20T06:25:43.923263+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 717, "domain": "fb.com", "robots": { "requested_url": "https://fb.com/robots.txt", "observed_at": "2026-09-20T06:25:43.924981+00:00", "status": 200, "final_url": "https://fb.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "ae34650d8199df7bcfbcd9da82b8fa911e522d3201af364ee6cae6e9872653d2", "bytes_read": 26872 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://fb.com/llms.txt", "observed_at": "2026-09-20T06:25:44.239560+00:00", "status": 200, "final_url": "https://www.facebook.com/llms.txt", "content_type": "text/html; charset=\"utf-8\"", "truncated": false, "body_sha256": "5a6d00e2d0a7a2bdf43a67d7579da66f2ed7dbbe6a3c7546f78dfc97c743a14f", "bytes_read": 321751 }, "llms_state": "not_detected", "control": null }, { "rank": 718, "domain": "marriott.com", "robots": { "requested_url": "https://marriott.com/robots.txt", "observed_at": "2026-09-20T06:25:44.323685+00:00", "status": 200, "final_url": "https://www.marriott.com/robots.txt", "content_type": "text/plain;charset=iso-8859-1", "truncated": false, "body_sha256": "3fde12219521507a4bd08941b71a20c72c84f13b2d64cafbcb01adeb2c76bed8", "bytes_read": 2643 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://marriott.com/llms.txt", "observed_at": "2026-09-20T06:25:45.180152+00:00", "status": 404, "final_url": "https://www.marriott.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "b035aef7b190c0761dc9be213278a5c313744238b4827ca01a8d238c2889b16f", "bytes_read": 273585 }, "llms_state": "not_detected", "control": null }, { "rank": 719, "domain": "pinimg.com", "robots": { "requested_url": "https://pinimg.com/robots.txt", "observed_at": "2026-09-20T06:25:44.460234+00:00", "status": 200, "final_url": "https://www.pinterest.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "d6b2d2e5111deab042626a98141062225d75bafeaaec7d8dacfbb1f4021b60af", "bytes_read": 30906 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://pinimg.com/llms.txt", "observed_at": "2026-09-20T06:25:44.912689+00:00", "status": 200, "final_url": "https://www.pinterest.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "803b0a6c2522c16af7a3af33e128fada186eabf1e08440f826e10bd6848f6758", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 720, "domain": "amazon.es", "robots": { "requested_url": "https://amazon.es/robots.txt", "observed_at": "2026-09-20T06:25:44.497491+00:00", "status": 200, "final_url": "https://www.amazon.es/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "c2404b36308e118eb694e1cc621e42b4c2c9e0db87a961397f40679ce8cf71ec", "bytes_read": 7063 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://amazon.es/llms.txt", "observed_at": "2026-09-20T06:25:45.691065+00:00", "status": 404, "final_url": "https://www.amazon.es/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "2562570d7bf7138f6f7056801f2b8e774e5ad9cf035c40709cfe4568768fd468", "bytes_read": 2284 }, "llms_state": "not_detected", "control": null }, { "rank": 721, "domain": "ring.com", "robots": { "requested_url": "https://ring.com/robots.txt", "observed_at": "2026-09-20T06:25:44.867979+00:00", "status": 200, "final_url": "https://ring.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "9e84de2b85d734c477495fd6912d3a5fce622e9e34e16b0e8408e7729bc5a3a3", "bytes_read": 1935 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://ring.com/llms.txt", "observed_at": "2026-09-20T06:25:45.174597+00:00", "status": 200, "final_url": "https://ring.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "e2c5cc8d009953838ad15ae1596f6a5eca7562a60f41c5990ddbb769e4209ec6", "bytes_read": 7398 }, "llms_state": "candidate", "control": { "requested_url": "https://ring.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:45.379589+00:00", "status": 404, "final_url": "https://ring.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "39d2ceee0d7f6851468a44bbeac7a6b1f3dda176eecbbe29c34ebdbad7af4cf2", "bytes_read": 16275 } }, { "rank": 722, "domain": "uber.com", "robots": { "requested_url": "https://uber.com/robots.txt", "observed_at": "2026-09-20T06:25:45.099563+00:00", "status": 200, "final_url": "https://www.uber.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "40a0335cf0146087d2473b20e8d2c437406e8eaf0bcd4ed93c24c874a1d560db", "bytes_read": 904 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://uber.com/llms.txt", "observed_at": "2026-09-20T06:25:45.792225+00:00", "status": 404, "final_url": "https://www.uber.com/us/en/llms.txt/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "d32571ffb5da9ef6d1751c091abe2dd1fc82b3ca7f7a35b4556f2552c387693d", "bytes_read": 409835 }, "llms_state": "not_detected", "control": null }, { "rank": 723, "domain": "wpguardian.com", "robots": { "requested_url": "https://wpguardian.com/robots.txt", "observed_at": "2026-09-20T06:25:45.252935+00:00", "status": 200, "final_url": "https://wpguardian.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "5a17bba063e4546358bd8c400658ff72cc543c1d0a1c48d154e7d3a1b1d7f3cf", "bytes_read": 172 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://wpguardian.com/llms.txt", "observed_at": "2026-09-20T06:25:45.382833+00:00", "status": 404, "final_url": "https://wpguardian.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "05c7a2248bc29604bc29b5e537f7589b227b78741544754563abb6260de2bacf", "bytes_read": 151403 }, "llms_state": "not_detected", "control": null }, { "rank": 724, "domain": "ya.ru", "robots": { "requested_url": "https://ya.ru/robots.txt", "observed_at": "2026-09-20T06:25:45.419864+00:00", "status": 200, "final_url": "https://ya.ru/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "f31f74f3b37fbfa2532875140bcb0bda4b145cdd9f63e78b82012e3d3b0e25f1", "bytes_read": 29928 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://ya.ru/llms.txt", "observed_at": "2026-09-20T06:25:46.327638+00:00", "status": 404, "final_url": "https://ya.ru/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "59bc8c70362f89e6b0b12866c84ef51e1c235066caad01e1fcb28030c3fdd7ad", "bytes_read": 11787 }, "llms_state": "not_detected", "control": null }, { "rank": 725, "domain": "amazon.it", "robots": { "requested_url": "https://amazon.it/robots.txt", "observed_at": "2026-09-20T06:25:45.628142+00:00", "status": 200, "final_url": "https://www.amazon.it/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "17d961baa7fe069088f772dc126b784e282d4fb87ff9f3dbacd21f2f70bb9c73", "bytes_read": 7450 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://amazon.it/llms.txt", "observed_at": "2026-09-20T06:25:46.755107+00:00", "status": 503, "final_url": "https://www.amazon.it/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "7267d758c59b084521948495be9ca1212bd762b870f991c1172f9e5aae26f365", "bytes_read": 2016 }, "llms_state": "unavailable", "control": null }, { "rank": 726, "domain": "nel.goog", "robots": { "requested_url": "https://nel.goog/robots.txt", "observed_at": "2026-09-20T06:25:45.692280+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://nel.goog/llms.txt", "observed_at": "2026-09-20T06:25:45.775658+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 727, "domain": "quantserve.com", "robots": { "requested_url": "https://quantserve.com/robots.txt", "observed_at": "2026-09-20T06:25:45.777770+00:00", "status": 200, "final_url": "https://quantserve.com/robots.txt", "content_type": "application/json; charset=utf-8", "truncated": false, "body_sha256": "3418fdf912ebd41572d2cdb5d45f66394f4deab915d9ca1a6d85b9bcd3b2e9f6", "bytes_read": 155 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://quantserve.com/llms.txt", "observed_at": "2026-09-20T06:25:46.240438+00:00", "status": 503, "final_url": "https://quantserve.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "38b9d40f680eabc18e7a1ac9c722a4d7130350f973408ff9e5cb4b381e51627f", "bytes_read": 1434 }, "llms_state": "unavailable", "control": null }, { "rank": 728, "domain": "asus.com", "robots": { "requested_url": "https://asus.com/robots.txt", "observed_at": "2026-09-20T06:25:45.847818+00:00", "status": 200, "final_url": "https://www.asus.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "ab9948183b688e5d0bd9e7e4aa92c063288e360a7d08972fa660cb0ef0356811", "bytes_read": 480 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://asus.com/llms.txt", "observed_at": "2026-09-20T06:25:46.831002+00:00", "status": 200, "final_url": "https://www.asus.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "3dec111e2c12961eeefe46d46a490e08444810bd9d83f526b4113779dce34c9c", "bytes_read": 37001 }, "llms_state": "candidate", "control": { "requested_url": "https://asus.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:48.239110+00:00", "status": 404, "final_url": "https://www.asus.com/suede-index-control-9c52d073-no-file.txt/", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "7976f46497b8abe9752b483af5cf739c0ada96748d0252738c2cdeb2bde8873c", "bytes_read": 279706 } }, { "rank": 729, "domain": "bamgrid.com", "robots": { "requested_url": "https://bamgrid.com/robots.txt", "observed_at": "2026-09-20T06:25:46.556663+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://bamgrid.com/llms.txt", "observed_at": "2026-09-20T06:25:46.627577+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 730, "domain": "itch.io", "robots": { "requested_url": "https://itch.io/robots.txt", "observed_at": "2026-09-20T06:25:46.574075+00:00", "status": 200, "final_url": "https://itch.io/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "82fad5f79355e907c73074625296b9af90299aa08851eb737f1104c85887f0dc", "bytes_read": 249 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://itch.io/llms.txt", "observed_at": "2026-09-20T06:25:46.777005+00:00", "status": 404, "final_url": "https://itch.io/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "925874187971cd26c4e029086a13ea3d515cdae8429638acf9ee30b366342d6c", "bytes_read": 6706 }, "llms_state": "not_detected", "control": null }, { "rank": 731, "domain": "independent.co.uk", "robots": { "requested_url": "https://independent.co.uk/robots.txt", "observed_at": "2026-09-20T06:25:46.579371+00:00", "status": 200, "final_url": "https://www.independent.co.uk/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "eac6e656a41e83ccd4ca0ecb1e0307df8ee4e5d2ad1ab84a5c887f6aaa889881", "bytes_read": 496 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://independent.co.uk/llms.txt", "observed_at": "2026-09-20T06:25:47.034510+00:00", "status": 410, "final_url": "https://independent.co.uk/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "8ac791a74f3e2a3e6ee3fdbe4e29693663446ce4cf194c900bb3c2ecf96eab3f", "bytes_read": 410 }, "llms_state": "not_detected", "control": null }, { "rank": 732, "domain": "xboxlive.com", "robots": { "requested_url": "https://xboxlive.com/robots.txt", "observed_at": "2026-09-20T06:25:46.629148+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://xboxlive.com/llms.txt", "observed_at": "2026-09-20T06:25:46.978563+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 733, "domain": "cnet.com", "robots": { "requested_url": "https://cnet.com/robots.txt", "observed_at": "2026-09-20T06:25:46.755454+00:00", "status": 200, "final_url": "https://www.cnet.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "b65ba970f461d6e5530f6a0fc14af06cbdd0896a6693a4b10f0e13a500730abb", "bytes_read": 14549 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://cnet.com/llms.txt", "observed_at": "2026-09-20T06:25:47.316800+00:00", "status": 404, "final_url": "https://www.cnet.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "881cdf9d9bf5217603e0119592a8b8ef75c769de6398ee15022cf06087676f9d", "bytes_read": 202256 }, "llms_state": "not_detected", "control": null }, { "rank": 734, "domain": "porkbun.com", "robots": { "requested_url": "https://porkbun.com/robots.txt", "observed_at": "2026-09-20T06:25:46.977920+00:00", "status": 200, "final_url": "https://porkbun.com/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "e87ac1e92c20def8ee02b969b74f8e30ab14172a001c682237505c7c77b182f3", "bytes_read": 840 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://porkbun.com/llms.txt", "observed_at": "2026-09-20T06:25:47.558243+00:00", "status": 200, "final_url": "https://porkbun.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "adef064a29e211583bfd3055c86a76604a736a29a8bc497a91d0e010d4ae0878", "bytes_read": 42945 }, "llms_state": "not_detected", "control": null }, { "rank": 735, "domain": "mailchimp.com", "robots": { "requested_url": "https://mailchimp.com/robots.txt", "observed_at": "2026-09-20T06:25:46.978875+00:00", "status": 200, "final_url": "https://mailchimp.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "e05b7ad0de825556f53a9fdbac0d36e1fb88bd38d4fef23f76bd0cd51dbc4810", "bytes_read": 101 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://mailchimp.com/llms.txt", "observed_at": "2026-09-20T06:25:47.334493+00:00", "status": 200, "final_url": "https://mailchimp.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": true, "body_sha256": "b31c2058bbc013d649dde432a0eed71835d8804ab627d66d08eca8b596cd5f4c", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 736, "domain": "globalsign.com", "robots": { "requested_url": "https://globalsign.com/robots.txt", "observed_at": "2026-09-20T06:25:47.168479+00:00", "status": 200, "final_url": "https://www.globalsign.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "423bc5c8ae7d04cacf2c601e13b91d24fea217304e833ad2e3fa4742038000b3", "bytes_read": 1727 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://globalsign.com/llms.txt", "observed_at": "2026-09-20T06:25:48.394854+00:00", "status": 404, "final_url": "https://www.globalsign.com/en/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "a5d4a2b01dae3cb33161ec5ef2bafa2e0f413a8ff5520820b9be1524ec4ab4fb", "bytes_read": 87481 }, "llms_state": "not_detected", "control": null }, { "rank": 737, "domain": "stripchat.com", "robots": { "requested_url": "https://stripchat.com/robots.txt", "observed_at": "2026-09-20T06:25:47.214670+00:00", "status": 200, "final_url": "https://stripchat.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "26d354d37efaa4e1a664f67f26060ca23815a49365c0a25258aab19d4866e119", "bytes_read": 699 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://stripchat.com/llms.txt", "observed_at": "2026-09-20T06:25:47.407956+00:00", "status": 406, "final_url": "https://stripchat.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "unavailable", "control": null }, { "rank": 738, "domain": "dbankcloud.com", "robots": { "requested_url": "https://dbankcloud.com/robots.txt", "observed_at": "2026-09-20T06:25:47.273674+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://dbankcloud.com/llms.txt", "observed_at": "2026-09-20T06:25:47.399207+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 739, "domain": "featureassets.org", "robots": { "requested_url": "https://featureassets.org/robots.txt", "observed_at": "2026-09-20T06:25:47.401383+00:00", "status": 403, "final_url": "https://featureassets.org/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "b5865d15f4439d43678f8f1165ca58106c42c0f1aff9419dfc642fa4dcdac269", "bytes_read": 19 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://featureassets.org/llms.txt", "observed_at": "2026-09-20T06:25:47.629379+00:00", "status": 403, "final_url": "https://featureassets.org/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "b5865d15f4439d43678f8f1165ca58106c42c0f1aff9419dfc642fa4dcdac269", "bytes_read": 19 }, "llms_state": "unavailable", "control": null }, { "rank": 740, "domain": "creativecdn.com", "robots": { "requested_url": "https://creativecdn.com/robots.txt", "observed_at": "2026-09-20T06:25:47.509905+00:00", "status": 200, "final_url": "https://creativecdn.com/robots.txt", "content_type": "", "truncated": false, "body_sha256": "331ea9090db0c9f6f597bd9840fd5b171830f6e0b3ba1cb24dfa91f0c95aedc1", "bytes_read": 26 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://creativecdn.com/llms.txt", "observed_at": "2026-09-20T06:25:47.989879+00:00", "status": 204, "final_url": "https://creativecdn.com/llms.txt", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "unavailable", "control": null }, { "rank": 741, "domain": "mlb.com", "robots": { "requested_url": "https://mlb.com/robots.txt", "observed_at": "2026-09-20T06:25:47.561641+00:00", "status": 200, "final_url": "https://www.mlb.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "f03474a9596d8e7193a8d57eb6d8f68f16d24583df6a93f60d1c555ded23e6b1", "bytes_read": 2224 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://mlb.com/llms.txt", "observed_at": "2026-09-20T06:25:48.025580+00:00", "status": 404, "final_url": "https://www.mlb.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "a7c2ab829b2487642dd1139f3b3364f4a7061a4e83d2a74f1a008cbbfcc3b943", "bytes_read": 118152 }, "llms_state": "not_detected", "control": null }, { "rank": 742, "domain": "githubusercontent.com", "robots": { "requested_url": "https://githubusercontent.com/robots.txt", "observed_at": "2026-09-20T06:25:47.791558+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://githubusercontent.com/llms.txt", "observed_at": "2026-09-20T06:25:47.863145+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 743, "domain": "agkn.com", "robots": { "requested_url": "https://agkn.com/robots.txt", "observed_at": "2026-09-20T06:25:47.800028+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://agkn.com/llms.txt", "observed_at": "2026-09-20T06:25:47.868853+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 744, "domain": "grammarly.io", "robots": { "requested_url": "https://grammarly.io/robots.txt", "observed_at": "2026-09-20T06:25:47.829377+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://grammarly.io/llms.txt", "observed_at": "2026-09-20T06:25:47.902047+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 745, "domain": "shopee.co.id", "robots": { "requested_url": "https://shopee.co.id/robots.txt", "observed_at": "2026-09-20T06:25:47.864839+00:00", "status": 200, "final_url": "https://shopee.co.id/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "5e3c47bbfc45f08e23a188c68384c286bdac93d28b086e6709bda3dbf954521f", "bytes_read": 2795 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://shopee.co.id/llms.txt", "observed_at": "2026-09-20T06:25:48.739212+00:00", "status": 200, "final_url": "https://shopee.co.id/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "317c2d29a522a09ea54aab3f672a0b1799b75a4b7ac07b1f4123b6eec8e50a52", "bytes_read": 197865 }, "llms_state": "not_detected", "control": null }, { "rank": 746, "domain": "bandcamp.com", "robots": { "requested_url": "https://bandcamp.com/robots.txt", "observed_at": "2026-09-20T06:25:47.870796+00:00", "status": 200, "final_url": "https://bandcamp.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "0adec2c36c009bfb6a0a226ee5d1b52953152e189ff14554804ebf2f441483e3", "bytes_read": 1053 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://bandcamp.com/llms.txt", "observed_at": "2026-09-20T06:25:48.028170+00:00", "status": 404, "final_url": "https://bandcamp.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "032d689072f154b6c02b304c6e0d98c8d570cf574bae1d3006038b1de8b6261c", "bytes_read": 47832 }, "llms_state": "not_detected", "control": null }, { "rank": 747, "domain": "bidr.io", "robots": { "requested_url": "https://bidr.io/robots.txt", "observed_at": "2026-09-20T06:25:47.903714+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://bidr.io/llms.txt", "observed_at": "2026-09-20T06:25:47.980366+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 748, "domain": "bugsnag.com", "robots": { "requested_url": "https://bugsnag.com/robots.txt", "observed_at": "2026-09-20T06:25:47.981840+00:00", "status": 200, "final_url": "https://www.bugsnag.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "8fa3036c68bfcbd32365f6225d24333264093b7cd38d306e106b4dbdc934fd5b", "bytes_read": 1248 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://bugsnag.com/llms.txt", "observed_at": "2026-09-20T06:25:48.545614+00:00", "status": 404, "final_url": "https://www.bugsnag.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "df9126b0104e85d7174f2976ae66b7918d817b7a295172cc86a50b8ffea2046d", "bytes_read": 126199 }, "llms_state": "not_detected", "control": null }, { "rank": 749, "domain": "1rx.io", "robots": { "requested_url": "https://1rx.io/robots.txt", "observed_at": "2026-09-20T06:25:48.178362+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://1rx.io/llms.txt", "observed_at": "2026-09-20T06:25:48.250401+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 750, "domain": "att.com", "robots": { "requested_url": "https://att.com/robots.txt", "observed_at": "2026-09-20T06:25:48.252176+00:00", "status": 200, "final_url": "https://www.att.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "71ba24a8c0bc15a15f5d2bc4fc53f9a777961977092f89959763dc343a422e69", "bytes_read": 10283 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://att.com/llms.txt", "observed_at": "2026-09-20T06:25:49.345122+00:00", "status": 200, "final_url": "https://www.att.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "c50469d1c1eab7d0bf791985105c54bfd150219f1ad5bb8b9cf8f14a9246b2d3", "bytes_read": 13132 }, "llms_state": "candidate", "control": { "requested_url": "https://att.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:49.839713+00:00", "status": 404, "final_url": "https://www.att.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "25ddea2147d64b1d67476fb16dc26241b9ec4c441cd4cff0c8ed22fd9d465693", "bytes_read": 282846 } }, { "rank": 751, "domain": "safebrowsing.apple", "robots": { "requested_url": "https://safebrowsing.apple/robots.txt", "observed_at": "2026-09-20T06:25:48.332951+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://safebrowsing.apple/llms.txt", "observed_at": "2026-09-20T06:25:48.407260+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 752, "domain": "nmrodam.com", "robots": { "requested_url": "https://nmrodam.com/robots.txt", "observed_at": "2026-09-20T06:25:48.348102+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://nmrodam.com/llms.txt", "observed_at": "2026-09-20T06:25:48.423312+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 753, "domain": "imrworldwide.com", "robots": { "requested_url": "https://imrworldwide.com/robots.txt", "observed_at": "2026-09-20T06:25:48.408845+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://imrworldwide.com/llms.txt", "observed_at": "2026-09-20T06:25:48.504984+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 754, "domain": "python.org", "robots": { "requested_url": "https://python.org/robots.txt", "observed_at": "2026-09-20T06:25:48.411037+00:00", "status": 200, "final_url": "https://www.python.org/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "18cb4cd525df8528491845e76f3af26c29c6795d02ea8133974d3b341a2ddd9f", "bytes_read": 537 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://python.org/llms.txt", "observed_at": "2026-09-20T06:25:48.908387+00:00", "status": 404, "final_url": "https://www.python.org/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "72f98c03b7997f37ef7a1de8174cccd0701f000b7fb279bd227b697aaa4b88a0", "bytes_read": 33102 }, "llms_state": "not_detected", "control": null }, { "rank": 755, "domain": "zillow.com", "robots": { "requested_url": "https://zillow.com/robots.txt", "observed_at": "2026-09-20T06:25:48.424895+00:00", "status": 200, "final_url": "https://www.zillow.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "48aa7b503b17187c1cb3340c6d9e2289e7dd6920eafbb35ed885a6025c1dc448", "bytes_read": 10524 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://zillow.com/llms.txt", "observed_at": "2026-09-20T06:25:49.133862+00:00", "status": 403, "final_url": "https://www.zillow.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "bb2fc2ca4429802b24971790f0558e1d6ac365c3b5abc5e793776e48b3c693c6", "bytes_read": 5800 }, "llms_state": "unavailable", "control": null }, { "rank": 756, "domain": "g.co", "robots": { "requested_url": "https://g.co/robots.txt", "observed_at": "2026-09-20T06:25:48.500835+00:00", "status": 400, "final_url": "https://g.co/robots.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "bae5d66fbe371f66a934935346906d25bb35df6fb4cd5504a359cd324e89ea11", "bytes_read": 9258 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://g.co/llms.txt", "observed_at": "2026-09-20T06:25:48.898052+00:00", "status": 400, "final_url": "https://g.co/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "bae5d66fbe371f66a934935346906d25bb35df6fb4cd5504a359cd324e89ea11", "bytes_read": 9258 }, "llms_state": "unavailable", "control": null }, { "rank": 757, "domain": "crwdcntrl.net", "robots": { "requested_url": "https://crwdcntrl.net/robots.txt", "observed_at": "2026-09-20T06:25:48.507711+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://crwdcntrl.net/llms.txt", "observed_at": "2026-09-20T06:26:48.569239+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 758, "domain": "google.co.uk", "robots": { "requested_url": "https://google.co.uk/robots.txt", "observed_at": "2026-09-20T06:25:48.529143+00:00", "status": 200, "final_url": "https://www.google.co.uk/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1aba570a1b32deb66941e0c29d4f37be71ee58aa1a624aa20754b3639538cf2c", "bytes_read": 6624 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://google.co.uk/llms.txt", "observed_at": "2026-09-20T06:25:49.045910+00:00", "status": 404, "final_url": "https://google.co.uk/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 759, "domain": "hostgator.com.br", "robots": { "requested_url": "https://hostgator.com.br/robots.txt", "observed_at": "2026-09-20T06:25:48.913165+00:00", "status": 403, "final_url": "https://hostgator.com.br/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "e536b694e9b550542d47e5c53b76d98d6e4d458593bccf907334fdf704bf0341", "bytes_read": 5530 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://hostgator.com.br/llms.txt", "observed_at": "2026-09-20T06:25:49.134764+00:00", "status": 403, "final_url": "https://hostgator.com.br/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "36452009ee8dfe748051527ad30db2b8fbea3c44ac6e0a00dab358fef03e46fe", "bytes_read": 5524 }, "llms_state": "unavailable", "control": null }, { "rank": 760, "domain": "homedepot.com", "robots": { "requested_url": "https://homedepot.com/robots.txt", "observed_at": "2026-09-20T06:25:49.135268+00:00", "status": 200, "final_url": "https://www.homedepot.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "395b103de9406961b0e18f618d92a03303d175fcb681d3c3f40e917248e7d444", "bytes_read": 1411 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://homedepot.com/llms.txt", "observed_at": "2026-09-20T06:25:49.747430+00:00", "status": 404, "final_url": "https://www.homedepot.com/llms.txt", "content_type": "", "truncated": false, "body_sha256": "39b11290814aa29d9ae9214699bb790500595ff1b5992d2e2754624ea368c18e", "bytes_read": 2522 }, "llms_state": "not_detected", "control": null }, { "rank": 761, "domain": "lijit.com", "robots": { "requested_url": "https://lijit.com/robots.txt", "observed_at": "2026-09-20T06:25:49.260040+00:00", "status": 403, "final_url": "https://lijit.com/robots.txt", "content_type": "application/xml", "truncated": false, "body_sha256": "a824bc7739e226e1b40ea0f8c4e4f4c6f796fc3b4abfa6e9abe3bd119a30d938", "bytes_read": 111 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://lijit.com/llms.txt", "observed_at": "2026-09-20T06:25:49.638962+00:00", "status": 403, "final_url": "https://lijit.com/llms.txt", "content_type": "application/xml", "truncated": false, "body_sha256": "a824bc7739e226e1b40ea0f8c4e4f4c6f796fc3b4abfa6e9abe3bd119a30d938", "bytes_read": 111 }, "llms_state": "unavailable", "control": null }, { "rank": 762, "domain": "buydomains.com", "robots": { "requested_url": "https://buydomains.com/robots.txt", "observed_at": "2026-09-20T06:25:49.290100+00:00", "status": 403, "final_url": "https://buydomains.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "9f35e3c4ba02f462924e689c4a65524795391caad946199bb50dfaa5f8327fae", "bytes_read": 5563 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://buydomains.com/llms.txt", "observed_at": "2026-09-20T06:25:49.504813+00:00", "status": 403, "final_url": "https://buydomains.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "251239095201a7412167e9d11fa4c2ac85b412fccd944e668fb649dae8e527dc", "bytes_read": 5557 }, "llms_state": "unavailable", "control": null }, { "rank": 763, "domain": "gumgum.com", "robots": { "requested_url": "https://gumgum.com/robots.txt", "observed_at": "2026-09-20T06:25:49.346736+00:00", "status": 200, "final_url": "https://gumgum.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "d19fe3b979fcc6b00dbda25a391265a2dbef8f5b143d815297298443c8e40036", "bytes_read": 694 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://gumgum.com/llms.txt", "observed_at": "2026-09-20T06:25:49.589869+00:00", "status": 200, "final_url": "https://gumgum.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "771cc4d4d42685e64b8d847a95d4f88852ce5692ee2f78f35363667a7fc14f2b", "bytes_read": 3784 }, "llms_state": "candidate", "control": { "requested_url": "https://gumgum.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:49.815109+00:00", "status": 404, "final_url": "https://gumgum.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "5932201b676fa81cb903ea9b55bf1b95f99a6f26a0234f56c50803a3ad30ed84", "bytes_read": 24 } }, { "rank": 764, "domain": "curseforge.com", "robots": { "requested_url": "https://curseforge.com/robots.txt", "observed_at": "2026-09-20T06:25:49.554102+00:00", "status": 200, "final_url": "https://www.curseforge.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "2a907bca45a4d6a07e34a4d646374c61c430273b196bf8dfaae8537735177065", "bytes_read": 1873 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://curseforge.com/llms.txt", "observed_at": "2026-09-20T06:25:50.016645+00:00", "status": 403, "final_url": "https://www.curseforge.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6cf1d2d5949c6879c4ec2750ae4b01ec3cefe7ddfb89107de88b69812107d350", "bytes_read": 5564 }, "llms_state": "unavailable", "control": null }, { "rank": 765, "domain": "steamstatic.com", "robots": { "requested_url": "https://steamstatic.com/robots.txt", "observed_at": "2026-09-20T06:25:49.617325+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://steamstatic.com/llms.txt", "observed_at": "2026-09-20T06:25:49.699616+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 766, "domain": "airbnb.com", "robots": { "requested_url": "https://airbnb.com/robots.txt", "observed_at": "2026-09-20T06:25:49.701152+00:00", "status": 200, "final_url": "https://www.airbnb.com/robots.txt", "content_type": "text/plain;charset=utf-8", "truncated": false, "body_sha256": "eee43f42bb76d3f0466014b8598e1cf60fa87944298a507ad31840390474f355", "bytes_read": 24175 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [] } }, "llms": { "requested_url": "https://airbnb.com/llms.txt", "observed_at": "2026-09-20T06:25:50.328743+00:00", "status": 404, "final_url": "https://www.airbnb.com/404", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "a71bb7d237aea979f1174a535454c4447a1c804e6f4a8f4574679906eef5d85a", "bytes_read": 2947 }, "llms_state": "not_detected", "control": null }, { "rank": 767, "domain": "ddnss.de", "robots": { "requested_url": "https://ddnss.de/robots.txt", "observed_at": "2026-09-20T06:25:49.941303+00:00", "status": 200, "final_url": "https://ddnss.de/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "64ad836aa6cb58c92874d578e748151e3b69b98ecafd111b671794dfcca540f0", "bytes_read": 286 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://ddnss.de/llms.txt", "observed_at": "2026-09-20T06:25:50.733053+00:00", "status": 404, "final_url": "https://ddnss.de/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "eead39b50cdf9f71ef6d47bd586921592e497336f031cb34ac6a4cc4569f9a01", "bytes_read": 20074 }, "llms_state": "not_detected", "control": null }, { "rank": 768, "domain": "bild.de", "robots": { "requested_url": "https://bild.de/robots.txt", "observed_at": "2026-09-20T06:25:50.066148+00:00", "status": 200, "final_url": "https://www.bild.de/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "f2af535c9e54f1817009854d5e5ac2980d80fd910b2a5af4ea6aa45e3322d497", "bytes_read": 3053 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://bild.de/llms.txt", "observed_at": "2026-09-20T06:25:51.400318+00:00", "status": 404, "final_url": "https://www.bild.de/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "e38e6079c7bdf94f55d117491da2bd432ca1b815b1bf9b4f96fe15c10927bb07", "bytes_read": 134317 }, "llms_state": "not_detected", "control": null }, { "rank": 769, "domain": "herokudns.com", "robots": { "requested_url": "https://herokudns.com/robots.txt", "observed_at": "2026-09-20T06:25:50.087515+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://herokudns.com/llms.txt", "observed_at": "2026-09-20T06:25:50.150925+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 770, "domain": "mediafire.com", "robots": { "requested_url": "https://mediafire.com/robots.txt", "observed_at": "2026-09-20T06:25:50.152697+00:00", "status": 200, "final_url": "https://mediafire.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "226a26d6bbc34931d6dcde870d6c66dbffe162979f1868d810ac4ac03ed55857", "bytes_read": 212 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://mediafire.com/llms.txt", "observed_at": "2026-09-20T06:25:50.394048+00:00", "status": 404, "final_url": "https://www.mediafire.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "efc5cec237cff778f73106998c0a5f2d18a4637d7c342458de51505c4cff9ac3", "bytes_read": 31057 }, "llms_state": "not_detected", "control": null }, { "rank": 771, "domain": "usgovcloudapi.net", "robots": { "requested_url": "https://usgovcloudapi.net/robots.txt", "observed_at": "2026-09-20T06:25:50.249465+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://usgovcloudapi.net/llms.txt", "observed_at": "2026-09-20T06:25:50.319537+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 772, "domain": "criteo.net", "robots": { "requested_url": "https://criteo.net/robots.txt", "observed_at": "2026-09-20T06:25:50.327264+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://criteo.net/llms.txt", "observed_at": "2026-09-20T06:25:50.426252+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 773, "domain": "box.com", "robots": { "requested_url": "https://box.com/robots.txt", "observed_at": "2026-09-20T06:25:50.428003+00:00", "status": 200, "final_url": "https://www.box.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "cb77a92c0a2fe2e45eb81e381bda7c843c0688e0066cd2274fb06a0644deffa9", "bytes_read": 1484 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://box.com/llms.txt", "observed_at": "2026-09-20T06:25:50.939690+00:00", "status": 403, "final_url": "https://www.box.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "df70a614e6ca69b347cca435bc9090897e0f89d9892d7448ea3d26b7914b4ca4", "bytes_read": 5554 }, "llms_state": "unavailable", "control": null }, { "rank": 774, "domain": "fda.gov", "robots": { "requested_url": "https://fda.gov/robots.txt", "observed_at": "2026-09-20T06:25:50.450397+00:00", "status": 404, "final_url": "https://www.fda.gov/apology_objects/abuse-detection-apology.html", "content_type": "", "truncated": false, "body_sha256": "493fda53120050f85836032324409be6c6484f90a0755ae0c6a673ba7626818b", "bytes_read": 10 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://fda.gov/llms.txt", "observed_at": "2026-09-20T06:25:51.489501+00:00", "status": 404, "final_url": "https://www.fda.gov/apology_objects/abuse-detection-apology.html", "content_type": "", "truncated": false, "body_sha256": "493fda53120050f85836032324409be6c6484f90a0755ae0c6a673ba7626818b", "bytes_read": 10 }, "llms_state": "not_detected", "control": null }, { "rank": 775, "domain": "ozone.ru", "robots": { "requested_url": "https://ozone.ru/robots.txt", "observed_at": "2026-09-20T06:25:50.566375+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ozone.ru/llms.txt", "observed_at": "2026-09-20T06:25:51.134877+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 776, "domain": "nikkei.com", "robots": { "requested_url": "https://nikkei.com/robots.txt", "observed_at": "2026-09-20T06:25:50.572319+00:00", "status": 200, "final_url": "https://www.nikkei.com:443/", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "17079ca17436860342aa990ce958dcf6f47ab892456053a07cb61480480deb55", "bytes_read": 512001 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://nikkei.com/llms.txt", "observed_at": "2026-09-20T06:25:52.499819+00:00", "status": 200, "final_url": "https://www.nikkei.com:443/", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "17079ca17436860342aa990ce958dcf6f47ab892456053a07cb61480480deb55", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 777, "domain": "cursor.sh", "robots": { "requested_url": "https://cursor.sh/robots.txt", "observed_at": "2026-09-20T06:25:50.887413+00:00", "status": 200, "final_url": "https://cursor.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "3f6b4f93ddf92ee721fafbbd93a2b28e59a60386b361e7dce6beb9f7afe9c491", "bytes_read": 1047 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://cursor.sh/llms.txt", "observed_at": "2026-09-20T06:25:51.369068+00:00", "status": 200, "final_url": "https://cursor.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "696809c9f2b0b9f8d43c9778744e45777978ec4f0835b56d28dd8e4296f43df4", "bytes_read": 21678 }, "llms_state": "not_detected", "control": null }, { "rank": 778, "domain": "wattpad.com", "robots": { "requested_url": "https://wattpad.com/robots.txt", "observed_at": "2026-09-20T06:25:50.911006+00:00", "status": 200, "final_url": "https://www.wattpad.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "2befe39d3ad6be6b31bb7c834916653042eda54117363c444d2b551ff08e967c", "bytes_read": 1580 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://wattpad.com/llms.txt", "observed_at": "2026-09-20T06:25:51.351413+00:00", "status": 404, "final_url": "https://www.wattpad.com/user/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "eb02e9ac8750d13f0377c1678c94c766c57ef3436cbdf998aedfe86b620a02d3", "bytes_read": 63225 }, "llms_state": "not_detected", "control": null }, { "rank": 779, "domain": "ovscdns.com", "robots": { "requested_url": "https://ovscdns.com/robots.txt", "observed_at": "2026-09-20T06:25:51.256501+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ovscdns.com/llms.txt", "observed_at": "2026-09-20T06:25:51.390876+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 780, "domain": "ad-tech.ru", "robots": { "requested_url": "https://ad-tech.ru/robots.txt", "observed_at": "2026-09-20T06:25:51.302235+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ad-tech.ru/llms.txt", "observed_at": "2026-09-20T06:25:52.196042+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 781, "domain": "gotinder.com", "robots": { "requested_url": "https://gotinder.com/robots.txt", "observed_at": "2026-09-20T06:25:51.392772+00:00", "status": 200, "final_url": "https://www.gotinder.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "fa1aca3bff932b8fde94997e5bb1386c8db6ce4a6c98082004c864faafe2b4d6", "bytes_read": 83 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://gotinder.com/llms.txt", "observed_at": "2026-09-20T06:25:52.186910+00:00", "status": 200, "final_url": "https://www.gotinder.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "808f2b1e564d77587b3e02d721bf91b3facbadd260f0efcd49c06fd343db7088", "bytes_read": 4569 }, "llms_state": "not_detected", "control": null }, { "rank": 782, "domain": "bestbuy.com", "robots": { "requested_url": "https://bestbuy.com/robots.txt", "observed_at": "2026-09-20T06:25:51.482789+00:00", "status": null, "error": "TimeoutError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://bestbuy.com/llms.txt", "observed_at": "2026-09-20T06:26:02.057982+00:00", "status": 404, "final_url": "https://www.bestbuy.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "7a3efca3eb18d7b6facd25b01bed61fdaba72e79a66c4626af855a8d701dbaa2", "bytes_read": 287240 }, "llms_state": "not_detected", "control": null }, { "rank": 783, "domain": "webempresa.eu", "robots": { "requested_url": "https://webempresa.eu/robots.txt", "observed_at": "2026-09-20T06:25:51.497122+00:00", "status": 404, "final_url": "https://webempresa.eu/robots.txt", "content_type": "text/html; charset=iso-8859-1", "truncated": false, "body_sha256": "80c3fe2ae1062abf56456f52518bd670f9ec3917b7f85e152b347ac6b6faf880", "bytes_read": 196 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://webempresa.eu/llms.txt", "observed_at": "2026-09-20T06:25:52.055615+00:00", "status": 404, "final_url": "https://webempresa.eu/llms.txt", "content_type": "text/html; charset=iso-8859-1", "truncated": false, "body_sha256": "80c3fe2ae1062abf56456f52518bd670f9ec3917b7f85e152b347ac6b6faf880", "bytes_read": 196 }, "llms_state": "not_detected", "control": null }, { "rank": 784, "domain": "sedo.com", "robots": { "requested_url": "https://sedo.com/robots.txt", "observed_at": "2026-09-20T06:25:51.746904+00:00", "status": 200, "final_url": "https://sedo.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "0d10cf2911f16a17d3f6d56a8041a2082ca1319fe8f54762284539411e28fc59", "bytes_read": 270 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://sedo.com/llms.txt", "observed_at": "2026-09-20T06:25:51.990804+00:00", "status": 404, "final_url": "https://sedo.com/us/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "9f6629c5bc03f5c92d538962cde5814e24bedeaf8b48fa587a237b802a5d1089", "bytes_read": 64388 }, "llms_state": "not_detected", "control": null }, { "rank": 785, "domain": "keenetic.io", "robots": { "requested_url": "https://keenetic.io/robots.txt", "observed_at": "2026-09-20T06:25:51.914911+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://keenetic.io/llms.txt", "observed_at": "2026-09-20T06:25:52.066388+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 786, "domain": "singular.net", "robots": { "requested_url": "https://singular.net/robots.txt", "observed_at": "2026-09-20T06:25:52.068010+00:00", "status": 200, "final_url": "https://www.singular.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "7ab13f5ede99a50cea1362e24c7b1e664991e8f44ed413f2223c3794a205db37", "bytes_read": 1087 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://singular.net/llms.txt", "observed_at": "2026-09-20T06:25:52.676962+00:00", "status": 200, "final_url": "https://www.singular.net/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "b617da39ed58999acf48ed41f7cf6e5c3e7d7f3a3df5ebf75a7f395217743c33", "bytes_read": 196755 }, "llms_state": "candidate", "control": { "requested_url": "https://singular.net/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:53.175474+00:00", "status": 404, "final_url": "https://www.singular.net/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 } }, { "rank": 787, "domain": "shopeemobile.com", "robots": { "requested_url": "https://shopeemobile.com/robots.txt", "observed_at": "2026-09-20T06:25:52.119828+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://shopeemobile.com/llms.txt", "observed_at": "2026-09-20T06:26:02.162395+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 788, "domain": "wcdnga.com", "robots": { "requested_url": "https://wcdnga.com/robots.txt", "observed_at": "2026-09-20T06:25:52.157293+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://wcdnga.com/llms.txt", "observed_at": "2026-09-20T06:25:52.223416+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 789, "domain": "craigslist.org", "robots": { "requested_url": "https://craigslist.org/robots.txt", "observed_at": "2026-09-20T06:25:52.225244+00:00", "status": 200, "final_url": "https://craigslist.org/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "5a9a92114f608593d9fe7d2ea52809032be3fbab3b0f019a980526acb695ee27", "bytes_read": 151 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://craigslist.org/llms.txt", "observed_at": "2026-09-20T06:25:52.507386+00:00", "status": 404, "final_url": "https://craigslist.org/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "ffa7bca2e590ab21a361d0ed8b8149f1a8f675ed3edbe5a5c6d78e973db30902", "bytes_read": 4544 }, "llms_state": "not_detected", "control": null }, { "rank": 790, "domain": "sc-cdn.net", "robots": { "requested_url": "https://sc-cdn.net/robots.txt", "observed_at": "2026-09-20T06:25:52.545233+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://sc-cdn.net/llms.txt", "observed_at": "2026-09-20T06:25:52.593839+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 791, "domain": "allaboutcookies.org", "robots": { "requested_url": "https://allaboutcookies.org/robots.txt", "observed_at": "2026-09-20T06:25:52.595662+00:00", "status": 403, "final_url": "https://allaboutcookies.org/robots.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "40ca5328d23ff37eecf2ac6a0ae6170dd191c2da6ad3037d77718ea096cb8f7a", "bytes_read": 779 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://allaboutcookies.org/llms.txt", "observed_at": "2026-09-20T06:25:52.870731+00:00", "status": 403, "final_url": "https://allaboutcookies.org/llms.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "65763915167fd5abf9344b81a2f3dfee83f8c97a0226e3d11c29c69eb8148b60", "bytes_read": 779 }, "llms_state": "unavailable", "control": null }, { "rank": 792, "domain": "mckinsey.com", "robots": { "requested_url": "https://mckinsey.com/robots.txt", "observed_at": "2026-09-20T06:25:52.635574+00:00", "status": null, "error": "TimeoutError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://mckinsey.com/llms.txt", "observed_at": "2026-09-20T06:26:03.209488+00:00", "status": null, "error": "TimeoutError" }, "llms_state": "unavailable", "control": null }, { "rank": 793, "domain": "usercontent.goog", "robots": { "requested_url": "https://usercontent.goog/robots.txt", "observed_at": "2026-09-20T06:25:52.771684+00:00", "status": 404, "final_url": "https://usercontent.goog/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6adee83a3eba87d30fdca3997d27097697f2767805f222a0f78c3f04392004d2", "bytes_read": 1571 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://usercontent.goog/llms.txt", "observed_at": "2026-09-20T06:25:53.060031+00:00", "status": 404, "final_url": "https://usercontent.goog/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 794, "domain": "google.com.br", "robots": { "requested_url": "https://google.com.br/robots.txt", "observed_at": "2026-09-20T06:25:52.926283+00:00", "status": 200, "final_url": "https://www.google.com.br/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1aba570a1b32deb66941e0c29d4f37be71ee58aa1a624aa20754b3639538cf2c", "bytes_read": 6624 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://google.com.br/llms.txt", "observed_at": "2026-09-20T06:25:53.344872+00:00", "status": 404, "final_url": "https://google.com.br/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 795, "domain": "lenovo.com", "robots": { "requested_url": "https://lenovo.com/robots.txt", "observed_at": "2026-09-20T06:25:53.010584+00:00", "status": 403, "final_url": "https://www.lenovo.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "938c119900c8fd5687ab49422891039b87ac566568a1149dd30363a688a34c15", "bytes_read": 382 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://lenovo.com/llms.txt", "observed_at": "2026-09-20T06:25:53.686129+00:00", "status": 403, "final_url": "https://www.lenovo.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "4760584ae18798f17ad4b11b78259ae3d313a79f231c3441f78e6e87e233925c", "bytes_read": 380 }, "llms_state": "unavailable", "control": null }, { "rank": 796, "domain": "fiverr.com", "robots": { "requested_url": "https://fiverr.com/robots.txt", "observed_at": "2026-09-20T06:25:53.091250+00:00", "status": 200, "final_url": "https://www.fiverr.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "604d3912766adf35fb1d6e90414ba5f22dedcb78edff3498eb5e5963232eb6c8", "bytes_read": 2049 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://fiverr.com/llms.txt", "observed_at": "2026-09-20T06:25:53.557331+00:00", "status": 200, "final_url": "https://www.fiverr.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "9695ce924240d9d71890018e5ceb68367dc44038c261388ffd1e0ab7ba87ad3f", "bytes_read": 5516 }, "llms_state": "unknown_control", "control": { "requested_url": "https://fiverr.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:53.903040+00:00", "status": 403, "final_url": "https://www.fiverr.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "be612b265f2226611388aedc8a0aee2115c38a0875d202ace06c8888060aa781", "bytes_read": 35960 } }, { "rank": 797, "domain": "rt.ru", "robots": { "requested_url": "https://rt.ru/robots.txt", "observed_at": "2026-09-20T06:25:53.270711+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://rt.ru/llms.txt", "observed_at": "2026-09-20T06:26:03.459217+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 798, "domain": "appcenter.ms", "robots": { "requested_url": "https://appcenter.ms/robots.txt", "observed_at": "2026-09-20T06:25:53.277366+00:00", "status": 404, "final_url": "https://appcenter.ms/robots.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "cc51ecbf3c82ffc569bf5212657b0481a7c66b03c6d15b0f82138c8dc1e7d81e", "bytes_read": 2448 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://appcenter.ms/llms.txt", "observed_at": "2026-09-20T06:25:53.571689+00:00", "status": 404, "final_url": "https://appcenter.ms/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "ae7715063505926b7081b43a6a6cfaa9bc60023e64f3a01d8a5065babed715bd", "bytes_read": 2448 }, "llms_state": "not_detected", "control": null }, { "rank": 799, "domain": "recaptcha.net", "robots": { "requested_url": "https://recaptcha.net/robots.txt", "observed_at": "2026-09-20T06:25:53.567819+00:00", "status": 200, "final_url": "https://recaptcha.net/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "cf6d9ec88a6d6cabc37293b0d53efc0a6c216737969767391861113ad87e63b7", "bytes_read": 153 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://recaptcha.net/llms.txt", "observed_at": "2026-09-20T06:25:53.815890+00:00", "status": 404, "final_url": "https://recaptcha.net/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "efa69c9a25484eb4e67bdb35955d30019f4575f2282a8538063e6aa6494b59fa", "bytes_read": 1621 }, "llms_state": "not_detected", "control": null }, { "rank": 800, "domain": "ted.com", "robots": { "requested_url": "https://ted.com/robots.txt", "observed_at": "2026-09-20T06:25:53.628570+00:00", "status": 200, "final_url": "https://www.ted.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e38b742627d22ff047175c73c29d1f0b24c1b9b3fd3bd260186fbf93149285ee", "bytes_read": 2552 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://ted.com/llms.txt", "observed_at": "2026-09-20T06:25:54.185074+00:00", "status": 404, "final_url": "https://www.ted.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "d36e9938475a49d23333807232d32406c8c23fdf19f8d27485a8c5a00b84b46d", "bytes_read": 28959 }, "llms_state": "not_detected", "control": null }, { "rank": 801, "domain": "liadm.com", "robots": { "requested_url": "https://liadm.com/robots.txt", "observed_at": "2026-09-20T06:25:53.863689+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://liadm.com/llms.txt", "observed_at": "2026-09-20T06:25:53.930062+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 802, "domain": "visma.com", "robots": { "requested_url": "https://visma.com/robots.txt", "observed_at": "2026-09-20T06:25:53.931428+00:00", "status": 200, "final_url": "https://www.visma.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "cc507d8daba62e94e31073a2ed50330509886c19066f059da9271e599652ce43", "bytes_read": 141 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://visma.com/llms.txt", "observed_at": "2026-09-20T06:25:54.408627+00:00", "status": 404, "final_url": "https://www.visma.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "69dd2112c8176002b965614371f47514425802702f37a4dfd3b1690f7cbc764a", "bytes_read": 71849 }, "llms_state": "not_detected", "control": null }, { "rank": 803, "domain": "apnews.com", "robots": { "requested_url": "https://apnews.com/robots.txt", "observed_at": "2026-09-20T06:25:53.950941+00:00", "status": 200, "final_url": "https://apnews.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "cda1601b3be07fd4441f840e2112fe8009e14c736517fcdd908c29cd6644ab35", "bytes_read": 1293 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://apnews.com/llms.txt", "observed_at": "2026-09-20T06:25:54.138322+00:00", "status": 404, "final_url": "https://apnews.com/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": true, "body_sha256": "c4f79f1c98c7e14bd4fcbccd573471bffca7cef14c019f0b9c500a6af7014097", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 804, "domain": "ameblo.jp", "robots": { "requested_url": "https://ameblo.jp/robots.txt", "observed_at": "2026-09-20T06:25:54.006727+00:00", "status": 200, "final_url": "https://ameblo.jp/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "3d6d58d8713dd7d7102e23dbdcde325b26cd2377e46056e65ec6f46a49db4b63", "bytes_read": 1964 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://ameblo.jp/llms.txt", "observed_at": "2026-09-20T06:25:54.888412+00:00", "status": 404, "final_url": "https://ameblo.jp/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "7c63b1811b5346ff1b5fc7bcda34272037f7a616269a776b73f52355560770e3", "bytes_read": 11074 }, "llms_state": "not_detected", "control": null }, { "rank": 805, "domain": "no-ip.com", "robots": { "requested_url": "https://no-ip.com/robots.txt", "observed_at": "2026-09-20T06:25:54.232668+00:00", "status": 200, "final_url": "https://www.noip.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "6745116f8f4c682740d3ddb0ce5e974a9667ab716bce00a45d617402205ea0d6", "bytes_read": 611 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://no-ip.com/llms.txt", "observed_at": "2026-09-20T06:25:55.213566+00:00", "status": 404, "final_url": "https://www.noip.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "2af2c0e7a901e91b60faaeb2493946e19367187d9ab68ee126bd2612bc13d6c0", "bytes_read": 22731 }, "llms_state": "not_detected", "control": null }, { "rank": 806, "domain": "welt.de", "robots": { "requested_url": "https://welt.de/robots.txt", "observed_at": "2026-09-20T06:25:54.434181+00:00", "status": 200, "final_url": "https://www.welt.de:443/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "6eeecdbdd6db728c39bcd6e295a966a4ce7b6e65f86efcd002d395a1295f9fde", "bytes_read": 3501 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://welt.de/llms.txt", "observed_at": "2026-09-20T06:25:55.479376+00:00", "status": 404, "final_url": "https://www.welt.de:443/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "af32831e27c416a4beda49f0dd9c38b3ca021a0d2cc7d39eed69a43aa5d27496", "bytes_read": 418114 }, "llms_state": "not_detected", "control": null }, { "rank": 807, "domain": "mixpanel.com", "robots": { "requested_url": "https://mixpanel.com/robots.txt", "observed_at": "2026-09-20T06:25:54.671155+00:00", "status": 200, "final_url": "https://mixpanel.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1c00eb6fd7dc9b749d107055f45975245cf5e60161115285c9a63dba4361c0f4", "bytes_read": 314 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://mixpanel.com/llms.txt", "observed_at": "2026-09-20T06:25:54.931546+00:00", "status": 404, "final_url": "https://mixpanel.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "63e1a41812b6a7e2f04d2a9d22d1b51001bb9861b076633539e2a4b6c89bdaf3", "bytes_read": 3150 }, "llms_state": "not_detected", "control": null }, { "rank": 808, "domain": "disneyplus.com", "robots": { "requested_url": "https://disneyplus.com/robots.txt", "observed_at": "2026-09-20T06:25:54.784596+00:00", "status": 200, "final_url": "https://www.disneyplus.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "ea1fedfeaf8ce8df678df9ffb629dbbd7b3492aaa57814b5d8390d8dda1719cd", "bytes_read": 2794 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://disneyplus.com/llms.txt", "observed_at": "2026-09-20T06:25:56.099899+00:00", "status": 404, "final_url": "https://www.disneyplus.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "15cfd1be10bb01c19c0127497f292cfcfcea34849a8c40371b820863e450aedb", "bytes_read": 76866 }, "llms_state": "not_detected", "control": null }, { "rank": 809, "domain": "adform.net", "robots": { "requested_url": "https://adform.net/robots.txt", "observed_at": "2026-09-20T06:25:55.135955+00:00", "status": 200, "final_url": "https://adform.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "0bc9bea28ea032363ea814b8ca1a331f557a117e36b4973620f334d4dcc4ab3c", "bytes_read": 139 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://adform.net/llms.txt", "observed_at": "2026-09-20T06:25:55.660608+00:00", "status": 404, "final_url": "https://adform.net/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 }, "llms_state": "not_detected", "control": null }, { "rank": 810, "domain": "people.com", "robots": { "requested_url": "https://people.com/robots.txt", "observed_at": "2026-09-20T06:25:55.759376+00:00", "status": 200, "final_url": "https://people.com/robots.txt", "content_type": "text/plain;charset=utf-8", "truncated": false, "body_sha256": "2d77bbf8969db67edcaa6f0f952cb3b145ae5f91e0628c199fbb2188fae5603a", "bytes_read": 3457 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://people.com/llms.txt", "observed_at": "2026-09-20T06:25:55.982460+00:00", "status": 404, "final_url": "https://people.com/llms.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "1883526f06e2fc9dbc0796ba0d6826ccedcf628f9fd5bcf9ad6be1549612ee50", "bytes_read": 167268 }, "llms_state": "not_detected", "control": null }, { "rank": 811, "domain": "networkadvertising.org", "robots": { "requested_url": "https://networkadvertising.org/robots.txt", "observed_at": "2026-09-20T06:25:56.006142+00:00", "status": 200, "final_url": "https://thenai.org/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "9286b5396e11ed5a6b30dc875b3f1f11f99ad473ceebe0e8fbbef6da535f9e9b", "bytes_read": 184 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://networkadvertising.org/llms.txt", "observed_at": "2026-09-20T06:25:56.486584+00:00", "status": 404, "final_url": "https://thenai.org/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "d8531a1ac331414d357ac014ac8813b93c2c0ea1756542ad8bacb6d352ed4891", "bytes_read": 60 }, "llms_state": "not_detected", "control": null }, { "rank": 812, "domain": "parklogic.com", "robots": { "requested_url": "https://parklogic.com/robots.txt", "observed_at": "2026-09-20T06:25:56.102216+00:00", "status": 200, "final_url": "https://parklogic.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "8d8536bd733d2994b309437238b100550d504499752fd9c8ff414a096de3282f", "bytes_read": 166 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://parklogic.com/llms.txt", "observed_at": "2026-09-20T06:25:56.451367+00:00", "status": 200, "final_url": "https://parklogic.com/", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "ef0543a8d953692f0faf95cf397ac2149b173b7899ca1eaea270623c27bf336f", "bytes_read": 183165 }, "llms_state": "not_detected", "control": null }, { "rank": 813, "domain": "xbox.com", "robots": { "requested_url": "https://xbox.com/robots.txt", "observed_at": "2026-09-20T06:25:56.236210+00:00", "status": 200, "final_url": "https://www.xbox.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "8241af4595ea463c0f1a5b1f4d1e5b6f49624c2a3a02d258b555b359b8e39c10", "bytes_read": 289 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://xbox.com/llms.txt", "observed_at": "2026-09-20T06:25:57.483231+00:00", "status": 404, "final_url": "https://www.xbox.com/en-US/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "8b1924cc0eb720364d96eb39535c53149a3a843147b2cf02a045fe7dae6019ec", "bytes_read": 337535 }, "llms_state": "not_detected", "control": null }, { "rank": 814, "domain": "rakuten.com", "robots": { "requested_url": "https://rakuten.com/robots.txt", "observed_at": "2026-09-20T06:25:56.431352+00:00", "status": 200, "final_url": "https://www.rakuten.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "8bccbc078da0f012742040175fe4dfabddedf46303e466ff0ec112aa8eeaab56", "bytes_read": 2621 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://rakuten.com/llms.txt", "observed_at": "2026-09-20T06:25:57.226030+00:00", "status": 404, "final_url": "http://www.rakuten.com/404", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "not_detected", "control": null }, { "rank": 815, "domain": "prnewswire.com", "robots": { "requested_url": "https://prnewswire.com/robots.txt", "observed_at": "2026-09-20T06:25:56.988200+00:00", "status": 200, "final_url": "https://www.prnewswire.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "e6c21282974bd5b2e9aead4a4e32c0da8a579d4c36775b4b60c785bedd872bc3", "bytes_read": 3353 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://prnewswire.com/llms.txt", "observed_at": "2026-09-20T06:25:57.568012+00:00", "status": 200, "final_url": "https://www.prnewswire.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "f6ef23aef8505d8dd6f24699a9cb71e4d81f5b05474904440dde31b4d965de25", "bytes_read": 2598 }, "llms_state": "candidate", "control": { "requested_url": "https://prnewswire.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:25:57.934003+00:00", "status": 404, "final_url": "https://www.prnewswire.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "55c3b9e7e0453f60ec4b80f5307a21aae0110a0e44e623f0a483ac6e89d66c14", "bytes_read": 185911 } }, { "rank": 816, "domain": "moloco.com", "robots": { "requested_url": "https://moloco.com/robots.txt", "observed_at": "2026-09-20T06:25:57.021317+00:00", "status": 200, "final_url": "https://www.moloco.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "539e1dae066185a0fed710bab10b91c2eae8ebcae516700c20235bc0f4d71f95", "bytes_read": 3372 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://moloco.com/llms.txt", "observed_at": "2026-09-20T06:25:57.582368+00:00", "status": 200, "final_url": "https://www.moloco.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "e7e7a6649a4e6388da419732b2a1d88f62081c4df8042a782f5641d1f8d14af3", "bytes_read": 6777 }, "llms_state": "not_detected", "control": null }, { "rank": 817, "domain": "33across.com", "robots": { "requested_url": "https://33across.com/robots.txt", "observed_at": "2026-09-20T06:25:57.380484+00:00", "status": 403, "final_url": "https://www.33across.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "34bdefe7425c95afcace35b8e054e31564577e728a471fa64ecc7a9bedc9b86a", "bytes_read": 5565 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://33across.com/llms.txt", "observed_at": "2026-09-20T06:25:57.948494+00:00", "status": 403, "final_url": "https://www.33across.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "39fb3b21e9a18395b9b441a462fba84fdef920a2197bc6314851b7176594d0f1", "bytes_read": 5559 }, "llms_state": "unavailable", "control": null }, { "rank": 818, "domain": "cloudsink.net", "robots": { "requested_url": "https://cloudsink.net/robots.txt", "observed_at": "2026-09-20T06:25:57.981343+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cloudsink.net/llms.txt", "observed_at": "2026-09-20T06:25:58.015276+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 819, "domain": "erome.com", "robots": { "requested_url": "https://erome.com/robots.txt", "observed_at": "2026-09-20T06:25:58.015034+00:00", "status": 200, "final_url": "https://www.erome.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "4a64635387cd21687a780cd39ea4895268d06fd31f0adab84b2803ac9740a520", "bytes_read": 31 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://erome.com/llms.txt", "observed_at": "2026-09-20T06:25:58.533980+00:00", "status": 404, "final_url": "https://www.erome.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "1016f313c91e54200e3cad4aafaf1572a18af70bff9327c652bb014240f9c1c1", "bytes_read": 167661 }, "llms_state": "not_detected", "control": null }, { "rank": 820, "domain": "teads.tv", "robots": { "requested_url": "https://teads.tv/robots.txt", "observed_at": "2026-09-20T06:25:58.017623+00:00", "status": 200, "final_url": "https://cdn.teads.tv:443/redirect.html", "content_type": "text/html", "truncated": false, "body_sha256": "e95484810ffbed5388ddfc811bedd19bf3c40c7e9c44df2438cc9c567536f81c", "bytes_read": 8345 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://teads.tv/llms.txt", "observed_at": "2026-09-20T06:25:59.483297+00:00", "status": 200, "final_url": "https://cdn.teads.tv:443/redirect.html", "content_type": "text/html", "truncated": false, "body_sha256": "e95484810ffbed5388ddfc811bedd19bf3c40c7e9c44df2438cc9c567536f81c", "bytes_read": 8345 }, "llms_state": "not_detected", "control": null }, { "rank": 821, "domain": "adjust.com", "robots": { "requested_url": "https://adjust.com/robots.txt", "observed_at": "2026-09-20T06:25:58.083304+00:00", "status": 429, "final_url": "https://www.adjust.com/robots.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "640fe23381c4a8c5b7e07685a934c0665a2acd0504aebf7d8858eb11010ebb0e", "bytes_read": 32183 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://adjust.com/llms.txt", "observed_at": "2026-09-20T06:25:58.587755+00:00", "status": 403, "final_url": "https://www.adjust.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "ec8fcd5198025857f57b1173dad5108aacd30dddc791b98784cb01bfd00143bb", "bytes_read": 59 }, "llms_state": "unavailable", "control": null }, { "rank": 822, "domain": "a-mo.net", "robots": { "requested_url": "https://a-mo.net/robots.txt", "observed_at": "2026-09-20T06:25:58.323091+00:00", "status": 403, "final_url": "https://a-mo.net/robots.txt", "content_type": "application/xml", "truncated": false, "body_sha256": "a824bc7739e226e1b40ea0f8c4e4f4c6f796fc3b4abfa6e9abe3bd119a30d938", "bytes_read": 111 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://a-mo.net/llms.txt", "observed_at": "2026-09-20T06:25:58.628835+00:00", "status": 403, "final_url": "https://a-mo.net/llms.txt", "content_type": "application/xml", "truncated": false, "body_sha256": "a824bc7739e226e1b40ea0f8c4e4f4c6f796fc3b4abfa6e9abe3bd119a30d938", "bytes_read": 111 }, "llms_state": "unavailable", "control": null }, { "rank": 823, "domain": "goskope.com", "robots": { "requested_url": "https://goskope.com/robots.txt", "observed_at": "2026-09-20T06:25:58.851642+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://goskope.com/llms.txt", "observed_at": "2026-09-20T06:25:58.908354+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 824, "domain": "vivo.com.cn", "robots": { "requested_url": "https://vivo.com.cn/robots.txt", "observed_at": "2026-09-20T06:25:58.909793+00:00", "status": 200, "final_url": "https://www.vivo.com.cn/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "ba424308ab36d2866713f27532ee943abcb3947004f23cc4ed59c03f6110b25b", "bytes_read": 602 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://vivo.com.cn/llms.txt", "observed_at": "2026-09-20T06:26:02.468384+00:00", "status": 404, "final_url": "https://www.vivo.com.cn/llms.txt", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "not_detected", "control": null }, { "rank": 825, "domain": "zoho.com", "robots": { "requested_url": "https://zoho.com/robots.txt", "observed_at": "2026-09-20T06:25:59.015710+00:00", "status": 200, "final_url": "https://www.zoho.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "95ce74091f7d71aa14c80114e6d1686c1b35472e994f9cdf2553d0447f2e2e1d", "bytes_read": 4674 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://zoho.com/llms.txt", "observed_at": "2026-09-20T06:25:59.767190+00:00", "status": 404, "final_url": "https://www.zoho.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "ce334c1b3ef3f077d56f08c66fff34ef35d3966dcf829497f83da9365323deda", "bytes_read": 4166 }, "llms_state": "not_detected", "control": null }, { "rank": 826, "domain": "discordapp.com", "robots": { "requested_url": "https://discordapp.com/robots.txt", "observed_at": "2026-09-20T06:25:59.050615+00:00", "status": 200, "final_url": "https://discord.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "fd74e7f8abd01d90ce80462af73d0896ed098dd0a61b20f24bf1137dd37979a5", "bytes_read": 1774 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://discordapp.com/llms.txt", "observed_at": "2026-09-20T06:25:59.421333+00:00", "status": 404, "final_url": "https://discord.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "d493a92be8d0c39e03ae609811571012543f014c226babf4e43f223b09c46ca5", "bytes_read": 44855 }, "llms_state": "not_detected", "control": null }, { "rank": 827, "domain": "deepl.com", "robots": { "requested_url": "https://deepl.com/robots.txt", "observed_at": "2026-09-20T06:25:59.051221+00:00", "status": 200, "final_url": "https://www.deepl.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "175afdacbfd99bd6af1de23dea2d150f5f0176f6b2fdb219c3813bee9c1944ce", "bytes_read": 1722 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://deepl.com/llms.txt", "observed_at": "2026-09-20T06:25:59.570832+00:00", "status": 404, "final_url": "https://www.deepl.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "3f78aad784589ca1a447ee91b50e7a799fb6a4daa97b9382c8ed7a30f4784d1c", "bytes_read": 58747 }, "llms_state": "not_detected", "control": null }, { "rank": 828, "domain": "unpkg.com", "robots": { "requested_url": "https://unpkg.com/robots.txt", "observed_at": "2026-09-20T06:25:59.862792+00:00", "status": 200, "final_url": "https://unpkg.com/robots.txt@1.1.0/index.js", "content_type": "text/javascript; charset=utf-8", "truncated": false, "body_sha256": "27248f02bc49b752201eaa2eb5ba3b20ed177bcea5a821db7beb66b0dba80985", "bytes_read": 733 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://unpkg.com/llms.txt", "observed_at": "2026-09-20T06:26:00.305252+00:00", "status": 404, "final_url": "https://unpkg.com/llms.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "175d7a0603bcf6fdfb182668fe4c0db9e80d54644dfb3dca9ccaa23b66f37fdc", "bytes_read": 27 }, "llms_state": "not_detected", "control": null }, { "rank": 829, "domain": "grab.co", "robots": { "requested_url": "https://grab.co/robots.txt", "observed_at": "2026-09-20T06:25:59.970830+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://grab.co/llms.txt", "observed_at": "2026-09-20T06:26:00.045322+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 830, "domain": "liftoff.io", "robots": { "requested_url": "https://liftoff.io/robots.txt", "observed_at": "2026-09-20T06:26:00.047387+00:00", "status": 200, "final_url": "https://liftoff.ai/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "6da154a70851f9f844cdf82ac5c59bcc8952b5672026e967e746624d6c852c52", "bytes_read": 92 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://liftoff.io/llms.txt", "observed_at": "2026-09-20T06:26:00.732385+00:00", "status": 404, "final_url": "https://liftoff.ai/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "d8531a1ac331414d357ac014ac8813b93c2c0ea1756542ad8bacb6d352ed4891", "bytes_read": 60 }, "llms_state": "not_detected", "control": null }, { "rank": 831, "domain": "ryanair.com", "robots": { "requested_url": "https://ryanair.com/robots.txt", "observed_at": "2026-09-20T06:26:00.063007+00:00", "status": 200, "final_url": "https://www.ryanair.com/us/en", "content_type": "text/html", "truncated": false, "body_sha256": "8330bd63ab82c03d698b4a714307e5db1b87ebe138b09f2041b5c6107a2bb9d6", "bytes_read": 4742 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ryanair.com/llms.txt", "observed_at": "2026-09-20T06:26:00.724020+00:00", "status": 200, "final_url": "https://www.ryanair.com/us/en", "content_type": "text/html", "truncated": false, "body_sha256": "8330bd63ab82c03d698b4a714307e5db1b87ebe138b09f2041b5c6107a2bb9d6", "bytes_read": 4742 }, "llms_state": "not_detected", "control": null }, { "rank": 832, "domain": "wb.ru", "robots": { "requested_url": "https://wb.ru/robots.txt", "observed_at": "2026-09-20T06:26:00.267614+00:00", "status": 498, "final_url": "https://www.wildberries.ru/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "9c80e22891da80ed8b62642ae5d0cb07e8b79cfa766c4d3480e2d1b08a6ffb53", "bytes_read": 1136 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://wb.ru/llms.txt", "observed_at": "2026-09-20T06:26:01.844568+00:00", "status": 498, "final_url": "https://www.wildberries.ru/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "3393f3b91403f26475b6892e87a300dc6ed77cdedefef0e8b7c12fadb19471c6", "bytes_read": 1136 }, "llms_state": "unavailable", "control": null }, { "rank": 833, "domain": "blackberry.com", "robots": { "requested_url": "https://blackberry.com/robots.txt", "observed_at": "2026-09-20T06:26:00.280381+00:00", "status": 200, "final_url": "https://www.blackberry.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "4b69050544d349d6117cddf93e082722cb30a34fe48475ec7bf637dbb06cbf85", "bytes_read": 910 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://blackberry.com/llms.txt", "observed_at": "2026-09-20T06:26:00.951326+00:00", "status": 404, "final_url": "https://www.blackberry.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "899cfa7103745491e75188376c5e93097619c25d3d289df746705d5aed4c95e3", "bytes_read": 98 }, "llms_state": "not_detected", "control": null }, { "rank": 834, "domain": "usps.com", "robots": { "requested_url": "https://usps.com/robots.txt", "observed_at": "2026-09-20T06:26:00.585710+00:00", "status": 200, "final_url": "https://www.usps.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1a0a3f1355a50c5f7e4519458957d4c765c4c29a880ac14b8f1fe04f52c73629", "bytes_read": 435 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://usps.com/llms.txt", "observed_at": "2026-09-20T06:26:01.963423+00:00", "status": 404, "final_url": "https://www.usps.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "5f12b00cf73ac33b5a4b337a6e4746e4cb0f280bb10db088f59451136bedbe04", "bytes_read": 54801 }, "llms_state": "not_detected", "control": null }, { "rank": 835, "domain": "academia.edu", "robots": { "requested_url": "https://academia.edu/robots.txt", "observed_at": "2026-09-20T06:26:01.230929+00:00", "status": 200, "final_url": "https://www.academia.edu/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "ee31c8913ef20ad3b94425af4546d20f2bd1c7494281b0bd57757214d68bd826", "bytes_read": 57366 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://academia.edu/llms.txt", "observed_at": "2026-09-20T06:26:02.134864+00:00", "status": 404, "final_url": "https://www.academia.edu/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "f18aa6f839900a746c76ba076a9afd1e32252c74b69755288b26ced878d1122f", "bytes_read": 7936 }, "llms_state": "not_detected", "control": null }, { "rank": 836, "domain": "turn.com", "robots": { "requested_url": "https://turn.com/robots.txt", "observed_at": "2026-09-20T06:26:01.300469+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://turn.com/llms.txt", "observed_at": "2026-09-20T06:26:01.377460+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 837, "domain": "odoo.com", "robots": { "requested_url": "https://odoo.com/robots.txt", "observed_at": "2026-09-20T06:26:01.380002+00:00", "status": 200, "final_url": "https://www.odoo.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "147ef54a7f2e347d0d5d5ca6d95310300e24ef80c6998b0b098c971289997972", "bytes_read": 1571 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://odoo.com/llms.txt", "observed_at": "2026-09-20T06:26:02.463870+00:00", "status": 404, "final_url": "https://www.odoo.com/llms.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "6972539bb7f807002cea4358fce943884f23208d2bb4dea1fd56e9ce5a5cd7c8", "bytes_read": 66928 }, "llms_state": "not_detected", "control": null }, { "rank": 838, "domain": "lefigaro.fr", "robots": { "requested_url": "https://lefigaro.fr/robots.txt", "observed_at": "2026-09-20T06:26:01.706864+00:00", "status": 200, "final_url": "https://www.lefigaro.fr/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "f6127a8af3818c9eddbeeabdf9dc8c496ddaf05baa2982b8d97ef56f8424a99a", "bytes_read": 12666 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://lefigaro.fr/llms.txt", "observed_at": "2026-09-20T06:26:02.289470+00:00", "status": 404, "final_url": "https://www.lefigaro.fr/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "829b0ea89f976df28a2ffcd9cfbf8413dec849e583179cabf9ca16895659aaec", "bytes_read": 173418 }, "llms_state": "not_detected", "control": null }, { "rank": 839, "domain": "daum.net", "robots": { "requested_url": "https://daum.net/robots.txt", "observed_at": "2026-09-20T06:26:02.562310+00:00", "status": 200, "final_url": "https://daum.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "3bc0353303e9a681e0d57b23ab9e4ebf63f94d35df5cd497d7fe634c97ede919", "bytes_read": 167 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://daum.net/llms.txt", "observed_at": "2026-09-20T06:26:03.327894+00:00", "status": 200, "final_url": "https://www.daum.net/", "content_type": "text/html", "truncated": true, "body_sha256": "3a4ebc7599c51889521217338413760acd12d795d46f9149d6ab8396fc3dc650", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 840, "domain": "me.com", "robots": { "requested_url": "https://me.com/robots.txt", "observed_at": "2026-09-20T06:26:02.877601+00:00", "status": 200, "final_url": "https://www.icloud.com/", "content_type": "text/html", "truncated": false, "body_sha256": "95437aea034183c268e16ee8f609477685d83eeef8a5b6d660befd636a44a9b0", "bytes_read": 31302 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://me.com/llms.txt", "observed_at": "2026-09-20T06:26:03.400678+00:00", "status": 200, "final_url": "https://www.icloud.com/", "content_type": "text/html", "truncated": false, "body_sha256": "95437aea034183c268e16ee8f609477685d83eeef8a5b6d660befd636a44a9b0", "bytes_read": 31302 }, "llms_state": "not_detected", "control": null }, { "rank": 841, "domain": "orderbox-dns.com", "robots": { "requested_url": "https://orderbox-dns.com/robots.txt", "observed_at": "2026-09-20T06:26:02.962787+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://orderbox-dns.com/llms.txt", "observed_at": "2026-09-20T06:26:03.043737+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 842, "domain": "pccc.com", "robots": { "requested_url": "https://pccc.com/robots.txt", "observed_at": "2026-09-20T06:26:03.046133+00:00", "status": 200, "final_url": "https://pccc.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "ee5cf1dce9b8ebaa7a825d00c21f07b9dad406370eb394b0205eda4665f0e769", "bytes_read": 35 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://pccc.com/llms.txt", "observed_at": "2026-09-20T06:26:03.323204+00:00", "status": 404, "final_url": "https://pccc.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "10958950b3da628cd9aaec177333a08e39b11a322d62d6ccae72b3518282c02f", "bytes_read": 43609 }, "llms_state": "not_detected", "control": null }, { "rank": 843, "domain": "a-msedge.net", "robots": { "requested_url": "https://a-msedge.net/robots.txt", "observed_at": "2026-09-20T06:26:03.245527+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://a-msedge.net/llms.txt", "observed_at": "2026-09-20T06:26:03.288383+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 844, "domain": "focus.de", "robots": { "requested_url": "https://focus.de/robots.txt", "observed_at": "2026-09-20T06:26:03.291043+00:00", "status": 403, "final_url": "https://focus.de/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "f86bc302cc861de4b250cba92e581701a50a0493423246179b9b7f9d446490f6", "bytes_read": 372 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://focus.de/llms.txt", "observed_at": "2026-09-20T06:26:03.483626+00:00", "status": 403, "final_url": "https://focus.de/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "379d9529ea56aeb9a26d1c04afde4de48bffce1284e7602d3440ffd2bf7753ff", "bytes_read": 370 }, "llms_state": "unavailable", "control": null }, { "rank": 845, "domain": "opera-api2.com", "robots": { "requested_url": "https://opera-api2.com/robots.txt", "observed_at": "2026-09-20T06:26:03.600214+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://opera-api2.com/llms.txt", "observed_at": "2026-09-20T06:26:13.645112+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 846, "domain": "usda.gov", "robots": { "requested_url": "https://usda.gov/robots.txt", "observed_at": "2026-09-20T06:26:03.736612+00:00", "status": 403, "final_url": "https://www.usda.gov/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "0967ccc3027e8d6eb6ef0c251a9ee9634b27f94b0a9e03630686bab490271e8a", "bytes_read": 380 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://usda.gov/llms.txt", "observed_at": "2026-09-20T06:26:04.479683+00:00", "status": 403, "final_url": "https://www.usda.gov/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "8c79f42602c55febe802ba3408eebf7fa15231c0b002d0b513a67537b3675a57", "bytes_read": 378 }, "llms_state": "unavailable", "control": null }, { "rank": 847, "domain": "tbank.ru", "robots": { "requested_url": "https://tbank.ru/robots.txt", "observed_at": "2026-09-20T06:26:03.869626+00:00", "status": 200, "final_url": "https://www.tbank.ru/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "6c69c9942a2f9d8cb585dbdfa5e26ec9fc26ac6e74b8bd06f749d5fc7fbc079f", "bytes_read": 30474 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://tbank.ru/llms.txt", "observed_at": "2026-09-20T06:26:05.335978+00:00", "status": 404, "final_url": "https://www.tbank.ru/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "fa202a2464855e6b2fcca0f0b1a33603847a4751f6dc511ee69b133947f9d61f", "bytes_read": 9780 }, "llms_state": "not_detected", "control": null }, { "rank": 848, "domain": "agoda.com", "robots": { "requested_url": "https://agoda.com/robots.txt", "observed_at": "2026-09-20T06:26:03.883598+00:00", "status": 200, "final_url": "https://www.agoda.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "c47b9fe96c3eb0eb5298b6787dea4b21f97f1616831d9399bf7a7525b68af873", "bytes_read": 5086 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://agoda.com/llms.txt", "observed_at": "2026-09-20T06:26:05.196417+00:00", "status": 404, "final_url": "https://www.agoda.com/pagenotfound.html", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "e6a620aa6f29d5cd4377671ab794c40699b97c359a2e9a61309f60892c3dc1bf", "bytes_read": 85813 }, "llms_state": "not_detected", "control": null }, { "rank": 849, "domain": "amazon.com.au", "robots": { "requested_url": "https://amazon.com.au/robots.txt", "observed_at": "2026-09-20T06:26:05.024657+00:00", "status": 200, "final_url": "https://www.amazon.com.au/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "6110089f37926ba4b9de6c53ace2f9e70a2b4441d9270beaca45909d7b4b1e51", "bytes_read": 6904 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://amazon.com.au/llms.txt", "observed_at": "2026-09-20T06:26:06.486854+00:00", "status": 404, "final_url": "https://www.amazon.com.au/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "0bc24a788276fbf3c32bbb9d8438e561cdc82febd066bd92ff16004b9fa25135", "bytes_read": 2166 }, "llms_state": "not_detected", "control": null }, { "rank": 850, "domain": "lgtvcommon.com", "robots": { "requested_url": "https://lgtvcommon.com/robots.txt", "observed_at": "2026-09-20T06:26:05.922964+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://lgtvcommon.com/llms.txt", "observed_at": "2026-09-20T06:26:05.988096+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 851, "domain": "theverge.com", "robots": { "requested_url": "https://theverge.com/robots.txt", "observed_at": "2026-09-20T06:26:05.989618+00:00", "status": 200, "final_url": "https://theverge.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "f74664eb7fc3cbdfafd82167d05cb7b9596142a73509a3ce6f5ea83a0d1046a8", "bytes_read": 5650 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://theverge.com/llms.txt", "observed_at": "2026-09-20T06:26:06.200996+00:00", "status": 404, "final_url": "https://www.theverge.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "c8a3b5e53d43b3d14e27f6c50aace4fd449e9b444f458d3f25bee419c43a983a", "bytes_read": 99214 }, "llms_state": "not_detected", "control": null }, { "rank": 852, "domain": "onlyfans.com", "robots": { "requested_url": "https://onlyfans.com/robots.txt", "observed_at": "2026-09-20T06:26:06.339382+00:00", "status": 200, "final_url": "https://onlyfans.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "4be2257546d4ea00879e81c2e88d9ac3739cf6fbd6cadc7e18990d0adad16536", "bytes_read": 162 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://onlyfans.com/llms.txt", "observed_at": "2026-09-20T06:26:06.555679+00:00", "status": 200, "final_url": "https://onlyfans.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "b5a40222e083c590ecc67ec77acc3676aa3ed7fdfeba8bf9e169d70534557fd1", "bytes_read": 17669 }, "llms_state": "not_detected", "control": null }, { "rank": 853, "domain": "eeroup.com", "robots": { "requested_url": "https://eeroup.com/robots.txt", "observed_at": "2026-09-20T06:26:06.487626+00:00", "status": 404, "final_url": "https://eeroup.com/robots.txt", "content_type": "application/xml", "truncated": false, "body_sha256": "f76a284dd0b6827a74f08b6833e7a01eee8967b0c657487878a12a9f3b5e8c4a", "bytes_read": 301 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://eeroup.com/llms.txt", "observed_at": "2026-09-20T06:26:06.804277+00:00", "status": 404, "final_url": "https://eeroup.com/llms.txt", "content_type": "application/xml", "truncated": false, "body_sha256": "784df2a25de88d5e554157c3eaa3cfaa4141150f5bc5e0c5f12f4727cac17ddc", "bytes_read": 299 }, "llms_state": "not_detected", "control": null }, { "rank": 854, "domain": "discogs.com", "robots": { "requested_url": "https://discogs.com/robots.txt", "observed_at": "2026-09-20T06:26:06.598335+00:00", "status": 403, "final_url": "https://www.discogs.com:443/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "4547f0f7361f58346dcbf6d42fb796a76b35d3db008c198388a15864350d8a69", "bytes_read": 5627 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://discogs.com/llms.txt", "observed_at": "2026-09-20T06:26:07.380155+00:00", "status": 403, "final_url": "https://www.discogs.com:443/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "5f745cfd3718c97d4ec85d56b0efecbef6a85ef61e04726bd2656dcc848cb69f", "bytes_read": 5621 }, "llms_state": "unavailable", "control": null }, { "rank": 855, "domain": "nintendo.net", "robots": { "requested_url": "https://nintendo.net/robots.txt", "observed_at": "2026-09-20T06:26:06.628480+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://nintendo.net/llms.txt", "observed_at": "2026-09-20T06:26:06.662922+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 856, "domain": "jetbrains.com", "robots": { "requested_url": "https://jetbrains.com/robots.txt", "observed_at": "2026-09-20T06:26:06.664828+00:00", "status": 200, "final_url": "https://www.jetbrains.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "9372e147d5d59b2578c0753fd07e7567f6db8f12b2aa3604248f7d7b13a480e9", "bytes_read": 1760 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://jetbrains.com/llms.txt", "observed_at": "2026-09-20T06:26:07.173843+00:00", "status": 404, "final_url": "https://www.jetbrains.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "685ef5cd2f442d06e113d094973412dbf900bf2a403f0074a964d1569249284b", "bytes_read": 44030 }, "llms_state": "not_detected", "control": null }, { "rank": 857, "domain": "onetag-sys.com", "robots": { "requested_url": "https://onetag-sys.com/robots.txt", "observed_at": "2026-09-20T06:26:06.726198+00:00", "status": 200, "final_url": "https://onetag-sys.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "8636b148f7f30b577172d0bea8065e399cad8c51e01ac50e275272d184c2b74b", "bytes_read": 923 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://onetag-sys.com/llms.txt", "observed_at": "2026-09-20T06:26:07.026703+00:00", "status": 200, "final_url": "https://onetag-sys.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "8636b148f7f30b577172d0bea8065e399cad8c51e01ac50e275272d184c2b74b", "bytes_read": 923 }, "llms_state": "not_detected", "control": null }, { "rank": 858, "domain": "zoom.com", "robots": { "requested_url": "https://zoom.com/robots.txt", "observed_at": "2026-09-20T06:26:07.049275+00:00", "status": 200, "final_url": "https://zoom.us/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "b3ed9f53f38c480b046b4d5a648aa7a879c6720957301506dc282b32fb2f5593", "bytes_read": 2005 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://zoom.com/llms.txt", "observed_at": "2026-09-20T06:26:07.647988+00:00", "status": 404, "final_url": "https://zoom.us/llms.txt", "content_type": "text/html;charset=utf-8", "truncated": true, "body_sha256": "11108beadefc560a0173c18b6bc4553f5c97dbfcb3f9aebec0a815a6ee811f63", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 859, "domain": "pendo.io", "robots": { "requested_url": "https://pendo.io/robots.txt", "observed_at": "2026-09-20T06:26:07.254730+00:00", "status": 200, "final_url": "https://www.pendo.io:443/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "f28cfbbe8d92d65cf039be99db1c81283aaaa67bd2c4a6d12183b8ac6a58d2cb", "bytes_read": 646 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://pendo.io/llms.txt", "observed_at": "2026-09-20T06:26:07.899388+00:00", "status": 200, "final_url": "https://www.pendo.io:443/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "e8b05729924d97878ac85e2f7d4f25a29ea5b128d9d2bf548ea1c69d42dc4028", "bytes_read": 20407 }, "llms_state": "candidate", "control": { "requested_url": "https://pendo.io/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:26:08.309241+00:00", "status": 404, "final_url": "https://www.pendo.io:443/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "117e817971298f58ad16f045097cdb511f1079616bb661962ff505e205e7d974", "bytes_read": 26079 } }, { "rank": 860, "domain": "ebay.de", "robots": { "requested_url": "https://ebay.de/robots.txt", "observed_at": "2026-09-20T06:26:07.378935+00:00", "status": 200, "final_url": "https://www.ebay.de/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "f1dd24674d48ccc3dd7826f7926c5483995467f553f6c37638b5977139c7c58b", "bytes_read": 15980 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://ebay.de/llms.txt", "observed_at": "2026-09-20T06:26:15.813654+00:00", "status": 404, "final_url": "https://www.ebay.de/n/error", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "9e5ce1614048c695b10dd2a5710e18b0a2f597e042b7da9929fcd5f0801a14da", "bytes_read": 79922 }, "llms_state": "not_detected", "control": null }, { "rank": 861, "domain": "amazon.com.br", "robots": { "requested_url": "https://amazon.com.br/robots.txt", "observed_at": "2026-09-20T06:26:07.697844+00:00", "status": 200, "final_url": "https://www.amazon.com.br/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1dc9dde302148f438f91498ab09a577371d1f8c188b0bb4cf156211163a28424", "bytes_read": 5764 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://amazon.com.br/llms.txt", "observed_at": "2026-09-20T06:26:08.542669+00:00", "status": 404, "final_url": "https://www.amazon.com.br/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "525a6a099caab72eff22e32a6e85fe428af5faf45cb01d044d7023630c637c23", "bytes_read": 2366 }, "llms_state": "not_detected", "control": null }, { "rank": 862, "domain": "figma.com", "robots": { "requested_url": "https://figma.com/robots.txt", "observed_at": "2026-09-20T06:26:07.726367+00:00", "status": 200, "final_url": "https://www.figma.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1ea35e282ac95ec3ce6530124bb1007d5065b2bd5c51c179406432295815224a", "bytes_read": 386687 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/$" ], [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://figma.com/llms.txt", "observed_at": "2026-09-20T06:26:11.389114+00:00", "status": 404, "final_url": "https://www.figma.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "5a642be6a1b044e2ca30ec3125ab5315b0ebf4e171ca7448bceb9475f198c370", "bytes_read": 9586 }, "llms_state": "not_detected", "control": null }, { "rank": 863, "domain": "msftauth.net", "robots": { "requested_url": "https://msftauth.net/robots.txt", "observed_at": "2026-09-20T06:26:07.991196+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://msftauth.net/llms.txt", "observed_at": "2026-09-20T06:26:08.030578+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 864, "domain": "livejournal.com", "robots": { "requested_url": "https://livejournal.com/robots.txt", "observed_at": "2026-09-20T06:26:08.032691+00:00", "status": 200, "final_url": "https://www.livejournal.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "2c2788645a1587a0c040fc420256414574a1634e91374291f9326168c8655e25", "bytes_read": 2643 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://livejournal.com/llms.txt", "observed_at": "2026-09-20T06:26:09.490324+00:00", "status": 404, "final_url": "https://www.livejournal.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "51b468d9b994d9123f027f82a0a84e29ab05e959c9c4a2d871ce029cada3652b", "bytes_read": 4704 }, "llms_state": "not_detected", "control": null }, { "rank": 865, "domain": "ebay.co.uk", "robots": { "requested_url": "https://ebay.co.uk/robots.txt", "observed_at": "2026-09-20T06:26:08.215398+00:00", "status": 200, "final_url": "https://www.ebay.co.uk/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1984ae3c90c886ba59a8606a158292f1d37aecbcd8c98a3cb9bd1f852e04084f", "bytes_read": 20258 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://ebay.co.uk/llms.txt", "observed_at": "2026-09-20T06:26:12.462310+00:00", "status": 404, "final_url": "https://www.ebay.co.uk/n/error", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "75e61883dd04aa9858a787a70e2c19053f765403ba990ee38f2df9923f07025d", "bytes_read": 76893 }, "llms_state": "not_detected", "control": null }, { "rank": 866, "domain": "pexels.com", "robots": { "requested_url": "https://pexels.com/robots.txt", "observed_at": "2026-09-20T06:26:08.915352+00:00", "status": 200, "final_url": "https://www.pexels.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "00eeeed2682d131755be248df2eafed604ef7f2d9740f9f92bbfc2eb6bea0788", "bytes_read": 11931 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://pexels.com/llms.txt", "observed_at": "2026-09-20T06:26:09.705822+00:00", "status": 403, "final_url": "https://www.pexels.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "55722244c4f9fcc9ae323c2c13b52205b22547dbe166654f9a0485d5314f9fab", "bytes_read": 5557 }, "llms_state": "unavailable", "control": null }, { "rank": 867, "domain": "adgrx.com", "robots": { "requested_url": "https://adgrx.com/robots.txt", "observed_at": "2026-09-20T06:26:08.965069+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://adgrx.com/llms.txt", "observed_at": "2026-09-20T06:26:09.035624+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 868, "domain": "oxylabs.io", "robots": { "requested_url": "https://oxylabs.io/robots.txt", "observed_at": "2026-09-20T06:26:09.051267+00:00", "status": 200, "final_url": "https://oxylabs.io/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "4eff0c58129f6dca07a24e68d5a484c73f9f8353a576e1ca8704ce246476c82f", "bytes_read": 121 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://oxylabs.io/llms.txt", "observed_at": "2026-09-20T06:26:09.581437+00:00", "status": 200, "final_url": "https://oxylabs.io/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "3d38efd3a06b47993beedaedc08a2fc5ae3969c30df64080eab6a12cd3bba63f", "bytes_read": 4354 }, "llms_state": "candidate", "control": { "requested_url": "https://oxylabs.io/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:26:10.178036+00:00", "status": 404, "final_url": "https://oxylabs.io/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "fa031021a4223846c168d37c538204ae1ef25d9f95521feddcfe83b8f5b3500c", "bytes_read": 313032 } }, { "rank": 869, "domain": "yellowblue.io", "robots": { "requested_url": "https://yellowblue.io/robots.txt", "observed_at": "2026-09-20T06:26:09.203318+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://yellowblue.io/llms.txt", "observed_at": "2026-09-20T06:26:09.260349+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 870, "domain": "chess.com", "robots": { "requested_url": "https://chess.com/robots.txt", "observed_at": "2026-09-20T06:26:09.268105+00:00", "status": 200, "final_url": "https://www.chess.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "82e84920deb356b876dc5c5609ed8e827f0f3d4c56087f4d2ba590014b865453", "bytes_read": 2396 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://chess.com/llms.txt", "observed_at": "2026-09-20T06:26:09.873464+00:00", "status": 404, "final_url": "https://www.chess.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "fd6644efc523cf1738b319a5c77f17b54bb8fb5910874f9cf6c459e75a8e0316", "bytes_read": 48869 }, "llms_state": "not_detected", "control": null }, { "rank": 871, "domain": "att.net", "robots": { "requested_url": "https://att.net/robots.txt", "observed_at": "2026-09-20T06:26:10.178274+00:00", "status": 200, "final_url": "https://www.att.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "a804d606801527a02f698717fb182b85c1fb7fc638a2f77873610b91f29d7523", "bytes_read": 250 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://att.net/llms.txt", "observed_at": "2026-09-20T06:26:11.276503+00:00", "status": null, "error": "TimeoutError" }, "llms_state": "unavailable", "control": null }, { "rank": 872, "domain": "pikabu.ru", "robots": { "requested_url": "https://pikabu.ru/robots.txt", "observed_at": "2026-09-20T06:26:10.513798+00:00", "status": 403, "final_url": "https://pikabu.ru/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "d48782a6d126baadc69beef3f37a643f10798916bc8e4460a3deacc36fbb8b51", "bytes_read": 902 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://pikabu.ru/llms.txt", "observed_at": "2026-09-20T06:26:10.801440+00:00", "status": 403, "final_url": "https://pikabu.ru/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "d48782a6d126baadc69beef3f37a643f10798916bc8e4460a3deacc36fbb8b51", "bytes_read": 902 }, "llms_state": "unavailable", "control": null }, { "rank": 873, "domain": "xvideos.com", "robots": { "requested_url": "https://xvideos.com/robots.txt", "observed_at": "2026-09-20T06:26:10.801916+00:00", "status": 200, "final_url": "https://xvideos.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "8c71185fec7461ed3ac2c6ba90b1ea39641c06fed9359754235a8f550db8f60b", "bytes_read": 61 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://xvideos.com/llms.txt", "observed_at": "2026-09-20T06:26:11.359756+00:00", "status": 404, "final_url": "https://www.xvideos.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "4ecea1548a22d458301240a8fdf233fcdd47e7269b392d77e5fcda6f7ae591ff", "bytes_read": 38968 }, "llms_state": "not_detected", "control": null }, { "rank": 874, "domain": "it.com", "robots": { "requested_url": "https://it.com/robots.txt", "observed_at": "2026-09-20T06:26:10.983980+00:00", "status": 404, "final_url": "https://it.com/robots.txt", "content_type": "application/json", "truncated": false, "body_sha256": "4adf46fd880f2694cd1a33808d59221b9b4b890e4f1a22504073f24402255148", "bytes_read": 99 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://it.com/llms.txt", "observed_at": "2026-09-20T06:26:11.551270+00:00", "status": 404, "final_url": "https://it.com/llms.txt", "content_type": "application/json", "truncated": false, "body_sha256": "f2289875bed772a019b7a337a15a9e30aeb909cd62567553778f28911f834b5c", "bytes_read": 97 }, "llms_state": "not_detected", "control": null }, { "rank": 875, "domain": "synology.com", "robots": { "requested_url": "https://synology.com/robots.txt", "observed_at": "2026-09-20T06:26:11.198172+00:00", "status": 200, "final_url": "https://www.synology.com:443/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "a3e6632f8256685265d8d1243ce9ed55c801059e1a452d70df4ac9b4acdb4b44", "bytes_read": 127 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://synology.com/llms.txt", "observed_at": "2026-09-20T06:26:12.493349+00:00", "status": 200, "final_url": "https://www.synology.com/en-global", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "8d392b207c4e6a0650ae6d8891e1653f7f033086c09c137c7be2d0beee1d794e", "bytes_read": 173443 }, "llms_state": "not_detected", "control": null }, { "rank": 876, "domain": "appspot.com", "robots": { "requested_url": "https://appspot.com/robots.txt", "observed_at": "2026-09-20T06:26:11.863494+00:00", "status": 200, "final_url": "https://accounts.google.com/v3/signin/identifier?continue=https://console.cloud.google.com/appengine&followup=https://console.cloud.google.com/appengine&osid=1&passive=1209600&service=cloudconsole&flowName=WebLiteSignIn&flowEntry=ServiceLogin&dsh=S-31294531:1789885572622428", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "7f27fb7c10fb7ce47c178555a45507fe46fe828b68eedaa297714504b55ff4a8", "bytes_read": 512001 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://appspot.com/llms.txt", "observed_at": "2026-09-20T06:26:13.060462+00:00", "status": 200, "final_url": "https://accounts.google.com/v3/signin/identifier?continue=https://console.cloud.google.com/appengine&followup=https://console.cloud.google.com/appengine&osid=1&passive=1209600&service=cloudconsole&flowName=WebLiteSignIn&flowEntry=ServiceLogin&dsh=S-31294531:1789885573612287", "content_type": "text/html; charset=utf-8", "truncated": true, "body_sha256": "c26377fafb3a0a399556a4be45210ae7fa8816836b80e760c5d3cfbb95a89417", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 877, "domain": "disqus.com", "robots": { "requested_url": "https://disqus.com/robots.txt", "observed_at": "2026-09-20T06:26:12.005705+00:00", "status": 200, "final_url": "https://disqus.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "0d5e718f2797bc47ee2f524ee817fae2c2c068145d99c896a0058acac1180bef", "bytes_read": 49 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://disqus.com/llms.txt", "observed_at": "2026-09-20T06:26:12.155712+00:00", "status": 404, "final_url": "https://disqus.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "00dae11fbce45c7bfab967dc6548f174a6c74e19b5cb37187053ce0271ad4338", "bytes_read": 5108 }, "llms_state": "not_detected", "control": null }, { "rank": 878, "domain": "life360.com", "robots": { "requested_url": "https://life360.com/robots.txt", "observed_at": "2026-09-20T06:26:12.148192+00:00", "status": 200, "final_url": "https://www.life360.com:443/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "4fd64c530ff3af3c9feab9b9b2690e7c08cc3920d8b3a9da8e4ce3f2e676e798", "bytes_read": 2999 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://life360.com/llms.txt", "observed_at": "2026-09-20T06:26:12.911626+00:00", "status": 200, "final_url": "https://www.life360.com:443/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "41dab8c31edc92230e85254b2b8d3f49e2147c0de0bd5eb83132d36751301797", "bytes_read": 3884 }, "llms_state": "candidate", "control": { "requested_url": "https://life360.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:26:13.455226+00:00", "status": 404, "final_url": "https://www.life360.com:443/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "97a2528f2d49d5ddabdeb816e238263c9c2b91421f381ec14d18b98500a2f02f", "bytes_read": 113802 } }, { "rank": 879, "domain": "amagi.tv", "robots": { "requested_url": "https://amagi.tv/robots.txt", "observed_at": "2026-09-20T06:26:12.166528+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://amagi.tv/llms.txt", "observed_at": "2026-09-20T06:26:22.483244+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 880, "domain": "fbsbx.com", "robots": { "requested_url": "https://fbsbx.com/robots.txt", "observed_at": "2026-09-20T06:26:12.290543+00:00", "status": 404, "final_url": "https://fbsbx.com/robots.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "5a5e6fcc42e2ddddb4b338526b9b600b008dc3a7c2aee42140fc48cbef6e5de3", "bytes_read": 126 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://fbsbx.com/llms.txt", "observed_at": "2026-09-20T06:26:12.489992+00:00", "status": 404, "final_url": "https://fbsbx.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "5a5e6fcc42e2ddddb4b338526b9b600b008dc3a7c2aee42140fc48cbef6e5de3", "bytes_read": 126 }, "llms_state": "not_detected", "control": null }, { "rank": 881, "domain": "hbr.org", "robots": { "requested_url": "https://hbr.org/robots.txt", "observed_at": "2026-09-20T06:26:12.617076+00:00", "status": 200, "final_url": "https://hbr.org/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "793e323f3d6c3a728741a62304f4a4e808c6c146aa57a22de3d3c044ba073b27", "bytes_read": 1016 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://hbr.org/llms.txt", "observed_at": "2026-09-20T06:26:12.946966+00:00", "status": 404, "final_url": "https://hbr.org/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "d508f77d3c146c49abdcfa41f3ce67e898d9678c01423f423bbc10e9a8d8c5b3", "bytes_read": 59585 }, "llms_state": "not_detected", "control": null }, { "rank": 882, "domain": "lg.com", "robots": { "requested_url": "https://lg.com/robots.txt", "observed_at": "2026-09-20T06:26:12.631826+00:00", "status": 200, "final_url": "https://www.lg.com:443/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "46035ac91b468df4320e916c6bb84bc0b8340f9ec76836d4636fdec164f25fdb", "bytes_read": 8054 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://lg.com/llms.txt", "observed_at": "2026-09-20T06:26:14.385676+00:00", "status": 200, "final_url": "https://www.lg.com:443/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "f402115a8a2e3cc98cc35b0fb2b13576c7c0b88351612c9f108bc0a749cb8d15", "bytes_read": 2253 }, "llms_state": "candidate", "control": { "requested_url": "https://lg.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:26:15.889392+00:00", "status": 404, "final_url": "https://www.lg.com:443/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "28b773abf78703ee4876626ba939c677607df05ceb271e6cc0409bfaced90f5c", "bytes_read": 3116 } }, { "rank": 883, "domain": "frontiersin.org", "robots": { "requested_url": "https://frontiersin.org/robots.txt", "observed_at": "2026-09-20T06:26:13.376566+00:00", "status": 200, "final_url": "https://www.frontiersin.org/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "65821a4794de1eed81dd505e9cd356d9a4000377c53fe79a608b7f5146efdf5d", "bytes_read": 862 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://frontiersin.org/llms.txt", "observed_at": "2026-09-20T06:26:14.709457+00:00", "status": 200, "final_url": "https://www.frontiersin.org/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "2e11fe9a75f4bd228a799628ca86bd08a41c05b66661676294529665a88e6d7e", "bytes_read": 5255 }, "llms_state": "candidate", "control": { "requested_url": "https://frontiersin.org/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:26:15.689270+00:00", "status": 404, "final_url": "https://www.frontiersin.org/suede-index-control-9c52d073-no-file.txt", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 } }, { "rank": 884, "domain": "www.gov.br", "robots": { "requested_url": "https://www.gov.br/robots.txt", "observed_at": "2026-09-20T06:26:13.463956+00:00", "status": 200, "final_url": "https://www.gov.br/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "665417a8e7d2184fe1effefd6a242111682a9e59e94c4aa8af17e9c5487a090a", "bytes_read": 2102 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://www.gov.br/llms.txt", "observed_at": "2026-09-20T06:26:14.009005+00:00", "status": 404, "final_url": "https://www.gov.br/llms.txt", "content_type": "application/json", "truncated": false, "body_sha256": "6bbcea2b37bd092cc60604157a0ceec8944866fc2478f7af8aea3c3df40bc258", "bytes_read": 26 }, "llms_state": "not_detected", "control": null }, { "rank": 885, "domain": "mercadolivre.com.br", "robots": { "requested_url": "https://mercadolivre.com.br/robots.txt", "observed_at": "2026-09-20T06:26:13.564587+00:00", "status": 403, "final_url": "https://mercadolivre.com.br/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "8bef02b55f4137dfadb3df0cdf5961411fbf157323d7909fbd81423b1cdd1504", "bytes_read": 2585 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://mercadolivre.com.br/llms.txt", "observed_at": "2026-09-20T06:26:13.810349+00:00", "status": 403, "final_url": "https://mercadolivre.com.br/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "8bef02b55f4137dfadb3df0cdf5961411fbf157323d7909fbd81423b1cdd1504", "bytes_read": 2585 }, "llms_state": "unavailable", "control": null }, { "rank": 886, "domain": "spotifycdn.com", "robots": { "requested_url": "https://spotifycdn.com/robots.txt", "observed_at": "2026-09-20T06:26:14.011542+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://spotifycdn.com/llms.txt", "observed_at": "2026-09-20T06:26:14.065817+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 887, "domain": "biblegateway.com", "robots": { "requested_url": "https://biblegateway.com/robots.txt", "observed_at": "2026-09-20T06:26:14.067854+00:00", "status": 200, "final_url": "https://www.biblegateway.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "ab5ea5dd4dfeec82cc8e223b65dc0da8f89b3bf3b5989fac4d4207e06fdd4add", "bytes_read": 743 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://biblegateway.com/llms.txt", "observed_at": "2026-09-20T06:26:14.686805+00:00", "status": 404, "final_url": "https://www.biblegateway.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "2823ee5c866c7bbdc655481d092aeabc35e9796e430e12feca9d2f864e257153", "bytes_read": 69309 }, "llms_state": "not_detected", "control": null }, { "rank": 888, "domain": "nbcnews.com", "robots": { "requested_url": "https://nbcnews.com/robots.txt", "observed_at": "2026-09-20T06:26:14.073683+00:00", "status": 200, "final_url": "https://www.nbcnews.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "70bf9770d32a48c5ff7cab70cbf257f8bc913b09e3a9f814afef31c516815e1c", "bytes_read": 13381 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://nbcnews.com/llms.txt", "observed_at": "2026-09-20T06:26:15.291841+00:00", "status": 404, "final_url": "https://www.nbcnews.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "5f39b26209c42baaef55e4bab5119c695fc7426b31bf7b4fb7be76b08769dd0f", "bytes_read": 164406 }, "llms_state": "not_detected", "control": null }, { "rank": 889, "domain": "360safe.com", "robots": { "requested_url": "https://360safe.com/robots.txt", "observed_at": "2026-09-20T06:26:14.251793+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://360safe.com/llms.txt", "observed_at": "2026-09-20T06:26:24.455067+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 890, "domain": "myspace.com", "robots": { "requested_url": "https://myspace.com/robots.txt", "observed_at": "2026-09-20T06:26:14.447895+00:00", "status": 200, "final_url": "https://myspace.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "9e0c859a675640d4754cb878ee4b6e2c78d670162bc58cbfc2142b913d896f50", "bytes_read": 2489 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://myspace.com/llms.txt", "observed_at": "2026-09-20T06:26:14.691274+00:00", "status": 404, "final_url": "https://myspace.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "46b15a1afe5524774659e5019ad5c80686df9bf604743cff8bf35a0214a72bde", "bytes_read": 87300 }, "llms_state": "not_detected", "control": null }, { "rank": 891, "domain": "ys7.com", "robots": { "requested_url": "https://ys7.com/robots.txt", "observed_at": "2026-09-20T06:26:14.592062+00:00", "status": 200, "final_url": "https://www.ys7.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "16ceb5ee3e0dc13aa9adf31a3ebbe45a1d965b8c2b9f72eaf84e5911e140ed95", "bytes_read": 23 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://ys7.com/llms.txt", "observed_at": "2026-09-20T06:26:17.047067+00:00", "status": 404, "final_url": "https://www.ys7.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "4b6c01e8a47624242f85494c4feb626aa49326227de610f6e5ec896a275ebf20", "bytes_read": 254822 }, "llms_state": "not_detected", "control": null }, { "rank": 892, "domain": "dnse0.com", "robots": { "requested_url": "https://dnse0.com/robots.txt", "observed_at": "2026-09-20T06:26:15.063739+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://dnse0.com/llms.txt", "observed_at": "2026-09-20T06:26:15.138354+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 893, "domain": "azure-devices.net", "robots": { "requested_url": "https://azure-devices.net/robots.txt", "observed_at": "2026-09-20T06:26:15.124727+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://azure-devices.net/llms.txt", "observed_at": "2026-09-20T06:26:15.181108+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 894, "domain": "kleinanzeigen.de", "robots": { "requested_url": "https://kleinanzeigen.de/robots.txt", "observed_at": "2026-09-20T06:26:15.139949+00:00", "status": 403, "final_url": "https://kleinanzeigen.de/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "749ce8763dba848acde3206cc2ae07abce4d4a1de0ad27335e12dd438d795579", "bytes_read": 767 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://kleinanzeigen.de/llms.txt", "observed_at": "2026-09-20T06:26:15.395723+00:00", "status": 403, "final_url": "https://kleinanzeigen.de/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "864bbbafc57908cac08a037e6ffe491b817add11aea8745786ea94a657c31b33", "bytes_read": 767 }, "llms_state": "unavailable", "control": null }, { "rank": 895, "domain": "ssl-images-amazon.com", "robots": { "requested_url": "https://ssl-images-amazon.com/robots.txt", "observed_at": "2026-09-20T06:26:15.182772+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ssl-images-amazon.com/llms.txt", "observed_at": "2026-09-20T06:26:15.236417+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 896, "domain": "mayoclinic.org", "robots": { "requested_url": "https://mayoclinic.org/robots.txt", "observed_at": "2026-09-20T06:26:15.237949+00:00", "status": 403, "final_url": "https://www.mayoclinic.org/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "b5f87233a1f6dfe0e6f8d7e4329e26c86deea067ee760d6e67038d0d712c32a3", "bytes_read": 386 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://mayoclinic.org/llms.txt", "observed_at": "2026-09-20T06:26:15.952724+00:00", "status": 403, "final_url": "https://www.mayoclinic.org/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "56a163aa642007f12f458e411a4a026404903d30ee617d35a7c0307f1ca5c2d0", "bytes_read": 384 }, "llms_state": "unavailable", "control": null }, { "rank": 897, "domain": "bankofamerica.com", "robots": { "requested_url": "https://bankofamerica.com/robots.txt", "observed_at": "2026-09-20T06:26:15.594916+00:00", "status": 200, "final_url": "https://www.bankofamerica.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "a5e538b28463bf76e653bcdf6af94ae510991a3c4341cdcf7535de75d9f5a691", "bytes_read": 3312 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://bankofamerica.com/llms.txt", "observed_at": "2026-09-20T06:26:16.312891+00:00", "status": 404, "final_url": "https://www.bankofamerica.com/banking-information/error-page-en.html", "content_type": "text/html", "truncated": false, "body_sha256": "18b000c6bf53c7f5eb55d3f9644756be725bec88a9cff187a249b84e5527a269", "bytes_read": 5872 }, "llms_state": "not_detected", "control": null }, { "rank": 898, "domain": "readthedocs.io", "robots": { "requested_url": "https://readthedocs.io/robots.txt", "observed_at": "2026-09-20T06:26:16.465967+00:00", "status": 200, "final_url": "https://about.readthedocs.com/?ref=app.readthedocs.org", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "0fc8ac064d2b576b7a146c59cc9997c244a6041152d0f72ed9d19091e0ef24b3", "bytes_read": 33498 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://readthedocs.io/llms.txt", "observed_at": "2026-09-20T06:26:17.740914+00:00", "status": 200, "final_url": "https://about.readthedocs.com/?ref=app.readthedocs.org", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "0fc8ac064d2b576b7a146c59cc9997c244a6041152d0f72ed9d19091e0ef24b3", "bytes_read": 33498 }, "llms_state": "not_detected", "control": null }, { "rank": 899, "domain": "pornhub.com", "robots": { "requested_url": "https://pornhub.com/robots.txt", "observed_at": "2026-09-20T06:26:16.590563+00:00", "status": 200, "final_url": "https://www.pornhub.com/", "content_type": "text/html; charset=UTF-8", "truncated": true, "body_sha256": "2bfadf0304f06b5a831f2a3157cf92a938f9cd8c7a92f7c013b04eb19f3f9118", "bytes_read": 512001 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://pornhub.com/llms.txt", "observed_at": "2026-09-20T06:26:18.301256+00:00", "status": 200, "final_url": "https://www.pornhub.com/", "content_type": "text/html; charset=UTF-8", "truncated": true, "body_sha256": "be6039ccb2f80c609229d4dea96451999fe2ea5e515431a53b716698d30cd727", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 900, "domain": "immedia-semi.com", "robots": { "requested_url": "https://immedia-semi.com/robots.txt", "observed_at": "2026-09-20T06:26:16.633371+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://immedia-semi.com/llms.txt", "observed_at": "2026-09-20T06:26:16.678367+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 901, "domain": "agora.io", "robots": { "requested_url": "https://agora.io/robots.txt", "observed_at": "2026-09-20T06:26:16.680152+00:00", "status": 200, "final_url": "https://www.agora.io/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "566064b66257fb1853c4af106ed0b184dfc2306570d18842bd8d8babb07e9baa", "bytes_read": 153 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://agora.io/llms.txt", "observed_at": "2026-09-20T06:26:17.667456+00:00", "status": 200, "final_url": "https://www.agora.io/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "f522a0a6e74713f7374d68f6fe23b9dc5a8243d94fc5b6f3ae2e41cc73ce6f64", "bytes_read": 8993 }, "llms_state": "not_detected", "control": null }, { "rank": 902, "domain": "meta.com", "robots": { "requested_url": "https://meta.com/robots.txt", "observed_at": "2026-09-20T06:26:17.011440+00:00", "status": 200, "final_url": "https://www.meta.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "d1cc02c8b2817d84ab456896b2c6bbe922da75a85bd341b3283d33bacd2dd6ea", "bytes_read": 3419 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://meta.com/llms.txt", "observed_at": "2026-09-20T06:26:17.863071+00:00", "status": 404, "final_url": "https://www.meta.com/llms.txt", "content_type": "text/html; charset=\"utf-8\"", "truncated": false, "body_sha256": "81fe1169ea0427927de44b536775bee84d2e836e133bb992ef0e5730e33d4a68", "bytes_read": 254935 }, "llms_state": "not_detected", "control": null }, { "rank": 903, "domain": "dv.tech", "robots": { "requested_url": "https://dv.tech/robots.txt", "observed_at": "2026-09-20T06:26:17.179545+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://dv.tech/llms.txt", "observed_at": "2026-09-20T06:26:17.455041+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 904, "domain": "dbankcloud.ru", "robots": { "requested_url": "https://dbankcloud.ru/robots.txt", "observed_at": "2026-09-20T06:26:17.324671+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://dbankcloud.ru/llms.txt", "observed_at": "2026-09-20T06:26:17.383708+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 905, "domain": "worldbank.org", "robots": { "requested_url": "https://worldbank.org/robots.txt", "observed_at": "2026-09-20T06:26:17.385239+00:00", "status": 200, "final_url": "https://www.worldbank.org/robots.txt", "content_type": "text/plain;charset=iso-8859-1", "truncated": false, "body_sha256": "5c36a22fe5b1a380eec660d48c98cc850aa242bb0ca9c5a817a4b49ca621f186", "bytes_read": 1505 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://worldbank.org/llms.txt", "observed_at": "2026-09-20T06:26:17.907088+00:00", "status": 404, "final_url": "https://www.worldbank.org/llms.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "ddbc2521b5647bb682a687454e6bb65bbaaaa0292aaaca0dab1cd6abb0d5186d", "bytes_read": 5128 }, "llms_state": "not_detected", "control": null }, { "rank": 906, "domain": "fwmrm.net", "robots": { "requested_url": "https://fwmrm.net/robots.txt", "observed_at": "2026-09-20T06:26:17.561608+00:00", "status": 200, "final_url": "https://www.freewheel.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "cd1ccbf901e9b583dbe57aded77d41f5b13a1e777440f243417951f2a7dcc2a8", "bytes_read": 191 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://fwmrm.net/llms.txt", "observed_at": "2026-09-20T06:26:18.460947+00:00", "status": 404, "final_url": "https://www.freewheel.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "d8531a1ac331414d357ac014ac8813b93c2c0ea1756542ad8bacb6d352ed4891", "bytes_read": 60 }, "llms_state": "not_detected", "control": null }, { "rank": 907, "domain": "princeton.edu", "robots": { "requested_url": "https://princeton.edu/robots.txt", "observed_at": "2026-09-20T06:26:18.239985+00:00", "status": 200, "final_url": "https://www.Princeton.EDU/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "9052eb9806da2eec4c0c24ff91feb640a55e6f91b8f584180450b6007931ebae", "bytes_read": 2275 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://princeton.edu/llms.txt", "observed_at": "2026-09-20T06:26:18.895817+00:00", "status": 404, "final_url": "https://www.Princeton.EDU/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "463e3c7fbdc7884de697b5ad7aaabfde0dca67e895b4125bf33aff3d5f95eef8", "bytes_read": 191 }, "llms_state": "not_detected", "control": null }, { "rank": 908, "domain": "washington.edu", "robots": { "requested_url": "https://washington.edu/robots.txt", "observed_at": "2026-09-20T06:26:18.271022+00:00", "status": 200, "final_url": "https://www.washington.edu/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "138bc1cd0cdf0733fefd5472d008e2d3e1eb295b56fc3d22772e5ffd96c9d6b7", "bytes_read": 76 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://washington.edu/llms.txt", "observed_at": "2026-09-20T06:26:19.277393+00:00", "status": 404, "final_url": "https://www.washington.edu/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "716c8ddf9a372e43aa9a62d04bc348713626b3fd73a33f7b932f73afc8f03e91", "bytes_read": 19665 }, "llms_state": "not_detected", "control": null }, { "rank": 909, "domain": "samsungosp.com", "robots": { "requested_url": "https://samsungosp.com/robots.txt", "observed_at": "2026-09-20T06:26:18.295259+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://samsungosp.com/llms.txt", "observed_at": "2026-09-20T06:26:18.334591+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 910, "domain": "bytefcdn.com", "robots": { "requested_url": "https://bytefcdn.com/robots.txt", "observed_at": "2026-09-20T06:26:18.336425+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://bytefcdn.com/llms.txt", "observed_at": "2026-09-20T06:26:18.401407+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 911, "domain": "coupang.com", "robots": { "requested_url": "https://coupang.com/robots.txt", "observed_at": "2026-09-20T06:26:18.403365+00:00", "status": 403, "final_url": "https://www.coupang.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "bd28384c44bd9dff546025afef57f0f4aa175cd1888c58ca4184c35bbdd5bea2", "bytes_read": 381 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://coupang.com/llms.txt", "observed_at": "2026-09-20T06:26:19.365424+00:00", "status": 403, "final_url": "https://www.coupang.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "1ba98e4877bd4393c8dcae0681e4009e253596785c2ac786de508f6439d887b6", "bytes_read": 381 }, "llms_state": "unavailable", "control": null }, { "rank": 912, "domain": "internetwarriors.net", "robots": { "requested_url": "https://internetwarriors.net/robots.txt", "observed_at": "2026-09-20T06:26:18.503953+00:00", "status": 200, "final_url": "https://tracker.cl/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "f819fa0e4e42cd89628288fe194a410171978cf0ac2de3620bde071f2580ad19", "bytes_read": 2875 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://internetwarriors.net/llms.txt", "observed_at": "2026-09-20T06:26:19.673377+00:00", "status": 200, "final_url": "https://tracker.cl/", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "f819fa0e4e42cd89628288fe194a410171978cf0ac2de3620bde071f2580ad19", "bytes_read": 2875 }, "llms_state": "not_detected", "control": null }, { "rank": 913, "domain": "herokuapp.com", "robots": { "requested_url": "https://herokuapp.com/robots.txt", "observed_at": "2026-09-20T06:26:19.341724+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://herokuapp.com/llms.txt", "observed_at": "2026-09-20T06:26:19.520713+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 914, "domain": "spiegel.de", "robots": { "requested_url": "https://spiegel.de/robots.txt", "observed_at": "2026-09-20T06:26:19.369811+00:00", "status": 200, "final_url": "https://www.spiegel.de/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "38f59abb052fccb45a8de64f9c017fdfef5dd8a88469f369a4a84527e3b8e779", "bytes_read": 6010 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://spiegel.de/llms.txt", "observed_at": "2026-09-20T06:26:20.021049+00:00", "status": 404, "final_url": "https://www.spiegel.de/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "6562b5d3c104cd19a6a5dcfffb9129e5423c6fe37ad65de28a7cf73794748e7f", "bytes_read": 18953 }, "llms_state": "not_detected", "control": null }, { "rank": 915, "domain": "eu-1-id5-sync.com", "robots": { "requested_url": "https://eu-1-id5-sync.com/robots.txt", "observed_at": "2026-09-20T06:26:19.627812+00:00", "status": null, "error": "TimeoutError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://eu-1-id5-sync.com/llms.txt", "observed_at": "2026-09-20T06:26:29.852764+00:00", "status": null, "error": "TimeoutError" }, "llms_state": "unavailable", "control": null }, { "rank": 916, "domain": "queniuaa.com", "robots": { "requested_url": "https://queniuaa.com/robots.txt", "observed_at": "2026-09-20T06:26:19.830501+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://queniuaa.com/llms.txt", "observed_at": "2026-09-20T06:26:19.886539+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 917, "domain": "dnse2.com", "robots": { "requested_url": "https://dnse2.com/robots.txt", "observed_at": "2026-09-20T06:26:19.888164+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://dnse2.com/llms.txt", "observed_at": "2026-09-20T06:26:19.971233+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 918, "domain": "expedia.com", "robots": { "requested_url": "https://expedia.com/robots.txt", "observed_at": "2026-09-20T06:26:19.915530+00:00", "status": 200, "final_url": "https://www.expedia.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "d536d28cbb4c81ddb9234180df5081e15df9d1e4314dd9c9fd703cc8b9b984c0", "bytes_read": 2069 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://expedia.com/llms.txt", "observed_at": "2026-09-20T06:26:20.630712+00:00", "status": 200, "final_url": "https://www.expedia.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "0257a8ed950141a03755cbdb98e3851909df6eaa72a985c2416a1db413e82978", "bytes_read": 7011 }, "llms_state": "not_detected", "control": null }, { "rank": 919, "domain": "latimes.com", "robots": { "requested_url": "https://latimes.com/robots.txt", "observed_at": "2026-09-20T06:26:19.972951+00:00", "status": 200, "final_url": "https://www.latimes.com:443/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "6dc754f82f55902704f5e1cdd1f406e0f91246ce2238c92e9634f3335f57aab7", "bytes_read": 1768 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://latimes.com/llms.txt", "observed_at": "2026-09-20T06:26:20.783374+00:00", "status": 404, "final_url": "https://www.latimes.com:443/llms.txt", "content_type": "text/html;charset=UTF-8", "truncated": false, "body_sha256": "2a150e8a3e8bd026361b7be82c8eaac997f6f33612b7608e746cd15f8513cd36", "bytes_read": 334650 }, "llms_state": "not_detected", "control": null }, { "rank": 920, "domain": "starlink.com", "robots": { "requested_url": "https://starlink.com/robots.txt", "observed_at": "2026-09-20T06:26:20.012085+00:00", "status": 200, "final_url": "https://starlink.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "4d634c4a33a101759b1da1b5a819b8d95b25970e450c5d03083a49f59311adc0", "bytes_read": 619 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://starlink.com/llms.txt", "observed_at": "2026-09-20T06:26:20.232961+00:00", "status": 200, "final_url": "https://starlink.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "c036dd310904e058251c08c6931481c8e994795881024cac1365ec373d0612e3", "bytes_read": 33862 }, "llms_state": "not_detected", "control": null }, { "rank": 921, "domain": "t-s1-msedge.net", "robots": { "requested_url": "https://t-s1-msedge.net/robots.txt", "observed_at": "2026-09-20T06:26:20.265419+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://t-s1-msedge.net/llms.txt", "observed_at": "2026-09-20T06:26:20.306636+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 922, "domain": "wyzecam.com", "robots": { "requested_url": "https://wyzecam.com/robots.txt", "observed_at": "2026-09-20T06:26:20.308173+00:00", "status": 200, "final_url": "https://www.wyze.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "c7d772ebed1a0325565c684751de36feb10bd5a5390dfebdbbd321a3ba35907c", "bytes_read": 6418 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://wyzecam.com/llms.txt", "observed_at": "2026-09-20T06:26:20.874604+00:00", "status": 200, "final_url": "https://www.wyze.com/llms.txt", "content_type": "text/markdown; charset=utf-8", "truncated": false, "body_sha256": "6d8bc169f8f0dcebfd4f9d149e648d966eb06630af3160fa92232f7e1081fc05", "bytes_read": 4421 }, "llms_state": "candidate", "control": { "requested_url": "https://wyzecam.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:26:21.240289+00:00", "status": 404, "final_url": "https://www.wyze.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 } }, { "rank": 923, "domain": "samsungiotcloud.com", "robots": { "requested_url": "https://samsungiotcloud.com/robots.txt", "observed_at": "2026-09-20T06:26:20.391358+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://samsungiotcloud.com/llms.txt", "observed_at": "2026-09-20T06:26:20.421556+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 924, "domain": "volcgslb.com", "robots": { "requested_url": "https://volcgslb.com/robots.txt", "observed_at": "2026-09-20T06:26:20.391956+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://volcgslb.com/llms.txt", "observed_at": "2026-09-20T06:26:20.683163+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 925, "domain": "blackhub.team", "robots": { "requested_url": "https://blackhub.team/robots.txt", "observed_at": "2026-09-20T06:26:20.423169+00:00", "status": 200, "final_url": "https://blackhub.team/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "7ea4052aea20413007bfc2686ba0ad7276882075c2606c86a9e6b08742b3a4ca", "bytes_read": 26 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://blackhub.team/llms.txt", "observed_at": "2026-09-20T06:26:21.256097+00:00", "status": 404, "final_url": "https://blackhub.team/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "09302a3d30b555fa4eeaac4eda84ea63e114c90ae23db4fec69c8e9df4814042", "bytes_read": 520 }, "llms_state": "not_detected", "control": null }, { "rank": 926, "domain": "arcgis.com", "robots": { "requested_url": "https://arcgis.com/robots.txt", "observed_at": "2026-09-20T06:26:20.613689+00:00", "status": 200, "final_url": "https://www.arcgis.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "d9da31493f06281ceebc4eaff54a0ba0cd4bc00f01217e6dbec7f084b263730d", "bytes_read": 61 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://arcgis.com/llms.txt", "observed_at": "2026-09-20T06:26:20.989862+00:00", "status": 404, "final_url": "https://www.arcgis.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "bd2f44aab8d7f3e9dd6d175e6c8f158ec489b028f4bb34097db9d9c72c86cd0c", "bytes_read": 1190 }, "llms_state": "not_detected", "control": null }, { "rank": 927, "domain": "auvik.com", "robots": { "requested_url": "https://auvik.com/robots.txt", "observed_at": "2026-09-20T06:26:20.663321+00:00", "status": 200, "final_url": "https://www.auvik.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "16ea17710ec1c2813e9e1d56777547c077ac29d7ad50e47a335ed69a10e603fc", "bytes_read": 370 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://auvik.com/llms.txt", "observed_at": "2026-09-20T06:26:21.450053+00:00", "status": 404, "final_url": "https://www.auvik.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "d8531a1ac331414d357ac014ac8813b93c2c0ea1756542ad8bacb6d352ed4891", "bytes_read": 60 }, "llms_state": "not_detected", "control": null }, { "rank": 928, "domain": "playrix.com", "robots": { "requested_url": "https://playrix.com/robots.txt", "observed_at": "2026-09-20T06:26:20.685026+00:00", "status": 200, "final_url": "https://playrix.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "2a66f9f5d4d46510d2a3e0a48c71ae6d612e9586e7d7d401955edf2b13f5e356", "bytes_read": 3042 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://playrix.com/llms.txt", "observed_at": "2026-09-20T06:26:21.137078+00:00", "status": 200, "final_url": "https://playrix.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "2a66f9f5d4d46510d2a3e0a48c71ae6d612e9586e7d7d401955edf2b13f5e356", "bytes_read": 3042 }, "llms_state": "not_detected", "control": null }, { "rank": 929, "domain": "qualtrics.com", "robots": { "requested_url": "https://qualtrics.com/robots.txt", "observed_at": "2026-09-20T06:26:21.245744+00:00", "status": 200, "final_url": "https://www.qualtrics.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "152c7d3a5b2974c13436268201289512a7c386d39309eba80d0a2ecb449a2560", "bytes_read": 30643 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://qualtrics.com/llms.txt", "observed_at": "2026-09-20T06:26:22.754258+00:00", "status": 404, "final_url": "https://www.qualtrics.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "f4be0222ac6dd5560577e6a6a89e27b84a927c960a2202d79a7b3db766e8d715", "bytes_read": 191 }, "llms_state": "not_detected", "control": null }, { "rank": 930, "domain": "seedtag.com", "robots": { "requested_url": "https://seedtag.com/robots.txt", "observed_at": "2026-09-20T06:26:21.275572+00:00", "status": 200, "final_url": "https://www.seedtag.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://seedtag.com/llms.txt", "observed_at": "2026-09-20T06:26:21.858114+00:00", "status": 404, "final_url": "https://www.seedtag.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "dc707cf981f2df1b76c5f5c7abee391fad9509d838823fc40d65771464240938", "bytes_read": 62000 }, "llms_state": "not_detected", "control": null }, { "rank": 931, "domain": "kunluncan.com", "robots": { "requested_url": "https://kunluncan.com/robots.txt", "observed_at": "2026-09-20T06:26:21.497143+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://kunluncan.com/llms.txt", "observed_at": "2026-09-20T06:26:31.573105+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 932, "domain": "zpath.net", "robots": { "requested_url": "https://zpath.net/robots.txt", "observed_at": "2026-09-20T06:26:21.594862+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://zpath.net/llms.txt", "observed_at": "2026-09-20T06:26:21.640897+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 933, "domain": "netlify.app", "robots": { "requested_url": "https://netlify.app/robots.txt", "observed_at": "2026-09-20T06:26:21.643769+00:00", "status": 200, "final_url": "https://netlify.app/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "80b19fcba5702d0cb4ea92b9f4542a9eeb52fd1e74c93ad1c006dd8e6c258535", "bytes_read": 147 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://netlify.app/llms.txt", "observed_at": "2026-09-20T06:26:21.912550+00:00", "status": 200, "final_url": "https://www.netlify.com/llms.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "3bfe3d869838aee64d1bddd4b1318a24c85b4e072dda16cc0fa8c458fc3c373e", "bytes_read": 2818 }, "llms_state": "candidate", "control": { "requested_url": "https://netlify.app/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:26:22.685689+00:00", "status": 404, "final_url": "https://www.netlify.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "0d26010ed0c7142792f3d9f0088bc9a92ce967d05bf9f1d315e06289b6cdb28d", "bytes_read": 101860 } }, { "rank": 934, "domain": "onet.pl", "robots": { "requested_url": "https://onet.pl/robots.txt", "observed_at": "2026-09-20T06:26:21.705339+00:00", "status": 200, "final_url": "https://www.onet.pl/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "a1b1c7a0532c52c03812e3f0729834353b0229391b3df2528ca0bfc02ac963b3", "bytes_read": 586 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://onet.pl/llms.txt", "observed_at": "2026-09-20T06:26:22.730614+00:00", "status": 404, "final_url": "https://www.onet.pl/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "9a36c4f87cb033f27de7ce088eb038b00d432e1df23e4c7e68afa4c0f598e6a1", "bytes_read": 111726 }, "llms_state": "not_detected", "control": null }, { "rank": 935, "domain": "flashtalking.com", "robots": { "requested_url": "https://flashtalking.com/robots.txt", "observed_at": "2026-09-20T06:26:21.711953+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://flashtalking.com/llms.txt", "observed_at": "2026-09-20T06:26:21.928100+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 936, "domain": "epa.gov", "robots": { "requested_url": "https://epa.gov/robots.txt", "observed_at": "2026-09-20T06:26:21.901478+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://epa.gov/llms.txt", "observed_at": "2026-09-20T06:26:26.094609+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 937, "domain": "smaato.net", "robots": { "requested_url": "https://smaato.net/robots.txt", "observed_at": "2026-09-20T06:26:22.141442+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://smaato.net/llms.txt", "observed_at": "2026-09-20T06:26:22.212556+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 938, "domain": "docker.io", "robots": { "requested_url": "https://docker.io/robots.txt", "observed_at": "2026-09-20T06:26:22.147660+00:00", "status": 200, "final_url": "https://www.docker.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "bd14e1bfcf77ea4dce9dc4e88b8160639a83a7d2dd30dd5dbc3248948ebfc0aa", "bytes_read": 1033 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://docker.io/llms.txt", "observed_at": "2026-09-20T06:26:22.697616+00:00", "status": 404, "final_url": "https://www.docker.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "1e298488aca3c505e299f41ef2d1b380370abf41c3537d740390156c3ae6939f", "bytes_read": 205348 }, "llms_state": "not_detected", "control": null }, { "rank": 939, "domain": "abovedomains.com", "robots": { "requested_url": "https://abovedomains.com/robots.txt", "observed_at": "2026-09-20T06:26:22.229883+00:00", "status": 200, "final_url": "https://abovedomains.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "5286d9afb82f2037d221d2241c5595ae30e6c3385533a02859558daa9700f4d0", "bytes_read": 136 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://abovedomains.com/llms.txt", "observed_at": "2026-09-20T06:26:22.651878+00:00", "status": 200, "final_url": "https://abovedomains.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "336c1d0f20b3cd3520f1d22f10919ffeee05ae1e4565d319bc1cad03b477b252", "bytes_read": 1015 }, "llms_state": "not_detected", "control": null }, { "rank": 940, "domain": "cdn-vk.ru", "robots": { "requested_url": "https://cdn-vk.ru/robots.txt", "observed_at": "2026-09-20T06:26:22.626778+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cdn-vk.ru/llms.txt", "observed_at": "2026-09-20T06:26:22.814852+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 941, "domain": "zenecn.net", "robots": { "requested_url": "https://zenecn.net/robots.txt", "observed_at": "2026-09-20T06:26:22.817727+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://zenecn.net/llms.txt", "observed_at": "2026-09-20T06:26:22.903081+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 942, "domain": "smartthings.com", "robots": { "requested_url": "https://smartthings.com/robots.txt", "observed_at": "2026-09-20T06:26:22.905479+00:00", "status": 200, "final_url": "https://partners.smartthings.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "b3e3bf0323c18b8abb498ce3b821b3e7a2bda5d7005c9dcb9704c44b84f6dde4", "bytes_read": 122 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://smartthings.com/llms.txt", "observed_at": "2026-09-20T06:26:23.827731+00:00", "status": 404, "final_url": "https://partners.smartthings.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "59a098ee9bae0ed8276b75fff3588bd6038ed856030f21281a13633c578520c7", "bytes_read": 110986 }, "llms_state": "not_detected", "control": null }, { "rank": 943, "domain": "miwifi.com", "robots": { "requested_url": "https://miwifi.com/robots.txt", "observed_at": "2026-09-20T06:26:22.960548+00:00", "status": 404, "final_url": "https://www.miwifi.com/robots.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "b00269a91efa124d58f3b291a1ef9da195318ec40df7464be82ecf28c3204533", "bytes_read": 205 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://miwifi.com/llms.txt", "observed_at": "2026-09-20T06:26:25.530402+00:00", "status": 404, "final_url": "https://www.miwifi.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "73db7cbfed8c1340d31d07e911253225b7c4aab47b320697e974d110f24670d4", "bytes_read": 203 }, "llms_state": "not_detected", "control": null }, { "rank": 944, "domain": "genius.com", "robots": { "requested_url": "https://genius.com/robots.txt", "observed_at": "2026-09-20T06:26:23.037848+00:00", "status": 200, "final_url": "https://genius.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "08ee08424b891a2c8ba99d15527d83e056fcf1e8f224c81f2a8f208d49264f01", "bytes_read": 1149 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://genius.com/llms.txt", "observed_at": "2026-09-20T06:26:23.309498+00:00", "status": 404, "final_url": "https://genius.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "not_detected", "control": null }, { "rank": 945, "domain": "tiktokw.us", "robots": { "requested_url": "https://tiktokw.us/robots.txt", "observed_at": "2026-09-20T06:26:23.191623+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://tiktokw.us/llms.txt", "observed_at": "2026-09-20T06:26:23.238082+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 946, "domain": "lwsdns.com", "robots": { "requested_url": "https://lwsdns.com/robots.txt", "observed_at": "2026-09-20T06:26:23.197959+00:00", "status": 200, "final_url": "https://lwsdns.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "92adfbfff679ff4d778e85623c52567454056c1be0793d4f43f89ca6de698fae", "bytes_read": 79 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://lwsdns.com/llms.txt", "observed_at": "2026-09-20T06:26:23.831456+00:00", "status": 404, "final_url": "https://lwsdns.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "a83b84545c1793ffbde43b53073617cd631e998cb0d2e55513231cb25ce69a96", "bytes_read": 1523 }, "llms_state": "not_detected", "control": null }, { "rank": 947, "domain": "deloitte.com", "robots": { "requested_url": "https://deloitte.com/robots.txt", "observed_at": "2026-09-20T06:26:23.239730+00:00", "status": 200, "final_url": "https://www.deloitte.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "3c0715c72dbf6cfe7a4ed3f706c1c175ef85c4a703a54b2cbce25b7335dc4442", "bytes_read": 503 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://deloitte.com/llms.txt", "observed_at": "2026-09-20T06:26:23.933518+00:00", "status": 404, "final_url": "https://www.deloitte.com/llms.txt", "content_type": "text/html; charset=iso-8859-1", "truncated": false, "body_sha256": "9448f8a1159c9b14e3e1b9d8eab1a6ddf88d26e1f888a34cef430c756e4e6e1e", "bytes_read": 236 }, "llms_state": "not_detected", "control": null }, { "rank": 948, "domain": "lowes.com", "robots": { "requested_url": "https://lowes.com/robots.txt", "observed_at": "2026-09-20T06:26:23.500234+00:00", "status": 403, "final_url": "https://www.lowes.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "64e13e4b518675a7109f5f446428470c37017bbae5223ab682014b12d345affa", "bytes_read": 379 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://lowes.com/llms.txt", "observed_at": "2026-09-20T06:26:24.014051+00:00", "status": 403, "final_url": "https://www.lowes.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "8a2782fc7cbe4b407019e9bc3fa2003bb42cfdbcb82e1b95e695542474b9509e", "bytes_read": 377 }, "llms_state": "unavailable", "control": null }, { "rank": 949, "domain": "youku.com", "robots": { "requested_url": "https://youku.com/robots.txt", "observed_at": "2026-09-20T06:26:23.650249+00:00", "status": 200, "final_url": "https://www.youku.com/index/y404", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "3b4e36bfce599388909c1905708215b5765b2d6cf5bcd57540e64f12fc390d14", "bytes_read": 15349 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://youku.com/llms.txt", "observed_at": "2026-09-20T06:26:26.930158+00:00", "status": 200, "final_url": "https://www.youku.com/index/y404", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "3b4e36bfce599388909c1905708215b5765b2d6cf5bcd57540e64f12fc390d14", "bytes_read": 15349 }, "llms_state": "not_detected", "control": null }, { "rank": 950, "domain": "go-mpulse.net", "robots": { "requested_url": "https://go-mpulse.net/robots.txt", "observed_at": "2026-09-20T06:26:23.915204+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://go-mpulse.net/llms.txt", "observed_at": "2026-09-20T06:26:23.947487+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 951, "domain": "sentinelone.net", "robots": { "requested_url": "https://sentinelone.net/robots.txt", "observed_at": "2026-09-20T06:26:23.949630+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://sentinelone.net/llms.txt", "observed_at": "2026-09-20T06:26:34.003259+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 952, "domain": "state.gov", "robots": { "requested_url": "https://state.gov/robots.txt", "observed_at": "2026-09-20T06:26:24.005455+00:00", "status": 403, "final_url": "https://www.state.gov/robots.txt", "content_type": "text/html", "truncated": true, "body_sha256": "49deb37ca28d2d31145352bf844e41645b94722c977bc31172abf614298fc7d8", "bytes_read": 512001 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://state.gov/llms.txt", "observed_at": "2026-09-20T06:26:24.725061+00:00", "status": 403, "final_url": "https://www.state.gov/llms.txt", "content_type": "text/html", "truncated": true, "body_sha256": "49deb37ca28d2d31145352bf844e41645b94722c977bc31172abf614298fc7d8", "bytes_read": 512001 }, "llms_state": "unavailable", "control": null }, { "rank": 953, "domain": "leiniao.com", "robots": { "requested_url": "https://leiniao.com/robots.txt", "observed_at": "2026-09-20T06:26:24.265528+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://leiniao.com/llms.txt", "observed_at": "2026-09-20T06:26:34.337335+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 954, "domain": "epam.com", "robots": { "requested_url": "https://epam.com/robots.txt", "observed_at": "2026-09-20T06:26:24.300242+00:00", "status": 403, "final_url": "https://epam.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "00d7788145292c398af7f5f4df1707eac56532495fe8138112c0b1a1ef18f662", "bytes_read": 5514 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://epam.com/llms.txt", "observed_at": "2026-09-20T06:26:24.520967+00:00", "status": 403, "final_url": "https://epam.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "6d716f68e56bf2d04b0b2479d807c661310616e9b8ba4eca404694dd0d97a625", "bytes_read": 5508 }, "llms_state": "unavailable", "control": null }, { "rank": 955, "domain": "eporner.com", "robots": { "requested_url": "https://eporner.com/robots.txt", "observed_at": "2026-09-20T06:26:24.650173+00:00", "status": 200, "final_url": "https://eporner.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "c59d657e617896b9f093a8e98de398b9584489afdd559edb39ae8d46dbf083dc", "bytes_read": 4679 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://eporner.com/llms.txt", "observed_at": "2026-09-20T06:26:25.133240+00:00", "status": 404, "final_url": "https://eporner.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 }, "llms_state": "not_detected", "control": null }, { "rank": 956, "domain": "hilton.com", "robots": { "requested_url": "https://hilton.com/robots.txt", "observed_at": "2026-09-20T06:26:24.686248+00:00", "status": 200, "final_url": "https://www.hilton.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "7dcfdd1c709ea42fe582aa4d5a72810208e7207bf9ad5acff48a57b4ab35ceec", "bytes_read": 770 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://hilton.com/llms.txt", "observed_at": "2026-09-20T06:26:25.545774+00:00", "status": 404, "final_url": "https://www.hilton.com/llms.txt", "content_type": "text/html; charset=iso-8859-1", "truncated": false, "body_sha256": "0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5", "bytes_read": 9 }, "llms_state": "not_detected", "control": null }, { "rank": 957, "domain": "tailscale.com", "robots": { "requested_url": "https://tailscale.com/robots.txt", "observed_at": "2026-09-20T06:26:24.724801+00:00", "status": 200, "final_url": "https://tailscale.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "0dc2c25957cbbcefb82240afee0e91db4a44c01c4ca93ed6607825a6c0f71cad", "bytes_read": 109 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://tailscale.com/llms.txt", "observed_at": "2026-09-20T06:26:24.993600+00:00", "status": 200, "final_url": "https://tailscale.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "9a38afc113269fb02a4b7741ae5133770ad30d8ac69388676b6b1c121bec6c22", "bytes_read": 415 }, "llms_state": "candidate", "control": { "requested_url": "https://tailscale.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:26:25.183939+00:00", "status": 404, "final_url": "https://tailscale.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "7579a3cfd9c12ab6c93d7b5230c403689f975caace333ff34f4d50613a2700e1", "bytes_read": 86016 } }, { "rank": 958, "domain": "mercadolibre.com.ar", "robots": { "requested_url": "https://mercadolibre.com.ar/robots.txt", "observed_at": "2026-09-20T06:26:25.368048+00:00", "status": 403, "final_url": "https://mercadolibre.com.ar/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "8bef02b55f4137dfadb3df0cdf5961411fbf157323d7909fbd81423b1cdd1504", "bytes_read": 2585 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://mercadolibre.com.ar/llms.txt", "observed_at": "2026-09-20T06:26:25.581839+00:00", "status": 403, "final_url": "https://mercadolibre.com.ar/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "8bef02b55f4137dfadb3df0cdf5961411fbf157323d7909fbd81423b1cdd1504", "bytes_read": 2585 }, "llms_state": "unavailable", "control": null }, { "rank": 959, "domain": "thenai.org", "robots": { "requested_url": "https://thenai.org/robots.txt", "observed_at": "2026-09-20T06:26:25.568336+00:00", "status": 200, "final_url": "https://thenai.org/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "9286b5396e11ed5a6b30dc875b3f1f11f99ad473ceebe0e8fbbef6da535f9e9b", "bytes_read": 184 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://thenai.org/llms.txt", "observed_at": "2026-09-20T06:26:25.686235+00:00", "status": 404, "final_url": "https://thenai.org/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "d8531a1ac331414d357ac014ac8813b93c2c0ea1756542ad8bacb6d352ed4891", "bytes_read": 60 }, "llms_state": "not_detected", "control": null }, { "rank": 960, "domain": "character.ai", "robots": { "requested_url": "https://character.ai/robots.txt", "observed_at": "2026-09-20T06:26:25.712095+00:00", "status": 200, "final_url": "https://character.ai/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "686f48ea0a3576a71286201597336dcad052a0cb8035f37cda078cbf6ca47f18", "bytes_read": 159 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://character.ai/llms.txt", "observed_at": "2026-09-20T06:26:25.948583+00:00", "status": 404, "final_url": "https://character.ai/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "7162b95de52bcbf479229d49c03090e9a519155329dc8762eedf0e88dc7ba8f4", "bytes_read": 7951 }, "llms_state": "not_detected", "control": null }, { "rank": 961, "domain": "chaturbate.com", "robots": { "requested_url": "https://chaturbate.com/robots.txt", "observed_at": "2026-09-20T06:26:25.742708+00:00", "status": 200, "final_url": "https://chaturbate.com/?next=/robots.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "dc2dc5471cc53e0a4d17437c96873f62dfaf46b1383775e9c71de1f7b141c87b", "bytes_read": 203091 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://chaturbate.com/llms.txt", "observed_at": "2026-09-20T06:26:26.264134+00:00", "status": 200, "final_url": "https://chaturbate.com/?next=/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "7353893cde5e19cde625a3db17b6b8eb36ed1fea81921da0fa07bf718da8a5c4", "bytes_read": 203082 }, "llms_state": "not_detected", "control": null }, { "rank": 962, "domain": "jd.com", "robots": { "requested_url": "https://jd.com/robots.txt", "observed_at": "2026-09-20T06:26:25.999105+00:00", "status": 200, "final_url": "https://jd.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "3e033ad639e1be74865f620abd440088a90fd889f9c55584ecaf4a166987be1b", "bytes_read": 786 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://jd.com/llms.txt", "observed_at": "2026-09-20T06:26:28.412279+00:00", "status": 200, "final_url": "https://corporate.jd.com/home", "content_type": "text/html", "truncated": false, "body_sha256": "a584a2509298e94847062ad6ee1f810938e7b65d3ee1d9edfd19f45c57b28527", "bytes_read": 686 }, "llms_state": "not_detected", "control": null }, { "rank": 963, "domain": "note.com", "robots": { "requested_url": "https://note.com/robots.txt", "observed_at": "2026-09-20T06:26:26.336884+00:00", "status": 200, "final_url": "https://note.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "72d9ff821af4d441fabe8bde18aa57cc0ca82a40c28f9b2369c438b4ecf2e840", "bytes_read": 1158 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://note.com/llms.txt", "observed_at": "2026-09-20T06:26:26.678004+00:00", "status": 404, "final_url": "https://note.com/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "not_detected", "control": null }, { "rank": 964, "domain": "webmd.com", "robots": { "requested_url": "https://webmd.com/robots.txt", "observed_at": "2026-09-20T06:26:26.685687+00:00", "status": 200, "final_url": "https://www.webmd.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "e7610682d0934ea2b4266992e2a88b3ca70c2e2ff9f8047223b03c2ca4fbce7a", "bytes_read": 886 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://webmd.com/llms.txt", "observed_at": "2026-09-20T06:26:27.178917+00:00", "status": 404, "final_url": "https://www.webmd.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "b751176038b966b4d307ddf7a21ca4318ad6a00ce312a5d4dc531037a969d398", "bytes_read": 294547 }, "llms_state": "not_detected", "control": null }, { "rank": 965, "domain": "trendyol.com", "robots": { "requested_url": "https://trendyol.com/robots.txt", "observed_at": "2026-09-20T06:26:27.351286+00:00", "status": 200, "final_url": "https://www.trendyol.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "d2041695a2845974ba0ff0b1e762141c49efa322d8c33bab9a782f07f5f639eb", "bytes_read": 15789 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://trendyol.com/llms.txt", "observed_at": "2026-09-20T06:26:28.410131+00:00", "status": 200, "final_url": "https://www.trendyol.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "537f0d2db9530ba51afce794c3f111a1c673e3676d2b8f7feaec6d0f4110ce34", "bytes_read": 4526 }, "llms_state": "candidate", "control": { "requested_url": "https://trendyol.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:26:28.804246+00:00", "status": 404, "final_url": "https://www.trendyol.com/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "fa38ecfad650c49eef264b0be3f25c0e5120e6d230c80c257392fe6d7a3457e0", "bytes_read": 20054 } }, { "rank": 966, "domain": "fidelity.com", "robots": { "requested_url": "https://fidelity.com/robots.txt", "observed_at": "2026-09-20T06:26:27.563947+00:00", "status": 200, "final_url": "https://www.fidelity.com:443/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "b0d6dc0a3b99913ed2e51e3460998e0b49fe141b1e7bbce202f1aabea1c51eb9", "bytes_read": 12403 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://fidelity.com/llms.txt", "observed_at": "2026-09-20T06:26:28.335856+00:00", "status": 403, "final_url": "https://www.fidelity.com:443/llms.txt", "content_type": "application/xml; charset=utf-8", "truncated": false, "body_sha256": "1bec4a6e5872f1c0335e8d3850a1b14d3e0e1a11fb431f4adfe28cad0acc44cd", "bytes_read": 243 }, "llms_state": "unavailable", "control": null }, { "rank": 967, "domain": "mailchi.mp", "robots": { "requested_url": "https://mailchi.mp/robots.txt", "observed_at": "2026-09-20T06:26:27.750887+00:00", "status": 200, "final_url": "https://mailchi.mp/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "5ab623d21052b3e98340b667fd8f9ec2be65fc26674585f20a9a3257ff5e51f5", "bytes_read": 138 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://mailchi.mp/llms.txt", "observed_at": "2026-09-20T06:26:27.965088+00:00", "status": 200, "final_url": "https://mailchi.mp/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "48e07cd00a1d06a55664f05715d01ade4610c9ff9dbceca43788331b0df008bc", "bytes_read": 2702 }, "llms_state": "not_detected", "control": null }, { "rank": 968, "domain": "xerox.com", "robots": { "requested_url": "https://xerox.com/robots.txt", "observed_at": "2026-09-20T06:26:28.559917+00:00", "status": 200, "final_url": "https://www.xerox.com/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "c4ec5584aa4ad5b2f4d17f5cbbaee1c577a8d047e0405e12cd762d912e5e0fd9", "bytes_read": 2303 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://xerox.com/llms.txt", "observed_at": "2026-09-20T06:26:29.628155+00:00", "status": 404, "final_url": "https://www.xerox.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "951934d9f280772cd5c1d157bd42896f40b4333fbc4ea13a15baf6fd8e19da7c", "bytes_read": 266611 }, "llms_state": "not_detected", "control": null }, { "rank": 969, "domain": "hm.com", "robots": { "requested_url": "https://hm.com/robots.txt", "observed_at": "2026-09-20T06:26:28.750055+00:00", "status": null, "error": "TimeoutError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://hm.com/llms.txt", "observed_at": "2026-09-20T06:26:38.912851+00:00", "status": null, "error": "TimeoutError" }, "llms_state": "unavailable", "control": null }, { "rank": 970, "domain": "repubblica.it", "robots": { "requested_url": "https://repubblica.it/robots.txt", "observed_at": "2026-09-20T06:26:29.190738+00:00", "status": 200, "final_url": "https://www.repubblica.it/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "24734a2dae51193f3bfb3025251f2f379d481375b083190dd4c5073049e79906", "bytes_read": 7743 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://repubblica.it/llms.txt", "observed_at": "2026-09-20T06:26:29.981665+00:00", "status": 200, "final_url": "https://www.repubblica.it/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "6a7332964fdd8c4746d949a6cb64c62cd12611078999cbbd0a4eacb5d86d58ed", "bytes_read": 2331 }, "llms_state": "candidate", "control": { "requested_url": "https://repubblica.it/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:26:30.541962+00:00", "status": 404, "final_url": "https://www.repubblica.it/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "95c1abefa9e4fbb6881f851c474ff32b459ae442d6c8d6205d8b0ca342746bdf", "bytes_read": 62172 } }, { "rank": 971, "domain": "stbid.ru", "robots": { "requested_url": "https://stbid.ru/robots.txt", "observed_at": "2026-09-20T06:26:29.471680+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://stbid.ru/llms.txt", "observed_at": "2026-09-20T06:26:29.936902+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 972, "domain": "xhamster46.desi", "robots": { "requested_url": "https://xhamster46.desi/robots.txt", "observed_at": "2026-09-20T06:26:30.119521+00:00", "status": 200, "final_url": "https://xhamster46.desi/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "8ebd6b73326801117db8965aec8abfaefcf2e9b369c72bda940485a9701be0c1", "bytes_read": 4504 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://xhamster46.desi/llms.txt", "observed_at": "2026-09-20T06:26:30.540009+00:00", "status": 404, "final_url": "https://xhamster46.desi/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 }, "llms_state": "not_detected", "control": null }, { "rank": 973, "domain": "merriam-webster.com", "robots": { "requested_url": "https://merriam-webster.com/robots.txt", "observed_at": "2026-09-20T06:26:30.215746+00:00", "status": 200, "final_url": "https://www.merriam-webster.com:443/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "1bfd69adeda2fc53f53641b4d25fd6f0134fd86776332fbb92ecd05604381efa", "bytes_read": 708 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://merriam-webster.com/llms.txt", "observed_at": "2026-09-20T06:26:30.591332+00:00", "status": 404, "final_url": "https://www.merriam-webster.com:443/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 }, "llms_state": "not_detected", "control": null }, { "rank": 974, "domain": "4dex.io", "robots": { "requested_url": "https://4dex.io/robots.txt", "observed_at": "2026-09-20T06:26:30.696503+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://4dex.io/llms.txt", "observed_at": "2026-09-20T06:26:30.735279+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 975, "domain": "ancestry.com", "robots": { "requested_url": "https://ancestry.com/robots.txt", "observed_at": "2026-09-20T06:26:30.736750+00:00", "status": 200, "final_url": "https://www.ancestry.com/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "096bf0b2e456c9df59fc35adb477795f77efb4896ab930bd39823ae0b046a790", "bytes_read": 3623 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://ancestry.com/llms.txt", "observed_at": "2026-09-20T06:26:31.115829+00:00", "status": 404, "final_url": "https://www.ancestry.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "c4970b02218609c8e8319563ddfd92ed5c49af7804c5411d1557cfd3ce72baa4", "bytes_read": 27487 }, "llms_state": "not_detected", "control": null }, { "rank": 976, "domain": "change.org", "robots": { "requested_url": "https://change.org/robots.txt", "observed_at": "2026-09-20T06:26:30.782855+00:00", "status": 200, "final_url": "https://www.change.org/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "445e8105ffccebc56021d5984c94bc9f0dcb154ad82e5de810d6096fe2489606", "bytes_read": 5062 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://change.org/llms.txt", "observed_at": "2026-09-20T06:26:31.188331+00:00", "status": 404, "final_url": "https://www.change.org/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "7ee57e79fd391f53f0c24cc77f15804a1e9a6867cd2045707d6d11d8bbb8e94c", "bytes_read": 153352 }, "llms_state": "not_detected", "control": null }, { "rank": 977, "domain": "corriere.it", "robots": { "requested_url": "https://corriere.it/robots.txt", "observed_at": "2026-09-20T06:26:30.886937+00:00", "status": 200, "final_url": "https://www.corriere.it/robots.txt", "content_type": "text/plain; charset=UTF-8", "truncated": false, "body_sha256": "d6a4b3965eab5057db4542f03d44bfaa2fbd4d4b343be9deea5c64eb63d1c0d0", "bytes_read": 10074 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://corriere.it/llms.txt", "observed_at": "2026-09-20T06:26:31.842982+00:00", "status": 404, "final_url": "https://www.corriere.it/llms.txt", "content_type": "text/html; charset=ISO-8859-1", "truncated": false, "body_sha256": "8a8b0ef11942c0e6aaaee8add33d4e70a036dff95ca7bcbe7c5133bb00aab777", "bytes_read": 142294 }, "llms_state": "not_detected", "control": null }, { "rank": 978, "domain": "nexusmods.com", "robots": { "requested_url": "https://nexusmods.com/robots.txt", "observed_at": "2026-09-20T06:26:31.204841+00:00", "status": 403, "final_url": "https://www.nexusmods.com/robots.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "bddac261721ac77b9e668fe792923beed4ca00322bee8ee803b99e793e2686da", "bytes_read": 5544 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://nexusmods.com/llms.txt", "observed_at": "2026-09-20T06:26:31.475291+00:00", "status": 403, "final_url": "https://www.nexusmods.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "66714c3bb7d3032aba251c94349200c3d67d92325b7f1b574d562fd8c1183142", "bytes_read": 5538 }, "llms_state": "unavailable", "control": null }, { "rank": 979, "domain": "investopedia.com", "robots": { "requested_url": "https://investopedia.com/robots.txt", "observed_at": "2026-09-20T06:26:31.296601+00:00", "status": 200, "final_url": "https://www.investopedia.com/robots.txt", "content_type": "text/plain;charset=utf-8", "truncated": false, "body_sha256": "b1d636c1cb75fb9e0f3940b1a72468516602b5244f9798c137f6d8610f1df0a1", "bytes_read": 3477 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://investopedia.com/llms.txt", "observed_at": "2026-09-20T06:26:31.636065+00:00", "status": 404, "final_url": "https://www.investopedia.com/llms.txt", "content_type": "text/html;charset=utf-8", "truncated": false, "body_sha256": "7304d65fb292ea9e7b47290d60cd165d72745787c3d3309a6201924cbd813d58", "bytes_read": 388653 }, "llms_state": "not_detected", "control": null }, { "rank": 980, "domain": "xnxx.com", "robots": { "requested_url": "https://xnxx.com/robots.txt", "observed_at": "2026-09-20T06:26:31.580890+00:00", "status": 200, "final_url": "https://xnxx.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "8c71185fec7461ed3ac2c6ba90b1ea39641c06fed9359754235a8f550db8f60b", "bytes_read": 61 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://xnxx.com/llms.txt", "observed_at": "2026-09-20T06:26:32.033421+00:00", "status": 404, "final_url": "https://www.xnxx.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "55ba033e1eb4783fc15d95d3292d3d6fe2562663fc608280393352ae18a05e7f", "bytes_read": 63662 }, "llms_state": "not_detected", "control": null }, { "rank": 981, "domain": "caixa.gov.br", "robots": { "requested_url": "https://caixa.gov.br/robots.txt", "observed_at": "2026-09-20T06:26:31.688511+00:00", "status": 302, "final_url": "https://www.caixa.gov.br/robots.txt", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://caixa.gov.br/llms.txt", "observed_at": "2026-09-20T06:26:34.568328+00:00", "status": 302, "final_url": "https://www.caixa.gov.br/llms.txt", "content_type": "", "truncated": false, "body_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "bytes_read": 0 }, "llms_state": "unavailable", "control": null }, { "rank": 982, "domain": "btloader.com", "robots": { "requested_url": "https://btloader.com/robots.txt", "observed_at": "2026-09-20T06:26:32.126388+00:00", "status": 200, "final_url": "https://btloader.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "331ea9090db0c9f6f597bd9840fd5b171830f6e0b3ba1cb24dfa91f0c95aedc1", "bytes_read": 26 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "ClaudeBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "PerplexityBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "CCBot": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": false, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://btloader.com/llms.txt", "observed_at": "2026-09-20T06:26:32.254844+00:00", "status": 403, "final_url": "https://btloader.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "c029d1c560ecb35bbb632f9f7102907b41a289e258055d477f92e21884f9ab79", "bytes_read": 10 }, "llms_state": "unavailable", "control": null }, { "rank": 983, "domain": "presage.io", "robots": { "requested_url": "https://presage.io/robots.txt", "observed_at": "2026-09-20T06:26:32.240507+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://presage.io/llms.txt", "observed_at": "2026-09-20T06:26:32.535368+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 984, "domain": "nationalgeographic.com", "robots": { "requested_url": "https://nationalgeographic.com/robots.txt", "observed_at": "2026-09-20T06:26:32.419269+00:00", "status": 200, "final_url": "https://www.nationalgeographic.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "54c1714b0a209f655aa356a413f05d9e5651baae92e47e05acf86639d7e82f2a", "bytes_read": 1396 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://nationalgeographic.com/llms.txt", "observed_at": "2026-09-20T06:26:32.867549+00:00", "status": 404, "final_url": "https://www.nationalgeographic.com/llms.txt", "content_type": "text/html", "truncated": false, "body_sha256": "36a9e7f1c95b82ffb99743e0c5c4ce95d83c9a430aac59f84ef3cbfab6145068", "bytes_read": 1 }, "llms_state": "not_detected", "control": null }, { "rank": 985, "domain": "ddns.net", "robots": { "requested_url": "https://ddns.net/robots.txt", "observed_at": "2026-09-20T06:26:32.691533+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ddns.net/llms.txt", "observed_at": "2026-09-20T06:26:42.730077+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 986, "domain": "cdnhwc2.com", "robots": { "requested_url": "https://cdnhwc2.com/robots.txt", "observed_at": "2026-09-20T06:26:32.815219+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://cdnhwc2.com/llms.txt", "observed_at": "2026-09-20T06:26:33.077251+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 987, "domain": "otto.de", "robots": { "requested_url": "https://otto.de/robots.txt", "observed_at": "2026-09-20T06:26:32.823606+00:00", "status": 200, "final_url": "https://www.otto.de/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "86abc355699826c5ac3f6b8c847f9889b7fa68fcc1176bad10a86892a8e99f85", "bytes_read": 2148 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": true, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://otto.de/llms.txt", "observed_at": "2026-09-20T06:26:34.197772+00:00", "status": 404, "final_url": "https://www.otto.de/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "7d04f7431bbfa41a04bcc7e6b98b9de0d919756c4c671c5785c99fff45f16402", "bytes_read": 13 }, "llms_state": "not_detected", "control": null }, { "rank": 988, "domain": "app-ads-services.com", "robots": { "requested_url": "https://app-ads-services.com/robots.txt", "observed_at": "2026-09-20T06:26:33.078880+00:00", "status": 200, "final_url": "https://app-ads-services.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "1aba570a1b32deb66941e0c29d4f37be71ee58aa1a624aa20754b3639538cf2c", "bytes_read": 6624 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://app-ads-services.com/llms.txt", "observed_at": "2026-09-20T06:26:33.309762+00:00", "status": 404, "final_url": "https://app-ads-services.com/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "47d71ed470dfe1ca13eb2af8fa019b95871a1bd1381a518ed71359e69b9c3cc7", "bytes_read": 1569 }, "llms_state": "not_detected", "control": null }, { "rank": 989, "domain": "kwai.com", "robots": { "requested_url": "https://kwai.com/robots.txt", "observed_at": "2026-09-20T06:26:33.081508+00:00", "status": 200, "final_url": "https://www.kwai.com/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "2eefee389f663c8f931a57f42be54da83c65103f982133438f15a9155e8cbfbd", "bytes_read": 231 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://kwai.com/llms.txt", "observed_at": "2026-09-20T06:26:35.488224+00:00", "status": 200, "final_url": "https://www.kwai.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "13fb36e78982a3944472873b1b2aafdbd00b22a127aa78911cfe0ce34fd990a9", "bytes_read": 5134 }, "llms_state": "not_detected", "control": null }, { "rank": 990, "domain": "sofascore.com", "robots": { "requested_url": "https://sofascore.com/robots.txt", "observed_at": "2026-09-20T06:26:33.167765+00:00", "status": 403, "final_url": "https://sofascore.com/robots.txt", "content_type": "application/json", "truncated": false, "body_sha256": "6b771bca0fe4271cc0baf29954f10c68be99102048e4df524b6d7d6bf38dad5d", "bytes_read": 48 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://sofascore.com/llms.txt", "observed_at": "2026-09-20T06:26:33.321977+00:00", "status": 403, "final_url": "https://sofascore.com/llms.txt", "content_type": "application/json", "truncated": false, "body_sha256": "6b771bca0fe4271cc0baf29954f10c68be99102048e4df524b6d7d6bf38dad5d", "bytes_read": 48 }, "llms_state": "unavailable", "control": null }, { "rank": 991, "domain": "ubi.com", "robots": { "requested_url": "https://ubi.com/robots.txt", "observed_at": "2026-09-20T06:26:33.429288+00:00", "status": 200, "final_url": "https://www.ubisoft.com/en-us", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "ead3d67bee71dce9f1c78b78abd83746133fb0d0dc1dd0e11e151918eb632995", "bytes_read": 474657 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ubi.com/llms.txt", "observed_at": "2026-09-20T06:26:34.731374+00:00", "status": 200, "final_url": "https://www.ubisoft.com/en-us", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "ead3d67bee71dce9f1c78b78abd83746133fb0d0dc1dd0e11e151918eb632995", "bytes_read": 474657 }, "llms_state": "not_detected", "control": null }, { "rank": 992, "domain": "healthline.com", "robots": { "requested_url": "https://healthline.com/robots.txt", "observed_at": "2026-09-20T06:26:33.445385+00:00", "status": 200, "final_url": "https://www.healthline.com/robots.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "bdd5879797eabcb955749cdf9e754cb3b23510de793835ba5588e3f91e87130a", "bytes_read": 8864 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://healthline.com/llms.txt", "observed_at": "2026-09-20T06:26:33.954623+00:00", "status": 404, "final_url": "https://www.healthline.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "410d0b3b8c44cc214445ada387a286d9a76e86ac8402ba9ab51560691b8af2dc", "bytes_read": 145697 }, "llms_state": "not_detected", "control": null }, { "rank": 993, "domain": "ln-msedge.net", "robots": { "requested_url": "https://ln-msedge.net/robots.txt", "observed_at": "2026-09-20T06:26:34.458066+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://ln-msedge.net/llms.txt", "observed_at": "2026-09-20T06:26:34.500038+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 994, "domain": "opera-api.com", "robots": { "requested_url": "https://opera-api.com/robots.txt", "observed_at": "2026-09-20T06:26:34.484158+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://opera-api.com/llms.txt", "observed_at": "2026-09-20T06:26:44.540923+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 995, "domain": "nexthink.cloud", "robots": { "requested_url": "https://nexthink.cloud/robots.txt", "observed_at": "2026-09-20T06:26:34.501914+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://nexthink.cloud/llms.txt", "observed_at": "2026-09-20T06:26:34.531784+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 996, "domain": "bol.com", "robots": { "requested_url": "https://bol.com/robots.txt", "observed_at": "2026-09-20T06:26:34.517979+00:00", "status": 200, "final_url": "https://www.bol.com/robots.txt", "content_type": "text/plain;charset=UTF-8", "truncated": false, "body_sha256": "0d586f74da79970569eb6f268ca039a61e38d283f996fac2edf493f744103a4e", "bytes_read": 6766 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [ [ "allow", "/" ] ] } }, "llms": { "requested_url": "https://bol.com/llms.txt", "observed_at": "2026-09-20T06:26:35.627800+00:00", "status": 404, "final_url": "https://www.bol.com/llms.txt", "content_type": "text/html; charset=utf-8", "truncated": false, "body_sha256": "84733b7c97f851b337c6467937426004e0758526496dc2e1dfd391238f2cd616", "bytes_read": 11435 }, "llms_state": "not_detected", "control": null }, { "rank": 997, "domain": "conviva.com", "robots": { "requested_url": "https://conviva.com/robots.txt", "observed_at": "2026-09-20T06:26:34.534362+00:00", "status": 200, "final_url": "https://www.conviva.ai/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "d5b15183273c5bbded707077ce7b2d5463f50d086ddc9d76b56e8bb7f282379c", "bytes_read": 201 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "anthropic-ai": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Google-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "meta-externalagent": { "root_disallowed": false, "named_group": false, "matching_rules": [] } }, "llms": { "requested_url": "https://conviva.com/llms.txt", "observed_at": "2026-09-20T06:26:35.260812+00:00", "status": 200, "final_url": "https://www.conviva.ai/llms.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "f55aae90b86e6bf7b5169de0984c6a5c0c357872e3832e632037e62f13ce41a3", "bytes_read": 11292 }, "llms_state": "candidate", "control": { "requested_url": "https://conviva.com/suede-index-control-9c52d073-no-file.txt", "observed_at": "2026-09-20T06:26:35.844827+00:00", "status": 404, "final_url": "https://www.conviva.ai/suede-index-control-9c52d073-no-file.txt", "content_type": "text/html", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 } }, { "rank": 998, "domain": "transsion-os.com", "robots": { "requested_url": "https://transsion-os.com/robots.txt", "observed_at": "2026-09-20T06:26:35.096047+00:00", "status": null, "error": "URLError" }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://transsion-os.com/llms.txt", "observed_at": "2026-09-20T06:26:35.212343+00:00", "status": null, "error": "URLError" }, "llms_state": "unavailable", "control": null }, { "rank": 999, "domain": "pixiv.net", "robots": { "requested_url": "https://pixiv.net/robots.txt", "observed_at": "2026-09-20T06:26:35.213861+00:00", "status": 200, "final_url": "https://www.pixiv.net/robots.txt", "content_type": "text/plain", "truncated": false, "body_sha256": "5f437fa0d546414ad39a64f8fa36897f77d96bd967e11de8b9794b094fe2298b", "bytes_read": 1032 }, "robots_readable": true, "policies": { "GPTBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "OAI-SearchBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ChatGPT-User": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "ClaudeBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "anthropic-ai": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Claude-Web": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "PerplexityBot": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "CCBot": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Google-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "Bytespider": { "root_disallowed": false, "named_group": false, "matching_rules": [] }, "Applebot-Extended": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] }, "meta-externalagent": { "root_disallowed": true, "named_group": true, "matching_rules": [ [ "disallow", "/" ] ] } }, "llms": { "requested_url": "https://pixiv.net/llms.txt", "observed_at": "2026-09-20T06:26:36.537172+00:00", "status": 404, "final_url": "https://www.pixiv.net/llms.txt", "content_type": "text/html; charset=UTF-8", "truncated": false, "body_sha256": "dd0049baf5e1e26adc0a7d0f0a29ad6f711fefedb12745356eb8c2f3763ab0eb", "bytes_read": 1711 }, "llms_state": "not_detected", "control": null }, { "rank": 1000, "domain": "tsyndicate.com", "robots": { "requested_url": "https://tsyndicate.com/robots.txt", "observed_at": "2026-09-20T06:26:35.651162+00:00", "status": 404, "final_url": "https://tsyndicate.com/robots.txt", "content_type": "text/html", "truncated": false, "body_sha256": "55f7d9e99b8e2d4e0e193b2f0275501e6d9c1ebd29cadbea6a0da48a8587e3e0", "bytes_read": 146 }, "robots_readable": false, "policies": null, "llms": { "requested_url": "https://tsyndicate.com/llms.txt", "observed_at": "2026-09-20T06:26:35.912465+00:00", "status": 404, "final_url": "https://tsyndicate.com/llms.txt", "content_type": "text/plain; charset=utf-8", "truncated": false, "body_sha256": "0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5", "bytes_read": 9 }, "llms_state": "not_detected", "control": null } ] }