{ "title": "How 83 major sites actually write robots.txt", "capturedAt": "2026-08-24", "instrument": "Direct HTTPS fetch of /robots.txt from each host, parsed with a matcher implementing RFC 9309 and the behaviours in google/robotstxt robots.cc", "method": [ "A fixed list of 93 well known hosts was fetched over HTTPS at /robots.txt with a 15 second timeout.", "83 returned a response body. The other 10 returned an HTTP error or refused the connection and are excluded from every figure.", "Each file was parsed into user-agent groups and allow or disallow rules. Lines that are not user-agent, allow, disallow or sitemap are counted as unknown directives, which is what Google's parser does with them.", "A sitemap line does not terminate a group, matching RFC 9309 section 2.2.4, so group counts here are the ones a crawler would actually see." ], "headline": { "sitesFetched": 83, "sitesAttempted": 93, "groups": 1023, "rules": 20703, "wildcardStarRules": 4218, "dollarAnchorRules": 9017, "sitesWithSitemap": 62, "sitesWithCrawlDelay": 15, "sitesNamingAiCrawler": 39, "largestFileBytes": 380638, "largestFileHost": "figma.com", "sitesOverGoogleLimit": 0 }, "aiCrawlers": [ [ "perplexity", 36 ], [ "ccbot", 30 ], [ "claudebot", 29 ], [ "gptbot", 29 ], [ "google-extended", 25 ], [ "omgili", 25 ], [ "meta-externalagent", 24 ], [ "bytespider", 23 ], [ "anthropic", 22 ], [ "amazonbot", 21 ], [ "applebot-extended", 21 ], [ "cohere", 19 ], [ "facebookbot", 16 ], [ "diffbot", 15 ], [ "youbot", 11 ], [ "petalbot", 10 ], [ "imagesiftbot", 9 ], [ "timpibot", 8 ], [ "img2dataset", 6 ] ], "topUserAgents": [ [ "*", 89 ], [ "ccbot", 30 ], [ "claudebot", 29 ], [ "gptbot", 29 ], [ "google-extended", 25 ], [ "chatgpt-user", 23 ], [ "meta-externalagent", 23 ], [ "perplexitybot", 23 ], [ "anthropic-ai", 22 ], [ "bytespider", 22 ], [ "applebot-extended", 21 ], [ "amazonbot", 20 ], [ "adsbot-google", 19 ], [ "oai-searchbot", 18 ], [ "twitterbot", 17 ], [ "facebookbot", 16 ], [ "bingbot", 15 ], [ "googlebot", 15 ], [ "claude-web", 15 ], [ "omgili", 15 ] ], "unknownDirectives": [ [ "crawl-delay", 20 ], [ "license", 2 ], [ "spdx-license-identifier", 2 ], [ "noindex", 2 ], [ "content-signal", 1 ], [ "