{
 "title": "Census host re-test, googlebot group vs star group verdicts",
 "capturedAt": "2026-08-29",
 "instrument": "The 83 census hosts re-fetched at /robots.txt over HTTPS, parsed with the same RFC 9309 and google/robotstxt semantics the page tool uses",
 "method": [
  "Each of the 83 hosts named in robots-census-2026-08-24.json perSite was fetched again on 2026-08-29.",
  "76 returned a usable robots.txt body. Hosts returning nothing or an HTML page are excluded.",
  "For every file, each allow or disallow pattern in the file, with $ removed and * replaced by the letter x, was used as a probe path.",
  "Each probe path was evaluated twice, once selecting the googlebot groups and once selecting only the * groups, both with pattern-length priority, strict-inequality ties to allow.",
  "A difference is one probe path where the two verdicts disagree."
 ],
 "totals": {
  "hostsTested": 76,
  "differences": 167,
  "sitesWithDifferences": 7
 },
 "perSite": [
  {
   "host": "nytimes.com",
   "differences": 88,
   "probePaths": 113,
   "hasGooglebotGroup": true
  },
  {
   "host": "yelp.com",
   "differences": 58,
   "probePaths": 253,
   "hasGooglebotGroup": true
  },
  {
   "host": "indeed.com",
   "differences": 16,
   "probePaths": 231,
   "hasGooglebotGroup": true
  },
  {
   "host": "wsj.com",
   "differences": 2,
   "probePaths": 51,
   "hasGooglebotGroup": true
  },
  {
   "host": "imdb.com",
   "differences": 1,
   "probePaths": 21,
   "hasGooglebotGroup": true
  },
  {
   "host": "jstor.org",
   "differences": 1,
   "probePaths": 25,
   "hasGooglebotGroup": true
  },
  {
   "host": "reuters.com",
   "differences": 1,
   "probePaths": 28,
   "hasGooglebotGroup": true
  }
 ],
 "limits": [
  "This is a re-fetch five days after the census snapshot, so the files are not byte-identical to the census-day files.",
  "The probe path set is derived from each file itself, so the count depends on that choice of paths. A different path set gives a different count.",
  "A difference says the googlebot verdict and the star-only verdict disagree for that path. It does not say which verdict the site intended."
 ]
}