from pathlib import Path
from html.parser import HTMLParser
import urllib.request, xml.etree.ElementTree as ET, datetime, json, hashlib, argparse
class Signals(HTMLParser):
 def __init__(self):
  super().__init__(); self.canonicals=[]; self.robots=[]; self.head=False
 def handle_starttag(self,tag,attrs):
  a=dict(attrs)
  if tag=='head':self.head=True
  if tag=='link' and 'canonical' in a.get('rel','').lower().split():self.canonicals.append({'href':a.get('href'),'inHead':self.head})
  if tag=='meta' and a.get('name','').lower() in ['robots','googlebot']:self.robots.append({'name':a.get('name'),'content':a.get('content'),'inHead':self.head})
 def handle_endtag(self,tag):
  if tag=='head':self.head=False
args_parser=argparse.ArgumentParser(description='Capture declared indexing signals from up to 20 sitemap URLs. No JavaScript or index-state checks.')
args_parser.add_argument('--sitemap',default='https://aiwebsitepipeline.com/sitemap.xml',help='Sitemap URL or local file; use saved pre-sprint sitemap to repeat the original population.')
args_parser.add_argument('--output',default='indexing-signals.json')
args=args_parser.parse_args()
if args.sitemap.startswith(('https://','http://')):
 with urllib.request.urlopen(args.sitemap,timeout=30) as response: sitemap=response.read(2000000)
else: sitemap=Path(args.sitemap).read_bytes()
urls=[e.text for e in ET.fromstring(sitemap).iter() if e.tag.endswith('}loc') and e.text and e.text.startswith(('https://','http://'))]
rows=[]
for url in urls[:20]:
 row={'url':url,'capturedAt':datetime.datetime.now(datetime.timezone.utc).isoformat()}
 try:
  req=urllib.request.Request(url,headers={'User-Agent':'AIWebsitePipeline-SprintVerification/1.0'})
  with urllib.request.urlopen(req,timeout=30) as r:
   raw=r.read(2000000);row.update(status=r.status,finalUrl=r.url,xRobotsTag=r.headers.get_all('X-Robots-Tag') or [],httpLink=r.headers.get_all('Link') or [],sha256=hashlib.sha256(raw).hexdigest())
  parser=Signals();parser.feed(raw.decode('utf-8',errors='replace'));row.update(canonicalDeclarations=parser.canonicals,robotsMeta=parser.robots)
 except Exception as e:row['error']=str(e)
 rows.append(row);print(url,row.get('status'),flush=True)
report={'captureDate':datetime.datetime.now(datetime.timezone.utc).date().isoformat(),'population':'First 20 URLs at most in the supplied sitemap; this is not a representative sample of the web','method':'Python urllib fetched each URL once, followed redirects and parsed link canonical plus robots/googlebot meta with html.parser; response headers recorded. No scripts executed. Declarations are not Google-selected canonicals or indexing outcomes. Raw source bytes are hashed, not retained.','rows':rows}
report['summary']={'urlCount':len(rows),'http200Count':sum(x.get('status')==200 for x in rows),'singleCanonicalCount':sum(len(x.get('canonicalDeclarations',[]))==1 for x in rows)}
Path(args.output).write_text(json.dumps(report,indent=2))