#!/usr/bin/env python3 """SEMPITE Research — image AI-readiness + provenance crawl. For each site, measures whether its images are usable by AI/agents and multimodal models, and whether any carry AI-provenance metadata. Non-rendering (stdlib only), mirroring the AI crawlers (GPTBot, ClaudeBot, PerplexityBot, CCBot) that don't run JS. Per site: - images in served HTML: real src vs JS-lazy-only (data-src/lazyload) vs CSS bg - alt-text coverage (present / empty / generic) - image referenced in structured data (Product.image, ImageObject, logo, primaryImageOfPage) - PROVENANCE: samples up to K real images, fetches head bytes, scans for C2PA/Content-Credentials manifests and IPTC "DigitalSourceType" AI-disclosure. Usage: python3 img_ready.py [n] [out.tsv] [inner_path] inner_path: optional path to also fetch (e.g. a product page); default homepage only. """ import os, urllib.request, urllib.error, ssl, json, re, random, sys, concurrent.futures as cf ctx = ssl.create_default_context(); ctx.check_hostname = False; ctx.verify_mode = ssl.CERT_NONE # UA overridable via env (SEMPITE_UA) — used to get past storefront bot-gates for reachability. UA = os.environ.get("SEMPITE_UA", "Mozilla/5.0 (compatible; SempiteResearch/1.0; +https://sempite.com/research/)") IMG_SAMPLE = 5 # images per site to byte-scan for provenance IMG_HEAD_BYTES = 300000 # bytes to pull per sampled image (C2PA/XMP live near the front) LAZY_ATTRS = ["data-src", "data-lazy-src", "data-original", "data-lazy", "data-srcset", "data-echo"] GENERIC_ALT = {"", "image", "img", "photo", "picture", "logo", "icon", "banner", "thumbnail", "product", "photo of", "placeholder", "untitled", "graphic"} # provenance byte markers (case-insensitive scan on raw bytes) C2PA_MARKERS = [b"c2pa", b"jumbf", b"contentauth", b"content credentials", b"claim_generator", b"cai/", b"urn:c2pa", b"c2pa.assertions"] AI_DISCLOSURE = [b"digitalsourcetype", b"trainedalgorithmicmedia", b"compositesynthetic", b"algorithmicmedia", b"digitalcapture"] # last is benign; kept for context AI_GEN_HINTS = [b"midjourney", b"dall-e", b"dalle", b"stable diffusion", b"stablediffusion", b"firefly", b"adobe firefly", b"imagen", b"gpt-image", b"leonardo.ai", b"ideogram", b"flux.1", b"sora"] def fetch(url, limit=500000, binary=False): req = urllib.request.Request(url, headers={"User-Agent": UA}) r = urllib.request.urlopen(req, timeout=10, context=ctx) body = r.read(limit) if not binary: body = body.decode("utf-8", "replace") return r.status, dict((k.lower(), v) for k, v in r.headers.items()), body def fetch_page(url, limit=500000): for scheme_url in ([url] if url.startswith("http") else [f"https://{url}", f"http://{url}"]): try: st, h, body = fetch(scheme_url, limit) return str(st), h, body, scheme_url except urllib.error.HTTPError as e: last = "blocked" if e.code in (401,403,406,429) else str(e.code) except Exception: last = "error" return last, {}, "", url def absolutize(src, base_url): if not src: return None src = src.strip() if src.startswith("data:"): return None if src.startswith("//"): return "https:" + src if src.startswith("http"): return src m = re.match(r'(https?://[^/]+)', base_url) root = m.group(1) if m else "" if src.startswith("/"): return root + src return base_url.rsplit("/",1)[0] + "/" + src def parse_images(html): """Return (n_img, n_real_src, n_lazy_only, n_noalt, n_emptyalt, n_generic, real_src_urls).""" n=n_real=n_lazy=n_noalt=n_empty=n_generic=0 urls=[] for tag in re.findall(r']*>', html, re.I): n+=1 attrs=dict((k.lower(), (v2 or v3 or v4)) for k,v2,v3,v4 in re.findall(r'([\w:-]+)\s*=\s*(?:"([^"]*)"|\'([^\']*)\'|([^\s>]+))', tag)) src=attrs.get("src","").strip() srcset=attrs.get("srcset","").strip() src_real = src.startswith(("http","//","/")) and not src.startswith("data:") and len(src)>1 # an image is "readable" if a real URL is in src OR srcset (standard attrs a crawler parses) has_real = src_real or (srcset.startswith(("http","//","/")) and len(srcset)>1) if has_real: n_real+=1 cand = src if src_real else srcset.split()[0] urls.append(cand) elif any(attrs.get(a) for a in LAZY_ATTRS): n_lazy+=1 if "alt" not in attrs: n_noalt+=1 else: a=attrs["alt"].strip().lower() if a=="": n_empty+=1 elif a in GENERIC_ALT or len(a)<=3: n_generic+=1 return n,n_real,n_lazy,n_noalt,n_empty,n_generic,urls def has_og_image(html): return int(bool(re.search(r']+(?:property|name)\s*=\s*["\'](?:og:image|twitter:image)["\'][^>]*>', html, re.I))) def schema_has_image(html): for m in re.finditer(r']+application/ld\+json[^>]*>(.*?)', html, re.S|re.I): raw=m.group(1).strip() try: data=json.loads(raw) except Exception: try: data=json.loads(raw[:raw.rfind("}")+1]) except Exception: continue blob=json.dumps(data).lower() if '"image"' in blob or 'imageobject' in blob or 'primaryimageofpage' in blob or '"logo"' in blob: return 1 return 0 def bg_image_count(html): return len(re.findall(r'background-image\s*:\s*url\(', html, re.I)) def scan_provenance(img_urls, base_url): out=dict(sampled=0, c2pa=0, ai_disclosed=0, ai_gen_hint="") picked=[] seen=set() for s in img_urls: u=absolutize(s, base_url) if u and u not in seen and re.search(r'\.(jpg|jpeg|png|webp|avif|gif|tiff)(\?|$)', u, re.I): seen.add(u); picked.append(u) if len(picked)>=IMG_SAMPLE: break for u in picked: try: st,h,body=fetch(u, IMG_HEAD_BYTES, binary=True) except Exception: continue out["sampled"]+=1 low=body.lower() if any(mk in low for mk in C2PA_MARKERS): out["c2pa"]+=1 if any(mk in low for mk in AI_DISCLOSURE if mk!=b"digitalcapture"): out["ai_disclosed"]+=1 for mk in AI_GEN_HINTS: if mk in low: out["ai_gen_hint"]=mk.decode(); break return out def probe(dom, inner=""): out=dict(domain=dom, status="", n_img=0, real_src=0, lazy_only=0, bg_img=0, og_image=0, no_alt=0, empty_alt=0, generic_alt=0, good_alt=0, schema_image=0, img_sampled=0, c2pa=0, ai_disclosed=0, ai_gen_hint="") status,h,html,used=fetch_page(dom, 600000) out["status"]=status if status!="200" or not html: return out # optionally append an inner page's HTML for richer image coverage if inner: try: st2,h2,html2,_=fetch_page(used.rstrip("/")+inner, 600000) if st2=="200": html=html+"\n"+html2 except Exception: pass n,nr,nl,na,ne,ng,urls=parse_images(html) out.update(n_img=n, real_src=nr, lazy_only=nl, no_alt=na, empty_alt=ne, generic_alt=ng, good_alt=max(0,n-na-ne-ng), bg_img=bg_image_count(html), og_image=has_og_image(html), schema_image=schema_has_image(html)) prov=scan_provenance(urls, used) out.update(img_sampled=prov["sampled"], c2pa=prov["c2pa"], ai_disclosed=prov["ai_disclosed"], ai_gen_hint=prov["ai_gen_hint"]) return out FIELDS=["domain","status","n_img","real_src","lazy_only","bg_img","og_image","no_alt","empty_alt", "generic_alt","good_alt","schema_image","img_sampled","c2pa","ai_disclosed","ai_gen_hint"] def main(): targets=sys.argv[1] if len(sys.argv)>1 else "re-targets.txt" n=int(sys.argv[2]) if len(sys.argv)>2 else 100000 out_path=sys.argv[3] if len(sys.argv)>3 else "img-ready-results.tsv" inner=sys.argv[4] if len(sys.argv)>4 else "" doms=[]; seen=set() for l in open(targets): d=l.strip().lower().replace("https://","").replace("http://","").strip("/").split("/")[0] if d and d not in seen: seen.add(d); doms.append(d) random.seed(613); random.shuffle(doms); sample=doms[:n] total=len(sample); done=0 with open(out_path,"w") as f, cf.ThreadPoolExecutor(max_workers=16) as ex: f.write("\t".join(FIELDS)+"\n") for res in ex.map(lambda d: probe(d, inner), sample): f.write("\t".join(str(res[k]) for k in FIELDS)+"\n"); done+=1 if done%100==0: f.flush(); sys.stderr.write(f"{done}/{total}\n"); sys.stderr.flush() sys.stderr.write(f"DONE {total}\n") if __name__=="__main__": main()