#!/usr/bin/env python3
"""SEMPITE Research — agentic-commerce readiness crawl.
Measures whether real Shopify storefronts are readable/usable by AI shopping agents.
Reuses the store frame from the llms.txt Shopify wave (shopify-targets.txt).
"""
import urllib.request, urllib.error, ssl, json, re, random, sys, concurrent.futures as cf

ctx = ssl.create_default_context(); ctx.check_hostname = False; ctx.verify_mode = ssl.CERT_NONE
UA = "Mozilla/5.0 (compatible; SempiteResearch/1.0; +https://sempite.com/research/)"

# Bots an AI shopping agent / answer engine uses to read the web.
AGENT_BOTS = ["gptbot", "oai-searchbot", "chatgpt-user", "perplexitybot", "perplexity-user",
              "claudebot", "claude-searchbot", "anthropic-ai", "google-extended", "amazonbot", "applebot-extended"]
# The four that most directly gate shopping-answer inclusion today.
CORE_BOTS = ["gptbot", "oai-searchbot", "perplexitybot", "claudebot"]

APPAREL = ["shirt","tee","t-shirt","dress","hoodie","jacket","pant","trouser","jean","shoe","sneaker",
           "boot","apparel","clothing","cloth","wear","sock","hat","cap","jewelry","jewellery","ring",
           "necklace","bracelet","earring","bag","handbag","scarf","legging","swimwear","bikini","lingerie",
           "sweater","coat","skirt","short","fashion","accessor"]
HOME = ["kitchen","home","decor","mug","candle","furniture","bedding","bed","towel","cookware","kitchenware",
        "homeware","rug","pillow","cushion","lamp","vase","plate","bowl","cutlery","utensil","tableware",
        "blanket","curtain","sofa","chair","table","dinnerware","glassware","organizer","storage","garden"]

def fetch(url, limit=200000):
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    r = urllib.request.urlopen(req, timeout=9, context=ctx)
    body = r.read(limit).decode("utf-8", "replace")
    return r.status, dict((k.lower(), v) for k, v in r.headers.items()), body

def parse_robots(txt):
    """Return set of AGENT_BOTS explicitly disallowed from / (via their UA block or *)."""
    groups = {}          # ua(lower) -> list of disallow paths
    cur = []
    for raw in txt.splitlines():
        line = raw.split("#", 1)[0].strip()
        if not line:
            continue
        if ":" not in line:
            continue
        field, val = line.split(":", 1)
        field = field.strip().lower(); val = val.strip()
        if field == "user-agent":
            ua = val.lower()
            cur = groups.setdefault(ua, [])
        elif field == "disallow" and cur is not None:
            cur.append(val)
    def blocked(ua):
        rules = groups.get(ua)
        if rules is None:
            rules = groups.get("*")
        if rules is None:
            return False
        return any(p == "/" for p in rules)
    return set(b for b in AGENT_BOTS if blocked(b))

def jsonld_types(html):
    """Collect schema.org signals from JSON-LD blocks on a product page."""
    has_product = has_price = has_avail = has_rating = False
    for m in re.finditer(r'<script[^>]+application/ld\+json[^>]*>(.*?)</script>', html, re.S | re.I):
        raw = m.group(1).strip()
        try:
            data = json.loads(raw)
        except Exception:
            # tolerate trailing junk
            try:
                data = json.loads(raw[:raw.rfind("}") + 1])
            except Exception:
                continue
        nodes = []
        def walk(x):
            if isinstance(x, dict):
                if "@graph" in x and isinstance(x["@graph"], list):
                    for g in x["@graph"]:
                        walk(g)
                nodes.append(x)
            elif isinstance(x, list):
                for i in x:
                    walk(i)
        walk(data)
        for n in nodes:
            t = n.get("@type", "")
            t = " ".join(t) if isinstance(t, list) else str(t)
            tl = t.lower()
            if "product" in tl:
                has_product = True
                offers = n.get("offers")
                offs = offers if isinstance(offers, list) else [offers] if offers else []
                for o in offs:
                    if isinstance(o, dict):
                        if o.get("price") or o.get("lowPrice") or (o.get("priceSpecification")):
                            has_price = True
                        if o.get("availability"):
                            has_avail = True
                if n.get("aggregateRating") or n.get("review"):
                    has_rating = True
            if "aggregaterating" in tl:
                has_rating = True
    return has_product, has_price, has_avail, has_rating

def vertical(text):
    t = text.lower()
    a = sum(1 for k in APPAREL if k in t)
    h = sum(1 for k in HOME if k in t)
    if a == 0 and h == 0:
        return "other"
    return "apparel" if a >= h else "home_kitchen"

def probe(dom):
    out = dict(domain=dom, alive=0, robots=0, blocked="", allows_core=0, catalog=0,
               product_schema=0, price=0, availability=0, rating=0, llms=0, vertical="", nprod=0)
    base = f"https://{dom}"
    # 1) robots.txt
    try:
        st, h, body = fetch(base + "/robots.txt", 60000)
        if st == 200 and "html" not in (h.get("content-type", "") or "").lower():
            out["robots"] = 1
            blocked = parse_robots(body)
            out["blocked"] = ",".join(sorted(blocked))
            out["allows_core"] = 0 if any(b in blocked for b in CORE_BOTS) else 1
        else:
            out["allows_core"] = 1  # no usable robots => not blocked
    except urllib.error.HTTPError:
        out["allows_core"] = 1
    except Exception:
        pass
    # 2) products.json (machine-readable catalog) + vertical + a product handle
    handle = None; vtext = ""
    try:
        st, h, body = fetch(base + "/products.json?limit=5", 120000)
        if st == 200 and body.strip().startswith("{"):
            data = json.loads(body)
            prods = data.get("products", [])
            if isinstance(prods, list) and prods:
                out["catalog"] = 1; out["alive"] = 1; out["nprod"] = len(prods)
                handle = prods[0].get("handle")
                for p in prods:
                    vtext += " " + str(p.get("product_type", "")) + " " + str(p.get("title", "")) + " " + " ".join(p.get("tags", []) if isinstance(p.get("tags"), list) else [str(p.get("tags", ""))])
                out["vertical"] = vertical(vtext)
    except Exception:
        pass
    # 3) product page JSON-LD
    if handle:
        try:
            st, h, body = fetch(base + f"/products/{handle}", 400000)
            if st == 200:
                out["alive"] = 1
                pr, pc, av, rt = jsonld_types(body)
                out["product_schema"] = int(pr); out["price"] = int(pc)
                out["availability"] = int(av); out["rating"] = int(rt)
        except Exception:
            pass
    # 4) llms.txt
    try:
        st, h, body = fetch(base + "/llms.txt", 8000)
        if st == 200 and "html" not in (h.get("content-type", "") or "").lower() and len(body.strip()) >= 25:
            out["llms"] = 1; out["alive"] = 1
    except Exception:
        pass
    return out

FIELDS = ["domain","alive","robots","blocked","allows_core","catalog","product_schema",
          "price","availability","rating","llms","vertical","nprod"]

def main():
    n = int(sys.argv[1]) if len(sys.argv) > 1 else 1600
    doms = [l.strip() for l in open("shopify-targets.txt") if l.strip()]
    random.seed(613)
    random.shuffle(doms)
    sample = doms[:n]
    total = len(sample); done = 0
    with open("agent-ready-results.tsv", "w") as f, cf.ThreadPoolExecutor(max_workers=30) as ex:
        f.write("\t".join(FIELDS) + "\n")
        for res in ex.map(probe, sample):
            f.write("\t".join(str(res[k]) for k in FIELDS) + "\n")
            done += 1
            if done % 200 == 0:
                f.flush(); sys.stderr.write(f"{done}/{total}\n"); sys.stderr.flush()
    sys.stderr.write(f"DONE {total}\n")

if __name__ == "__main__":
    main()