#!/usr/bin/env python3
"""SEMPITE Research — real estate AI-readiness crawl (Part 2).
Measures whether independent agent/brokerage websites are readable by AI answer
engines and agents: JavaScript-invisibility of listings, real-estate structured
data, machine-readable reviews, AI-crawler robots policy, contact/entity exposure,
and IDX-platform fingerprint. Stdlib only (urllib+re) — same non-rendering posture
as the AI crawlers we're modeling (GPTBot, ClaudeBot, PerplexityBot, CCBot).

Usage:  python3 re_ready.py <targets_file> [n] [out.tsv]
Reads one domain (bare host, no scheme) per line.
"""
import urllib.request, urllib.error, ssl, json, re, random, sys, concurrent.futures as cf

ctx = ssl.create_default_context(); ctx.check_hostname = False; ctx.verify_mode = ssl.CERT_NONE
UA = "Mozilla/5.0 (compatible; SempiteResearch/1.0; +https://sempite.com/research/)"

# Bots that feed AI answers / agents. None of these render JavaScript (per Vercel/MERJ).
AGENT_BOTS = ["gptbot", "oai-searchbot", "chatgpt-user", "perplexitybot", "perplexity-user",
              "claudebot", "claude-searchbot", "anthropic-ai", "google-extended", "ccbot",
              "bytespider", "amazonbot", "applebot-extended", "meta-externalagent"]
# The four that most directly gate AI-answer inclusion today.
CORE_BOTS = ["gptbot", "oai-searchbot", "perplexitybot", "claudebot"]

# Real-estate schema.org types we look for in JSON-LD.
RE_TYPES = ["realestateagent", "realestatelisting", "residence", "singlefamilyresidence",
            "apartment", "house", "product", "offer", "aggregaterating", "review",
            "localbusiness", "organization", "realestateorganization", "person",
            "faqpage", "breadcrumblist", "webpage", "postaladdress"]

# Common IDX / real-estate website platforms — SPECIFIC footprint markers in served
# HTML (domains/product slugs only; no generic substrings that cause false positives).
IDX_VENDORS = {
    "idxbroker": ["idxbroker.com", "dsidxpress"],
    "kvcore": ["kvcore", "insiderealestate.com", "cdn.chime.me"],
    "realgeeks": ["realgeeks.com", "rgstatic"],
    "boomtown": ["boomtownroi.com"],
    "sierra": ["sierrainteractive.com"],
    "placester": ["placester.com", "placester.net"],
    "ihomefinder": ["ihomefinder.com"],
    "showcaseidx": ["showcaseidx.com"],
    "wolfnet": ["wolfnet.com"],
    "luxurypresence": ["luxurypresence.com"],
    "squarespace": ["static1.squarespace.com"],
    "wix": ["wixstatic.com", "parastorage.com"],
}

PHONE_RE = re.compile(r'(?:\+?1[\s.\-]?)?\(?\d{3}\)?[\s.\-]\d{3}[\s.\-]\d{4}')
LISTING_SIGNALS = [
    re.compile(r'\$\s?\d{3}(?:,\d{3})+'),                       # $450,000
    re.compile(r'\b\d(?:\.\d)?\s*(?:bed|bd|beds|bedroom)', re.I),
    re.compile(r'\b\d(?:\.\d)?\s*(?:bath|ba|baths|bathroom)', re.I),
    re.compile(r'\b[\d,]{3,}\s*(?:sq\.?\s?ft|sqft|square\s?f)', re.I),
    re.compile(r'\bMLS\s*#?\s*[:]?\s*\w', re.I),
]
LISTING_LINK_RE = re.compile(
    r'href=["\']([^"\']*(?:idx|listing|homes-for-sale|properties|property-search|'
    r'featured|for-sale|search|mls|homesearch)[^"\']*)["\']', re.I)
SCRIPT_RE = re.compile(r'<script[^>]*>.*?</script>', re.S | re.I)
STYLE_RE = re.compile(r'<style[^>]*>.*?</style>', re.S | re.I)
TAG_RE = re.compile(r'<[^>]+>')
SPA_ROOT_RE = re.compile(r'<div[^>]+id=["\'](?:root|app|__next|__nuxt|gatsby-focus-wrapper)["\'][^>]*>\s*</div>', re.I)

def fetch(url, limit=400000):
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    r = urllib.request.urlopen(req, timeout=10, context=ctx)
    body = r.read(limit).decode("utf-8", "replace")
    return r.status, dict((k.lower(), v) for k, v in r.headers.items()), body

def fetch_home(dom, limit=400000):
    """Try https then http; return (status_label, headers, body). status_label is an
    HTTP code as str, or an error tag (blocked/timeout/dnserr/error) so alive=0 is explainable."""
    last = "error"
    for scheme in ("https", "http"):
        try:
            st, h, body = fetch(f"{scheme}://{dom}/", limit)
            return str(st), h, body
        except urllib.error.HTTPError as e:
            last = "blocked" if e.code in (401, 403, 406, 429) else str(e.code)
        except (TimeoutError, ssl.SSLError):
            last = "timeout"
        except urllib.error.URLError as e:
            reason = str(getattr(e, "reason", "")).lower()
            last = "dnserr" if ("resolve" in reason or "name" in reason) else "urlerr"
        except Exception:
            last = "error"
    return last, {}, ""

def parse_robots(txt):
    groups = {}; cur = []
    for raw in txt.splitlines():
        line = raw.split("#", 1)[0].strip()
        if not line or ":" not in line:
            continue
        field, val = line.split(":", 1)
        field = field.strip().lower(); val = val.strip()
        if field == "user-agent":
            cur = groups.setdefault(val.lower(), [])
        elif field == "disallow" and cur is not None:
            cur.append(val)
    def blocked(ua):
        rules = groups.get(ua, groups.get("*"))
        return bool(rules) and any(p == "/" for p in rules)
    return set(b for b in AGENT_BOTS if blocked(b))

def schema_signals(html):
    """Which real-estate schema types + review data appear in JSON-LD."""
    found = set(); has_rating = False
    for m in re.finditer(r'<script[^>]+application/ld\+json[^>]*>(.*?)</script>', html, re.S | re.I):
        raw = m.group(1).strip()
        try:
            data = json.loads(raw)
        except Exception:
            try:
                data = json.loads(raw[:raw.rfind("}") + 1])
            except Exception:
                continue
        nodes = []
        def walk(x):
            if isinstance(x, dict):
                if isinstance(x.get("@graph"), list):
                    for g in x["@graph"]:
                        walk(g)
                nodes.append(x)
            elif isinstance(x, list):
                for i in x:
                    walk(i)
        walk(data)
        for n in nodes:
            t = n.get("@type", "")
            t = " ".join(t) if isinstance(t, list) else str(t)
            for part in re.split(r'[\s,]+', t.lower()):
                if part in RE_TYPES:
                    found.add(part)
            if n.get("aggregateRating") or n.get("review"):
                has_rating = True; found.add("aggregaterating")
    return found, has_rating

def visible_len(html):
    t = STYLE_RE.sub(" ", SCRIPT_RE.sub(" ", html))
    t = TAG_RE.sub(" ", t)
    return len(re.sub(r'\s+', ' ', t).strip())

def listing_signal_count(html):
    return sum(1 for rx in LISTING_SIGNALS if rx.search(html))

def detect_idx(html):
    low = html.lower()
    return [v for v, marks in IDX_VENDORS.items() if any(mk in low for mk in marks)]

def probe(dom):
    out = dict(domain=dom, alive=0, home_status="", robots=0, blocked="", allows_core=1,
               home_schema="", reviews=0, phone=0, email=0,
               listing_url=0, listing_signals=0, listing_vislen=0, spa_shell=0,
               idx_vendor="", invisible_listings=0)
    base = f"https://{dom}"
    # 1) robots.txt AI-bot policy
    try:
        st, h, body = fetch(base + "/robots.txt", 60000)
        if st == 200 and "html" not in (h.get("content-type", "") or "").lower():
            out["robots"] = 1
            blocked = parse_robots(body)
            out["blocked"] = ",".join(sorted(blocked))
            out["allows_core"] = 0 if any(b in blocked for b in CORE_BOTS) else 1
    except Exception:
        pass
    # 2) homepage (https->http fallback, explainable failure)
    status, h, home = fetch_home(dom, 400000)
    out["home_status"] = status
    if status == "200" and home:
        out["alive"] = 1
        found, rating = schema_signals(home)
        out["home_schema"] = ",".join(sorted(found))
        out["reviews"] = int(rating)
        out["phone"] = int(bool(PHONE_RE.search(home)) or "tel:" in home.lower())
        out["email"] = int("mailto:" in home.lower())
        out["idx_vendor"] = ",".join(detect_idx(home))
    if not out["alive"]:
        return out
    # 3) find + fetch a listings/search page, measure what a non-rendering crawler sees
    listing_html = None
    m = LISTING_LINK_RE.search(home)
    if m:
        href = m.group(1)
        if href.startswith("http"):
            lurl = href
        elif href.startswith("/"):
            lurl = base + href
        else:
            lurl = base + "/" + href
        try:
            st, h, listing_html = fetch(lurl, 500000)
            if st == 200 and listing_html:
                out["listing_url"] = 1
        except Exception:
            listing_html = None
    target = listing_html if listing_html else home
    out["listing_signals"] = listing_signal_count(target)
    out["listing_vislen"] = visible_len(target)
    out["spa_shell"] = int(bool(SPA_ROOT_RE.search(target)))
    if not out["idx_vendor"]:
        out["idx_vendor"] = ",".join(detect_idx(target))
    # invisible_listings: a non-rendering AI crawler sees no property data on the
    # listings surface — no listing signals AND (SPA shell OR very little text).
    out["invisible_listings"] = int(out["listing_signals"] == 0 and
                                     (out["spa_shell"] == 1 or out["listing_vislen"] < 500))
    return out

FIELDS = ["domain", "alive", "home_status", "robots", "blocked", "allows_core", "home_schema",
          "reviews", "phone", "email", "listing_url", "listing_signals", "listing_vislen",
          "spa_shell", "idx_vendor", "invisible_listings"]

def main():
    targets = sys.argv[1] if len(sys.argv) > 1 else "re-targets.txt"
    n = int(sys.argv[2]) if len(sys.argv) > 2 else 100000
    out_path = sys.argv[3] if len(sys.argv) > 3 else "re-ready-results.tsv"
    doms = []
    seen = set()
    for l in open(targets):
        d = l.strip().lower().replace("https://", "").replace("http://", "").strip("/")
        d = d.split("/")[0]
        if d and d not in seen:
            seen.add(d); doms.append(d)
    random.seed(613); random.shuffle(doms)
    sample = doms[:n]
    total = len(sample); done = 0
    with open(out_path, "w") as f, cf.ThreadPoolExecutor(max_workers=24) as ex:
        f.write("\t".join(FIELDS) + "\n")
        for res in ex.map(probe, sample):
            f.write("\t".join(str(res[k]) for k in FIELDS) + "\n")
            done += 1
            if done % 100 == 0:
                f.flush(); sys.stderr.write(f"{done}/{total}\n"); sys.stderr.flush()
    sys.stderr.write(f"DONE {total}\n")

if __name__ == "__main__":
    main()