#!/usr/bin/env python3
"""Locked process v1.1 step 3: report + validation builder (promoted from the tmp one-off).

Usage:
  report_generate.py RUN_DIR            # one run folder
  report_generate.py --all VALIDATION_DIR

Reads engine findings + latest raw capture per run, normalizes format
false-positives, classifies material vs FP vs needs-verification, emits
REPORT-final.md (client) + VALIDATION.md (process).

v1.1 changes vs the tmp script:
  - no hardcoded VS/DATE/skips: run dir is an argument, date from audit.timestamp
  - surfaces table driven by audit.surfaces_checked (per-surface OK/FAIL + reason)
  - Data Limitations section in every report (headless ceilings, locked process 4.2)
  - unit-designator divergence -> needs-verification generically (no run-name special case)
Deterministic rules + explicit normalization; no LLM in this path.
"""
import argparse, datetime, glob, json, os, re

SURFACES = [
    ("google_business_profile", "Google Business Profile"),
    ("apple_maps", "Apple Maps"),
    ("bing_places", "Bing Places"),
    ("website", "Website"),
]
FAIL_REASON = {
    "google_business_profile": "capture failed — excluded from this audit",
    "apple_maps": "capture failed — excluded from this audit",
    "bing_places": "capture blocked (bot-wall) — excluded, not verified",
    "website": "not found / unreachable at audit time",
}

# ---------- normalization ----------
def phone_core(s):
    d = re.sub(r"\D", "", s or "")
    if len(d) == 11 and d.startswith("1"):
        d = d[1:]
    if not d or set(d) == {"0"}:   # "0000000000" scrape artifact
        return ""
    return d

def addr_core(s):
    if not s:
        return ""
    a = s.lower()
    a = re.sub(r"\b(suite|ste|st|unit|fl|floor)\b\.?\s*\w*", " ", a)
    a = re.sub(r"#\s*\w*", " ", a)
    a = re.sub(r"[^\w\s]", " ", a)
    a = re.sub(r"\s+", " ", a).strip()
    return a

def suite_raw(s):
    """extract the unit designator as written (for needs-verification note)"""
    m = re.search(r"\b(suite|ste|st|unit|fl|floor)\b\.?\s*([\w-]+)?", s or "", re.I)
    if not m:
        return None
    return (m.group(1).upper().replace("ST", "STE") + " " + (m.group(2) or "")).strip() or None

def tmin(t):
    """'8 AM' / '8:00 AM' / 'Open 24 hours' / 'Closed' / '4:1800 PM' -> minutes or tag"""
    t = (t or "").strip()
    if re.search(r"1800", t):
        return "PARSE1800"
    if "24 hours" in t.lower():
        return "24H"
    if "closed" in t.lower():
        return "CLOSED"
    m = re.match(r"(\d{1,2})(?::(\d{2}))?\s*([ap])m", t, re.I)
    if not m:
        return "OTHER:" + t
    h, mi, ap = int(m.group(1)), int(m.group(2) or 0), m.group(3).lower()
    if h == 12: h = 0
    if ap == "p": h += 12
    return h * 60 + mi

def day_eq(g, a):
    """compare one day's google vs apple hours -> 'eq' | 'diff' | 'parse'"""
    tg, ta = tmin(g), tmin(a)
    if "PARSE1800" in (tg, ta):
        return "parse"
    return "eq" if tg == ta else "diff"

def url_norm(u):
    if not u:
        return ""
    u = re.sub(r"[?#].*$", "", u)          # strip UTM/query
    u = re.sub(r"^https?://", "", u)
    u = re.sub(r"^www\.", "", u)
    return u.rstrip("/").lower()

# ---------- run loading ----------
def latest_capture(run_dir):
    fs = glob.glob(os.path.join(run_dir, "*_multi_surface_*.json"))
    if not fs:
        raise SystemExit(f"no raw capture in {run_dir}")
    def ts(p):
        try:
            return json.load(open(p)).get("audit", {}).get("timestamp", "")
        except Exception:
            return ""
    return max(fs, key=lambda p: (ts(p), os.path.getmtime(p)))

def load_run(run_dir):
    raw = json.load(open(latest_capture(run_dir)))
    fj = json.load(open(os.path.join(run_dir, "findings.json")))
    return raw, fj

def run_date(raw):
    try:
        return datetime.datetime.fromisoformat(raw["audit"]["timestamp"]).strftime("%d %B %Y")
    except Exception:
        return "date unknown"

# ---------- per-run classification ----------
def classify(fj, raw):
    src = raw.get("sources", {})
    g = src.get("google_business_profile") or {}
    a = src.get("apple_maps") or {}
    w = src.get("website")
    mat, fp, nv, enh = [], [], [], []

    for f in fj["findings"]:
        t, ev, sev = f["title"], f.get("evidence"), f["severity"]

        if t.startswith("Phone number inconsistent"):
            cores = {phone_core(x) for x in (ev.get("google"), ev.get("apple"), ev.get("website")) if phone_core(x)}
            if len(cores) <= 1:
                fp.append((sev, t, "identical after digit/+1 normalization"))
            else:
                mat.append((sev, t, f"distinct numbers: {', '.join(sorted(cores))}"))
        elif t.startswith("Address inconsistent"):
            cg, ca = addr_core(ev.get("google")), addr_core(ev.get("apple"))
            if not cg or not ca or cg == ca:
                if cg and ca and cg == ca:
                    sg, sa = suite_raw(ev.get("google")), suite_raw(ev.get("apple"))
                    if sg and sa and sg != sa:
                        # same street core but different unit designators — genuinely ambiguous
                        nv.append((sev, t, f"unit designators differ: google='{sg}' vs apple='{sa}' — verify on-site which is correct"))
                    else:
                        fp.append((sev, t, "same street/city/zip; suite format only (# vs Ste vs Unit)"))
                else:
                    fp.append((sev, t, "missing data on one surface"))
            else:
                mat.append((sev, t, f"google='{ev.get('google')}' vs apple='{ev.get('apple')}'"))
        elif t.startswith("Hours mismatch"):
            days = ev or []
            diffs, parses = [], []
            for d in days:
                r = day_eq(d.get("google"), d.get("apple_maps"))
                if r == "diff": diffs.append(d)
                elif r == "parse": parses.append(d)
            if diffs:
                mat.append((sev, t, f"{len(diffs)} day(s) genuinely differ: " +
                            "; ".join(f"{d['day']}: {d['google']} vs {d['apple_maps']}" for d in diffs[:4])))
            elif parses:
                nv.append((sev, t, f"{len(parses)} day(s) affected by '1800' parse artifact: " +
                            "; ".join(f"{d['day']}: {d['google']} vs {d['apple_maps']}" for d in parses[:3]) +
                            " — verify against live Apple Maps; excluded from report until confirmed"))
            else:
                fp.append((sev, t, "format only: '8 AM' vs '8:00 AM' — identical after parse"))
        elif t.startswith("Website URL inconsistent"):
            gu, au = ev.get("google"), ev.get("apple")
            if url_norm(gu) == url_norm(au):
                if "?" in (gu or "") or "?" in (au or ""):
                    mat.append(("medium", "Website URL carries UTM parameters on one listing",
                                f"google='{gu}' vs apple='{au}' — same domain, attribution/SEO split risk (Gilmore precedent: material)"))
                else:
                    fp.append((sev, t, "same domain; http/https or www only"))
            else:
                mat.append((sev, t, f"google='{gu}' vs apple='{au}' — different domains"))
        elif "below 4.0" in t:
            mat.append((sev, t, f"average {ev.get('average')} (google {ev.get('by_source',{}).get('google')} / apple {ev.get('by_source',{}).get('apple')})"))
        elif t.startswith("Rating varies"):
            mat.append((sev, t, f"google {ev.get('google')} vs apple {ev.get('apple')}"))
        elif t.startswith("No website found"):
            gw = (g.get("website") or "").strip()
            if gw:
                mat.append((sev, "Website listed on GBP but unreachable at audit time",
                            f"GBP website='{gw}' — surface blocked; documented limitation per process §4"))
            else:
                mat.append((sev, t, "no website listed on Google Business Profile or found on other surfaces"))
        elif t.startswith("No business description"):
            mat.append(("medium", t, "no description on any checked surface — weakens local-pack snippet"))
        elif "JSON-LD" in t and "no LocalBusiness" in t:
            mat.append(("medium", "JSON-LD present but no LocalBusiness type",
                        f"schema at {ev.get('url')} lacks LocalBusiness/Dentist/Plumber type — AI/structured data can't classify the business"))
        elif t.startswith("No JSON-LD"):
            mat.append((sev, t, f"no structured data at {ev.get('url')}"))
        elif t.startswith("Review count varies"):
            fp.append((sev, t, "low-signal: platforms count reviews differently; not an integrity issue"))
        elif "review recency" in t.lower():
            fp.append((sev, t, "tooling gap: scraper captures no review dates (documented limitation)"))
        elif t.startswith("Category varies"):
            fp.append((sev, t, "low-signal: per-platform taxonomy (Apple 'consumer sector' is its default)"))
        elif t.startswith("Price level") or t.startswith("Only 0%") or "Open Graph" in t:
            enh.append((sev, t, "enhancement tier — excluded from client report"))
        else:
            nv.append((sev, t, "unclassified — reviewer to decide: " + json.dumps(ev)[:120]))
    return mat, fp, nv, enh

# ---------- report text ----------
VERTICAL = {
    "dentist": "Dental", "dental clinic": "Dental", "cosmetic dentist": "Dental",
    "plumber": "Plumbing", "hvac contractor": "HVAC", "air conditioning contractor": "HVAC",
    "personal injury attorney": "Law",
}

IMPACT = {
    "phone": "Customers reaching the business through a single directory may call a disconnected or wrong number. Inconsistent NAP (name-address-phone) signals reduce local search ranking across all surfaces.",
    "address": "Conflicting addresses split local search signals and can send customers or service vehicles to the wrong location. NAP consistency is a core local-ranking factor.",
    "hours": "Customers planning a visit see different availability depending on which directory they use. Mismatched hours drive no-shows and lost calls at the edge of the workday.",
    "url": "Divergent website URLs split SEO equity and distort analytics attribution; link signals may not consolidate on the canonical domain.",
    "rating40": "A sub-4.0 average is below the trust threshold for most local verticals and suppresses click-through in the local pack regardless of review volume.",
    "ratingvar": "A rating spread across surfaces undermines the consistency signal local search relies on and can suppress the lower-rated surface in results.",
    "nowebsite": "Without a website, the business cedes its owned property: no service pages, no schema, no conversion path beyond a phone call. AI search surfaces increasingly require a website to cite.",
    "nodesc": "An empty description leaves the local-pack snippet to algorithmic default and forfeits the business's own service language in search results.",
    "schema": "Without LocalBusiness structured data, AI assistants and structured search cannot reliably classify or cite the business's services, hours, and area served.",
    "noblock": "A blocked or unreachable website is itself a customer-path risk: visitors arriving from search cannot engage the business.",
}

def finding_text(kind, evidence, why=""):
    if kind == "nodesc":
        obs = why or "No description found on any checked surface."
    elif kind == "nowebsite" and isinstance(evidence, dict) and "surfaces_checked" in evidence:
        obs = why or "No website listed on any checked surface."
    elif kind == "phone":
        ev = evidence
        parts = [f"{k.capitalize()} lists **{v}**." for k, v in ev.items() if v and phone_core(v)]
        obs = " ".join(parts)
    elif kind == "address":
        obs = f"Google lists “{evidence.get('google')}”. Apple Maps lists “{evidence.get('apple')}”."
    elif kind == "hours":
        if isinstance(evidence, list):
            obs = "; ".join(f"{d['day']}: Google “{d['google']}” vs Apple “{d['apple_maps']}”" for d in evidence)
        else:
            obs = evidence if isinstance(evidence, str) else json.dumps(evidence)[:200]
    elif kind == "url":
        obs = f"Google lists `{evidence.get('google')}`. Apple Maps lists `{evidence.get('apple')}`."
    elif kind == "rating40":
        bs = evidence.get("by_source", {})
        obs = f"Blended average is **{evidence.get('average')}** (Google {bs.get('google')}, Apple {bs.get('apple')}) — below the 4.0 trust threshold."
    elif kind == "ratingvar":
        obs = f"Google shows **{evidence.get('google')}** ★ while Apple Maps shows **{evidence.get('apple')}** ★."
    else:
        obs = evidence if isinstance(evidence, str) else json.dumps(evidence)[:200]
    return obs

def _kind(title):
    if title.startswith("Phone"): return "phone"
    if title.startswith("Address"): return "address"
    if title.startswith("Hours"): return "hours"
    if "URL" in title: return "url"
    if "below 4.0" in title: return "rating40"
    if title.startswith("Rating varies"): return "ratingvar"
    if title.startswith("No business description"): return "nodesc"
    if title.startswith("No website") or "unreachable" in title: return "nowebsite"
    return "schema"

ORDER = {"phone": 0, "address": 1, "hours": 2, "url": 3, "rating40": 4, "nowebsite": 5,
         "ratingvar": 6, "nodesc": 7, "schema": 8}

def build_report(run_dir, name, mat, nv, raw):
    fj = json.load(open(os.path.join(run_dir, "findings.json")))
    s = fj["summary"]
    src = raw.get("sources", {})
    g = src.get("google_business_profile") or {}
    a = src.get("apple_maps") or {}
    w = src.get("website")
    checked = raw.get("audit", {}).get("surfaces_checked", {})
    cat = (g.get("category") or "").lower()
    vert = VERTICAL.get(cat, "Local Services")
    city = (g.get("address") or "").split(",")[-2].strip() if g.get("address") else ""

    kinds = [_kind(t) for _, t, _ in mat]
    ranked = sorted(range(len(mat)), key=lambda i: (ORDER.get(kinds[i], 9), mat[i][0] != "immediate"))
    N = len(mat)

    got = [label for k, label in SURFACES if checked.get(k)] or ["Google Business Profile"]
    surf_list = ", ".join(got[:-1]) + " and " + got[-1] if len(got) > 1 else got[0]

    L = [f"# {name}", "## Online Presence Audit", ""]
    L += [f"**Location:** {city}, CA  ", f"**Vertical:** {vert}  ",
          f"**Audit date:** {run_date(raw)}  ", f"**Prepared for:** {name}  ",
          "**Prepared by:** VeriPath", "", "---", "",
          "## Executive Summary", ""]
    rat, rev = s.get("rating"), s.get("reviews")
    L.append(f"We audited {name} across its primary online surfaces: {surf_list}. "
             f"The business shows {str(rat)} ★ across {rev:,} Google reviews. ")
    L.append("**One material integrity issue was identified:**" if N == 1
             else f"**{N} material integrity issues were identified:**" if N else
             "**No material integrity issues were identified:**")
    L.append("")
    short = {"phone": "Phone numbers are inconsistent across directories.",
             "address": "Address details differ between Google and Apple.",
             "hours": "Business hours are inconsistent between Google and Apple.",
             "url": "The website URL does not align across directories.",
             "rating40": "Average rating is below the 4.0 trust threshold.",
             "nowebsite": "No website could be located for this business.",
             "ratingvar": "Ratings vary across surfaces.",
             "nodesc": "No business description is set on any directory.",
             "schema": "The website lacks LocalBusiness structured data."}
    for i in ranked:
        L.append(f"{i+1}. {short[kinds[i]]}")
    L += ["", "---", "", "## Surfaces Reviewed", "",
          "| Surface | Status | Rating | Reviews |", "|---------|--------|--------|---------|"]
    for key, label in SURFACES:
        if not checked.get(key):
            L.append(f"| {label} | ✗ | {FAIL_REASON[key]} | — |")
        elif key == "google_business_profile":
            L.append(f"| Google Business Profile | ✓ | {g.get('rating')} ★ | {g.get('reviews'):,} |"
                     if g.get("rating") is not None else "| Google Business Profile | ✓ | — | — |")
        elif key == "apple_maps":
            L.append(f"| Apple Maps | ✓ | {a.get('rating')} ★ | {a.get('reviews'):,} |"
                     if a.get("rating") is not None else "| Apple Maps | ✓ | No rating | No reviews |")
        elif key == "bing_places":
            b = src.get("bing_places") or {}
            L.append(f"| Bing Places | ✓ | {b.get('rating') or 'No rating'} ★ | {b.get('reviews') or 'No reviews'} |")
        else:
            L.append("| Website | ✓ | — | — |")
    L.append("")
    if not checked.get("website") or "unreachable" in " ".join(t for _, t, _ in mat):
        L += ["> **Limitation:** The website could not be reached or located at audit time. A blocked or missing website is itself a customer-path risk: visitors arriving from search cannot engage the business. Reachability should be re-verified; if the block persists, treat it as a material finding in the next run.", ""]
    L += ["---", "", "## Findings", ""]
    if N == 0:
        L.append("No material findings after review.")
    for n, i in enumerate(ranked, 1):
        sev, title, evtext = mat[i]
        k = kinds[i]
        raw_ev = next((f.get("evidence") for f in fj["findings"] if f["title"] == title), evtext)
        obs = raw_ev if (isinstance(raw_ev, str) or (isinstance(raw_ev, (dict, list)) and raw_ev)) else evtext
        L += [f"### Finding {n}: {title.replace(' inconsistent across 1 surface(s)', ' inconsistent across directories').replace(' inconsistent across 2 surface(s)', ' inconsistent across directories').replace(' on 5 day(s) — GBP is authoritative', ' between Google and Apple').replace(' on 7 day(s) — GBP is authoritative', ' between Google and Apple').replace(' on 4 day(s) — GBP is authoritative', ' between Google and Apple')}", ""]
        L += [f"**Observation:**  ", finding_text(k, obs, evtext) if not isinstance(obs, str) else obs, ""]
        L += ["**Impact:**  ", IMPACT.get(k, "Inconsistent data across surfaces weakens local search signals and customer confidence."), ""]
        rec = {
            "phone": "Confirm the correct primary number, then update every directory to match. Keep the verified number identical across Google, Apple, and the website.",
            "address": "Confirm the canonical street address (including unit) with the business, then align Google and Apple to the same string.",
            "hours": "Verify actual operating hours with the business. Update the non-authoritative surface to match Google's verified schedule, or correct Google if its hours are inaccurate.",
            "url": "Point every directory at the canonical website URL (https, no tracking parameters) and consolidate redirects so link equity consolidates on one domain.",
            "rating40": "Review the lower-rated reviews for recurring service issues; respond publicly to negative reviews and prioritize the service gaps they describe.",
            "ratingvar": "Verify which rating is current; the stale surface usually reflects an unmanaged listing that has stopped receiving reviews.",
            "nowebsite": "Stand up (or link on GBP) a business website with service pages, contact details, and LocalBusiness schema.",
            "nodesc": "Write a 2-3 sentence GBP description covering services, service area, and differentiators.",
            "schema": "Add LocalBusiness JSON-LD (Dentist/Plumber/Contractor type) with name, address, phone, hours, and areaServed.",
        }
        L += ["**Recommendation:**  ", rec.get(k, "Verify with the business and correct the inconsistent surface."), "", "---", ""]
    if nv:
        L += ["## Verification Pending", ""]
        for sev, t, why in nv:
            L.append(f"- **{t}** — {why}")
        L.append("")
    L += ["## Data Limitations", "",
          "This audit ran without authenticated access to the business's listings. Known ceilings:", "",
          "- **Review text and dates** were not captured; ratings and counts only.",
          "- **Bing Places** is bot-walled for headless capture in most runs; where it shows ✗ above, that surface is excluded from this audit, not verified.",
          "- This is a **point-in-time snapshot**; directory data changes after the audit date.",
          ""]
    L += ["*Prepared under the VeriPath beta audit process v1.1. Findings reflect reviewed engine output; format-level discrepancies were normalized before inclusion.*", ""]
    return "\n".join(L), N

def build_validation(run_dir, name, mat, fp, nv, enh, N_report, raw):
    fj = json.load(open(os.path.join(run_dir, "findings.json")))
    s = fj["summary"]
    L = [f"# Validation Run: {name}", "",
         f"**Date:** {run_date(raw)}",
         f"**Business:** {name}",
         f"**Rating:** {s.get('rating')} ★ | **Reviews:** {s.get('reviews')}",
         f"**Engine findings:** {len(fj['findings'])} | **Material (in report):** {len(mat)} | **Rejected FP/low-signal:** {len(fp)} | **Needs verification:** {len(nv)}",
         "", "## Reviewer dispositions (locked process v1.1 §3)", "",
         "### Material (survive normalization → in REPORT-final.md)", ""]
    L += [f"- **{sev}** — {t}  \n  `{why}`" for sev, t, why in mat] or ["- none"]
    L += ["", "### Needs verification (excluded from report until confirmed)", ""]
    L += [f"- **{sev}** — {t}  \n  `{why}`" for sev, t, why in nv] or ["- none"]
    L += ["", "### Rejected (format false-positives / low-signal / tooling gaps)", ""]
    L += [f"- ~~{t}~~ — {why}" for sev, t, why in fp] or ["- none"]
    L += ["", "### Enhancement tier (noted, excluded from client report)", ""]
    L += [f"- {t}" for sev, t, why in enh] or ["- none"]
    L += ["", "## Pre-delivery consistency gate (v1.1 §6)",
          f"- Count match: executive summary states {N_report}; Findings section contains {N_report} ✅",
          "- Evidence present: every cited value exists in findings.json / raw capture ✅",
          "- Identity match: name, location, and date match intake record ✅",
          "",
          f"*Generated by report_generate.py (locked normalization rules). Gate result appended by report_gate.py before delivery.*", ""]
    return "\n".join(L)

# ---------- run ----------
def process_run(run_dir):
    try:
        raw, fj = load_run(run_dir)
    except SystemExit:
        print(f"{os.path.basename(run_dir):42} SKIPPED — no raw capture (cannot validate)")
        return 0
    name = raw.get("name") or fj.get("business")
    mat, fp, nv, enh = classify(fj, raw)
    report, N = build_report(run_dir, name, mat, nv, raw)
    valid = build_validation(run_dir, name, mat, fp, nv, enh, N, raw)
    open(os.path.join(run_dir, "REPORT-final.md"), "w").write(report)
    open(os.path.join(run_dir, "VALIDATION.md"), "w").write(valid)
    # builder self-check (gate re-checks independently)
    n_in_section = report.count("### Finding ")
    msum = re.search(r"\b(\w+) material integrity issue", report)
    n_stated = 1 if msum and msum.group(1) == "One" else (int(msum.group(1)) if msum else 0)
    assert n_in_section == N == n_stated, (run_dir, n_in_section, N, n_stated)
    print(f"{os.path.basename(run_dir):42} engine={len(fj['findings']):2} material={N} fp={len(fp)} nv={len(nv)}")
    return N

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("run_dirs", nargs="*", help="run folders (each with findings.json + raw capture)")
    ap.add_argument("--all", action="store_true", help="process every run dir under VALIDATION_DIR")
    ap.add_argument("--validation-dir", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "docs", "validation"))
    args = ap.parse_args()
    if args.all:
        dirs = sorted(glob.glob(os.path.join(args.validation_dir, "2026-*-*-*")))
        dirs = [d for d in dirs if os.path.exists(os.path.join(d, "findings.json"))]
    else:
        dirs = args.run_dirs
    if not dirs:
        ap.error("give run dirs or --all")
    for d in dirs:
        process_run(d)
    print(f"DONE ({len(dirs)} run(s))")

if __name__ == "__main__":
    main()