#!/usr/bin/env python3
"""Mine Athena's DB for real AI-news engagement patterns.
Goal: tell us WHAT KIND of AI news people consume/click, with numbers.
Read-only against oracle.db."""
import json, math, sqlite3, re, os
from collections import Counter, defaultdict

DB = os.path.join(os.path.dirname(__file__), "oracle.db")
c = sqlite3.connect(DB)

rows = c.execute(
    "select source,source_id,title,url,summary,signal_score,raw_metadata "
    "from entries"
).fetchall()
print(f"TOTAL ENTRIES: {len(rows)}")

# ---- engagement extractor (per source native units) ----
def eng(src, md):
    if src == "hackernews":
        return {"points": md.get("score",0), "comments": md.get("descendants",0),
                "composite": (md.get("score",0) or 0) + 2*(md.get("descendants",0) or 0)}
    if src == "reddit":
        return {"ups": md.get("ups",0), "comments": md.get("num_comments",0),
                "composite": (md.get("ups",0) or 0) + 2*(md.get("num_comments",0) or 0)}
    if src == "huggingface":
        return {"likes": md.get("likes",0), "downloads": md.get("downloads",0),
                "composite": (md.get("likes",0) or 0)*10 + (md.get("downloads",0) or 0)*0.01}
    if src == "github":
        return {"stars": md.get("stars",0), "stars_per_day": md.get("stars_per_day",0),
                "composite": (md.get("stars_per_day",0) or 0)*50 + (md.get("stars",0) or 0)*0.001}
    if src == "arxiv":
        return {"points": None, "comments": None, "composite": 0}
    return {"composite": 0}

# ---- content-type classifier (keyword on title+summary) ----
def ctype(title, summary, src):
    t = (title + " " + (summary or "")).lower()
    # order matters: most specific first
    if src == "huggingface" or re.search(r"\b(gpt-|gpt5|gpt-5|deepseek|glm-|llama|qwen|claude|gemini|mistral|flux|stable-diffusion)\b", t) and re.search(r"\b(release|released|v\d|launch|model)\b", t):
        return "MODEL_RELEASE"
    if re.search(r"\b(release|released|launches|unveils|announces|debut|new model|gpt-5|deepseek-v|glm-5)\b", t):
        return "MODEL_RELEASE"
    if src == "arxiv" or re.search(r"\b(paper|study|benchmark|arxiv|proposes|learns?|novel|framework for|towards)\b", t):
        return "RESEARCH"
    if re.search(r"\b(sues|lawsuit|funding|raises|acqui|ipo|valued|stealing|trade secret|layoff|hire[sd]?|exec|ceo|openai|anthropic|google|meta|microsoft)\b", t) and not re.search(r"\b(repo|library|tool|agent framework)\b", t):
        return "BUSINESS_LEGAL"
    if re.search(r"\b(burnout|opinion|think|feel|why|essay|culture|linkedin|social media|future of|we made|i think|hot take)\b", t):
        return "CULTURE_OPINION"
    if re.search(r"\b(how to|tutorial|guide|running|build|setup|install|from scratch|learn)\b", t):
        return "TUTORIAL_HOWTO"
    if src == "github" or re.search(r"\b(repo|library|framework|tool|agent|sdk|cli|extension|plugin|app|engine)\b", t):
        return "DEV_TOOL"
    return "OTHER"

# ---- aggregate ----
by_src = defaultdict(list)
for r in rows:
    src, sid, title, url, summary, sig, raw = r
    try: md = json.loads(raw or "{}")
    except: md = {}
    e = eng(src, md)
    ct = ctype(title, summary, src)
    by_src[src].append({"title": title, "sig": sig, "eng": e, "ct": ct, "src": src})

print("\n=== PER-SOURCE ENGAGEMENT (native units) ===")
for src, items in by_src.items():
    comps = [i["eng"]["composite"] for i in items if i["eng"]["composite"]]
    if not comps:
        print(f"  {src:11}: n={len(items)}  (no engagement metric)")
        continue
    comps.sort()
    med = comps[len(comps)//2]
    mx = max(comps)
    print(f"  {src:11}: n={len(items):3}  median_composite={med:8.1f}  max={mx:10.1f}")

print("\n=== CONTENT-TYPE MIX (all sources) ===")
ct_counter = Counter(i["ct"] for items in by_src.values() for i in items)
for ct, n in ct_counter.most_common():
    print(f"  {ct:16}: {n:3}  ({100*n/len(rows):.0f}%)")

print("\n=== ENGAGEMENT BY CONTENT-TYPE WITHIN HN (comparable units) ===")
hn = by_src["hackernews"]
hn_by_ct = defaultdict(list)
for i in hn:
    hn_by_ct[i["ct"]].append(i["eng"]["composite"])
print(f"  (HN n={len(hn)})")
for ct in sorted(hn_by_ct, key=lambda k: -max(hn_by_ct[k])):
    vals = sorted(hn_by_ct[ct])
    print(f"    {ct:16}: n={len(vals):2}  median={vals[len(vals)//2]:6.0f}  max={max(vals):6.0f}")

print("\n=== ENGAGEMENT BY CONTENT-TYPE WITHIN REDDIT ===")
rd = by_src["reddit"]
rd_by_ct = defaultdict(list)
for i in rd:
    rd_by_ct[i["ct"]].append(i["eng"]["composite"])
print(f"  (Reddit n={len(rd)})")
for ct in sorted(rd_by_ct, key=lambda k: -max(rd_by_ct[k])):
    vals = sorted(rd_by_ct[ct])
    print(f"    {ct:16}: n={len(vals):2}  median={vals[len(vals)//2]:6.0f}  max={max(vals):6.0f}")

print("\n=== GITHUB: top repos by stars/day ===")
gh = sorted(by_src["github"], key=lambda i: -(i["eng"]["stars_per_day"] or 0))[:8]
for i in gh:
    print(f"    {i['eng']['stars_per_day']:7.0f}/day  {i['eng']['stars']:6}★  {i['ct']:14} | {i['title'][:55]}")

print("\n=== HN: top 10 by composite engagement ===")
hn_top = sorted(hn, key=lambda i: -(i["eng"]["composite"] or 0))[:10]
for i in hn_top:
    print(f"    pts={i['eng']['points']:5} cmt={i['eng']['comments']:5}  [{i['ct']:14}] {i['title'][:55]}")

print("\n=== CORRELATION: Athena signal_score vs HN engagement ===")
# Does our relevance score track real clicks? (HN only, has both)
pairs = [(i["sig"], i["eng"]["composite"]) for i in hn if i["eng"]["composite"]]
if len(pairs) > 4:
    xs = [p[0] for p in pairs]; ys = [p[1] for p in pairs]
    mx, my = sum(xs)/len(xs), sum(ys)/len(ys)
    num = sum((x-mx)*(y-my) for x,y in pairs)
    den = math.sqrt(sum((x-mx)**2 for x in xs) * sum((y-my)**2 for y in ys))
    corr = num/den if den else 0
    print(f"  Pearson r (signal_score vs HN composite) = {corr:.2f}  (n={len(pairs)})")
    print(f"  -> {'signal tracks engagement' if corr>0.3 else 'signal does NOT track engagement; separate clickability score needed'}")

print("\n=== CROSS-SOURCE CORROBORATION (same story, 2+ sources) ===")
# crude: match by normalized title token overlap across sources
def toks(s):
    return set(re.findall(r"[a-z0-9]{4,}", s.lower()))
cross = 0
all_items = [i for items in by_src.values() for i in items]
for a in all_items:
    for b in all_items:
        if a["src"] >= b["src"]: continue
        ta, tb = toks(a["title"]), toks(b["title"])
        if ta and tb and len(ta & tb) >= 3:
            cross += 1
            break
print(f"  items appearing in 2+ sources (approx): {cross}")

c.close()
