athena-oracle/_live_compare.py

#!/usr/bin/env python3
"""Live re-fetch right now and compare to today's cron snapshot in oracle.db.
Read-only against the DB (only reads). Does NOT store anything."""
import json, math, sys, os, time
sys.path.insert(0, os.path.dirname(__file__))
from datetime import datetime, timezone
import sqlite3

import importlib

SOURCES = ["github", "arxiv", "reddit", "hackernews", "huggingface"]

ADAPTER_CLASSES = {
    "github": lambda: importlib.import_module("adapters.github").GitHubAdapter(),
    "arxiv": lambda: importlib.import_module("adapters.arxiv").ArxivAdapter(),
    "reddit": lambda: importlib.import_module("adapters.reddit").RedditAdapter(),
    "hackernews": lambda: importlib.import_module("adapters.hackernews").HackerNewsAdapter(),
    "huggingface": lambda: importlib.import_module("adapters.huggingface").HuggingFaceAdapter(),
}
NOW = datetime.now(timezone.utc)
TODAY = NOW.strftime("%Y-%m-%d")

# ---- DB: what cron already pulled today ----
db = sqlite3.connect(os.path.join(os.path.dirname(__file__), "oracle.db"))
db_rows = db.execute(
    "select source,source_id,title,url,signal_score,raw_metadata "
    "from entries where first_seen >= ?", (TODAY + "T00:00:00",)
).fetchall()
db_by_srcid = {}
for r in db_rows:
    src, sid = r[0], r[1]
    db_by_srcid.setdefault(src, {})[sid] = r
print(f"[DB today] {len(db_rows)} entries across {len(set(r[0] for r in db_rows))} sources")

# ---- LIVE re-fetch right now ----
live = {}   # source -> list of entries
fail = {}
for name in SOURCES:
    try:
        adapter = ADAPTER_CLASSES[name]()
        t0 = time.time()
        entries = adapter.fetch(limit=20)
        live[name] = entries
        print(f"[LIVE] {name:11} {len(entries):2} entries in {time.time()-t0:5.1f}s")
    except Exception as e:
        fail[name] = str(e)
        live[name] = []
        print(f"[LIVE] {name:11} FAILED: {e}")

def norm(metric, values):
    """0-100 normalization via log scale for skewed engagement."""
    vals = [v for v in values if v is not None and v >= 0]
    if not vals:
        return {}
    mx = max(vals)
    out = {}
    for k, v in metric.items():
        if v is None or v <= 0:
            out[k] = 0.0
        else:
            out[k] = round(100 * math.log(1 + v) / math.log(1 + mx), 1) if mx > 0 else 0.0
    return out

# ---- engagement extraction per source ----
def engagement(e):
    """Return (raw_engagement, metrics_dict, virality_0_100) per source."""
    src = e["source"]
    md = json.loads(e.get("raw_metadata", "{}") or "{}")
    if src == "hackernews":
        score = md.get("score", 0) or 0
        desc = md.get("descendants", 0) or 0
        eng = score + desc * 2  # comments weighted as stronger virality signal
        metrics = {"points": score, "comments": desc}
    elif src == "reddit":
        ups = md.get("ups", 0) or 0
        comments = md.get("num_comments", 0) or 0
        eng = ups + comments * 2
        metrics = {"ups": ups, "comments": comments}
    elif src == "huggingface":
        likes = md.get("likes", 0) or 0
        dl = md.get("downloads", 0) or 0
        eng = likes * 10 + dl * 0.01  # downloads are huge; down-weight
        metrics = {"likes": likes, "downloads": dl}
    elif src == "github":
        spd = md.get("stars_per_day", 0) or 0
        stars = md.get("stars", 0) or 0
        eng = spd * 50 + stars * 0.001
        metrics = {"stars/day": round(spd, 1), "stars": stars}
    elif src == "arxiv":
        eng = 0  # no engagement metrics; virality N/A, treat as research signal only
        metrics = {"categories": ", ".join(md.get("categories", [])[:3])}
    else:
        eng = 0
        metrics = {}
    return eng, metrics

# ---- Build unified compare list ----
unified = []
for src, entries in live.items():
    dbset = db_by_srcid.get(src, {})
    for e in entries:
        sid = e.get("source_id")
        eng, metrics = engagement(e)
        is_new = sid not in dbset  # NEW since cron
        unified.append({
            "source": src,
            "title": e.get("title", "")[:90],
            "url": e.get("url", ""),
            "signal": e.get("signal_score", 0),
            "engagement": eng,
            "metrics": metrics,
            "new": is_new,
            "sid": sid,
        })

# ---- Normalize virality across ALL live items (cross-source) ----
all_eng = {u["sid"]: u["engagement"] for u in unified if u["engagement"] > 0}
norm_map = norm({k: v for k, v in all_eng.items()}, list(all_eng.values()))
for u in unified:
    u["virality"] = norm_map.get(u["sid"], 0.0)

# ---- RANK by virality (desc), tiebreak signal_score ----
ranked = sorted(unified, key=lambda u: (u["virality"], u["signal"]), reverse=True)

print("\n" + "="*100)
print(f"LIVE PULL @ {NOW.strftime('%Y-%m-%d %H:%M UTC')}  |  {len(ranked)} items  |  NEW since cron: {sum(1 for u in unified if u['new'])}")
print("="*100)

# Full list
print("\n### FULL RANKED LIST (by virality)")
for i, u in enumerate(ranked, 1):
    tag = "NEW" if u["new"] else "    "
    m = " ".join(f"{k}={v}" for k, v in u["metrics"].items())
    print(f"{i:2}. [{u['virality']:5.1f}] {tag} {u['source']:10} | {u['title']}")
    if m:
        print(f"     {m}  (signal {u['signal']:.2f})")

# Trending = NEW items ranked by virality
trending = [u for u in ranked if u["new"]]
print("\n### TRENDING NOW (NEW since today's cron pull, by virality)")
if not trending:
    print("  (no new items — live pull matches today's snapshot exactly)")
for i, u in enumerate(trending, 1):
    m = " ".join(f"{k}={v}" for k, v in u["metrics"].items())
    print(f"{i:2}. [{u['virality']:5.1f}] {u['source']:10} | {u['title']}")
    if m: print(f"     {m}")

# Source-level engagement summary
print("\n### PER-SOURCE VIRALITY CEILING (top live engagement)")
by_src = {}
for u in unified:
    by_src.setdefault(u["source"], []).append(u)
for src in SOURCES:
    items = by_src.get(src, [])
    if not items:
        print(f"  {src:11}: (no live data)"); continue
    top = max(items, key=lambda x: x["virality"])
    print(f"  {src:11}: top_virality={top['virality']:.1f}  items_live={len(items)}  new={sum(1 for x in items if x['new'])}")

db.close()

# ---- Persist full list as markdown (downloadable) ----
import os as _os
out_md = f"# Athena Live Compare — {NOW.strftime('%Y-%m-%d %H:%M UTC')}\n\n"
out_md += f"- **Live pull:** {len(ranked)} items (Reddit excluded — rate-limited)\n"
out_md += f"- **NEW since today's 13:00 UTC cron:** {sum(1 for u in unified if u['new'])}\n"
out_md += f"- **Caveat:** HF scores = cumulative traction, not 24h velocity. Velocity signals = GitHub stars/day, HN points/comments.\n\n"
out_md += "## FULL RANKED LIST (by cross-source virality)\n\n"
out_md += "| # | Virality | New | Source | Title | Key metric | Signal |\n"
out_md += "|---|---|---|---|---|---|---|\n"
for i, u in enumerate(ranked, 1):
    tag = "NEW" if u["new"] else ""
    m = ", ".join(f"{k}={v}" for k, v in u["metrics"].items())
    out_md += f"| {i} | {u['virality']:.1f} | {tag} | {u['source']} | {u['title']} | {m} | {u['signal']:.2f} |\n"
out_path = _os.path.join(_os.path.dirname(__file__), f"live_compare_{NOW.strftime('%Y%m%d_%H%M')}.md")
with open(out_path, "w") as f:
    f.write(out_md)
print(f"\n[SAVED] {out_path}")