WIP before Manual-Headline-Insertion v1 edits (clickability 14d lifecycle pending)
This commit is contained in:
Leonard
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live re-fetch right now and compare to today's cron snapshot in oracle.db.
|
||||
Read-only against the DB (only reads). Does NOT store anything."""
|
||||
import json, math, sys, os, time
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from datetime import datetime, timezone
|
||||
import sqlite3
|
||||
|
||||
import importlib
|
||||
|
||||
SOURCES = ["github", "arxiv", "reddit", "hackernews", "huggingface"]
|
||||
|
||||
ADAPTER_CLASSES = {
|
||||
"github": lambda: importlib.import_module("adapters.github").GitHubAdapter(),
|
||||
"arxiv": lambda: importlib.import_module("adapters.arxiv").ArxivAdapter(),
|
||||
"reddit": lambda: importlib.import_module("adapters.reddit").RedditAdapter(),
|
||||
"hackernews": lambda: importlib.import_module("adapters.hackernews").HackerNewsAdapter(),
|
||||
"huggingface": lambda: importlib.import_module("adapters.huggingface").HuggingFaceAdapter(),
|
||||
}
|
||||
NOW = datetime.now(timezone.utc)
|
||||
TODAY = NOW.strftime("%Y-%m-%d")
|
||||
|
||||
# ---- DB: what cron already pulled today ----
|
||||
db = sqlite3.connect(os.path.join(os.path.dirname(__file__), "oracle.db"))
|
||||
db_rows = db.execute(
|
||||
"select source,source_id,title,url,signal_score,raw_metadata "
|
||||
"from entries where first_seen >= ?", (TODAY + "T00:00:00",)
|
||||
).fetchall()
|
||||
db_by_srcid = {}
|
||||
for r in db_rows:
|
||||
src, sid = r[0], r[1]
|
||||
db_by_srcid.setdefault(src, {})[sid] = r
|
||||
print(f"[DB today] {len(db_rows)} entries across {len(set(r[0] for r in db_rows))} sources")
|
||||
|
||||
# ---- LIVE re-fetch right now ----
|
||||
live = {} # source -> list of entries
|
||||
fail = {}
|
||||
for name in SOURCES:
|
||||
try:
|
||||
adapter = ADAPTER_CLASSES[name]()
|
||||
t0 = time.time()
|
||||
entries = adapter.fetch(limit=20)
|
||||
live[name] = entries
|
||||
print(f"[LIVE] {name:11} {len(entries):2} entries in {time.time()-t0:5.1f}s")
|
||||
except Exception as e:
|
||||
fail[name] = str(e)
|
||||
live[name] = []
|
||||
print(f"[LIVE] {name:11} FAILED: {e}")
|
||||
|
||||
def norm(metric, values):
|
||||
"""0-100 normalization via log scale for skewed engagement."""
|
||||
vals = [v for v in values if v is not None and v >= 0]
|
||||
if not vals:
|
||||
return {}
|
||||
mx = max(vals)
|
||||
out = {}
|
||||
for k, v in metric.items():
|
||||
if v is None or v <= 0:
|
||||
out[k] = 0.0
|
||||
else:
|
||||
out[k] = round(100 * math.log(1 + v) / math.log(1 + mx), 1) if mx > 0 else 0.0
|
||||
return out
|
||||
|
||||
# ---- engagement extraction per source ----
|
||||
def engagement(e):
|
||||
"""Return (raw_engagement, metrics_dict, virality_0_100) per source."""
|
||||
src = e["source"]
|
||||
md = json.loads(e.get("raw_metadata", "{}") or "{}")
|
||||
if src == "hackernews":
|
||||
score = md.get("score", 0) or 0
|
||||
desc = md.get("descendants", 0) or 0
|
||||
eng = score + desc * 2 # comments weighted as stronger virality signal
|
||||
metrics = {"points": score, "comments": desc}
|
||||
elif src == "reddit":
|
||||
ups = md.get("ups", 0) or 0
|
||||
comments = md.get("num_comments", 0) or 0
|
||||
eng = ups + comments * 2
|
||||
metrics = {"ups": ups, "comments": comments}
|
||||
elif src == "huggingface":
|
||||
likes = md.get("likes", 0) or 0
|
||||
dl = md.get("downloads", 0) or 0
|
||||
eng = likes * 10 + dl * 0.01 # downloads are huge; down-weight
|
||||
metrics = {"likes": likes, "downloads": dl}
|
||||
elif src == "github":
|
||||
spd = md.get("stars_per_day", 0) or 0
|
||||
stars = md.get("stars", 0) or 0
|
||||
eng = spd * 50 + stars * 0.001
|
||||
metrics = {"stars/day": round(spd, 1), "stars": stars}
|
||||
elif src == "arxiv":
|
||||
eng = 0 # no engagement metrics; virality N/A, treat as research signal only
|
||||
metrics = {"categories": ", ".join(md.get("categories", [])[:3])}
|
||||
else:
|
||||
eng = 0
|
||||
metrics = {}
|
||||
return eng, metrics
|
||||
|
||||
# ---- Build unified compare list ----
|
||||
unified = []
|
||||
for src, entries in live.items():
|
||||
dbset = db_by_srcid.get(src, {})
|
||||
for e in entries:
|
||||
sid = e.get("source_id")
|
||||
eng, metrics = engagement(e)
|
||||
is_new = sid not in dbset # NEW since cron
|
||||
unified.append({
|
||||
"source": src,
|
||||
"title": e.get("title", "")[:90],
|
||||
"url": e.get("url", ""),
|
||||
"signal": e.get("signal_score", 0),
|
||||
"engagement": eng,
|
||||
"metrics": metrics,
|
||||
"new": is_new,
|
||||
"sid": sid,
|
||||
})
|
||||
|
||||
# ---- Normalize virality across ALL live items (cross-source) ----
|
||||
all_eng = {u["sid"]: u["engagement"] for u in unified if u["engagement"] > 0}
|
||||
norm_map = norm({k: v for k, v in all_eng.items()}, list(all_eng.values()))
|
||||
for u in unified:
|
||||
u["virality"] = norm_map.get(u["sid"], 0.0)
|
||||
|
||||
# ---- RANK by virality (desc), tiebreak signal_score ----
|
||||
ranked = sorted(unified, key=lambda u: (u["virality"], u["signal"]), reverse=True)
|
||||
|
||||
print("\n" + "="*100)
|
||||
print(f"LIVE PULL @ {NOW.strftime('%Y-%m-%d %H:%M UTC')} | {len(ranked)} items | NEW since cron: {sum(1 for u in unified if u['new'])}")
|
||||
print("="*100)
|
||||
|
||||
# Full list
|
||||
print("\n### FULL RANKED LIST (by virality)")
|
||||
for i, u in enumerate(ranked, 1):
|
||||
tag = "NEW" if u["new"] else " "
|
||||
m = " ".join(f"{k}={v}" for k, v in u["metrics"].items())
|
||||
print(f"{i:2}. [{u['virality']:5.1f}] {tag} {u['source']:10} | {u['title']}")
|
||||
if m:
|
||||
print(f" {m} (signal {u['signal']:.2f})")
|
||||
|
||||
# Trending = NEW items ranked by virality
|
||||
trending = [u for u in ranked if u["new"]]
|
||||
print("\n### TRENDING NOW (NEW since today's cron pull, by virality)")
|
||||
if not trending:
|
||||
print(" (no new items — live pull matches today's snapshot exactly)")
|
||||
for i, u in enumerate(trending, 1):
|
||||
m = " ".join(f"{k}={v}" for k, v in u["metrics"].items())
|
||||
print(f"{i:2}. [{u['virality']:5.1f}] {u['source']:10} | {u['title']}")
|
||||
if m: print(f" {m}")
|
||||
|
||||
# Source-level engagement summary
|
||||
print("\n### PER-SOURCE VIRALITY CEILING (top live engagement)")
|
||||
by_src = {}
|
||||
for u in unified:
|
||||
by_src.setdefault(u["source"], []).append(u)
|
||||
for src in SOURCES:
|
||||
items = by_src.get(src, [])
|
||||
if not items:
|
||||
print(f" {src:11}: (no live data)"); continue
|
||||
top = max(items, key=lambda x: x["virality"])
|
||||
print(f" {src:11}: top_virality={top['virality']:.1f} items_live={len(items)} new={sum(1 for x in items if x['new'])}")
|
||||
|
||||
db.close()
|
||||
|
||||
# ---- Persist full list as markdown (downloadable) ----
|
||||
import os as _os
|
||||
out_md = f"# Athena Live Compare — {NOW.strftime('%Y-%m-%d %H:%M UTC')}\n\n"
|
||||
out_md += f"- **Live pull:** {len(ranked)} items (Reddit excluded — rate-limited)\n"
|
||||
out_md += f"- **NEW since today's 13:00 UTC cron:** {sum(1 for u in unified if u['new'])}\n"
|
||||
out_md += f"- **Caveat:** HF scores = cumulative traction, not 24h velocity. Velocity signals = GitHub stars/day, HN points/comments.\n\n"
|
||||
out_md += "## FULL RANKED LIST (by cross-source virality)\n\n"
|
||||
out_md += "| # | Virality | New | Source | Title | Key metric | Signal |\n"
|
||||
out_md += "|---|---|---|---|---|---|---|\n"
|
||||
for i, u in enumerate(ranked, 1):
|
||||
tag = "NEW" if u["new"] else ""
|
||||
m = ", ".join(f"{k}={v}" for k, v in u["metrics"].items())
|
||||
out_md += f"| {i} | {u['virality']:.1f} | {tag} | {u['source']} | {u['title']} | {m} | {u['signal']:.2f} |\n"
|
||||
out_path = _os.path.join(_os.path.dirname(__file__), f"live_compare_{NOW.strftime('%Y%m%d_%H%M')}.md")
|
||||
with open(out_path, "w") as f:
|
||||
f.write(out_md)
|
||||
print(f"\n[SAVED] {out_path}")
|
||||
|
||||
Reference in New Issue
Block a user