Sprint 0+1: Package restructure, source tiers, verdicts, multi-variant editions
- New oracle/ package (11 modules) with unified CLI (python -m oracle) - Source tiers: Tier 1 (arxiv/github/hf), Tier 2 (rss/hn), Tier 3 (reddit) - Composite verdicts: PUBLISH/WATCH/ARCHIVE/DROP based on signal score + age - Content-hash dedup: SHA-256[:16] normalized, atomic at insert time - Multi-variant editions: 4 YAML configs (default/research/devops/brief) - Variant engine: filter → rank → render (HTML + JSON, themed) - Per-adapter timeout (10s) + threading fallback - Consolidated 12 root scripts → thin wrappers + oracle/ package - Archived stale scripts (_engagement, _live_compare, reddit_proof) - Updated .gitignore, README.md, schema.sql
This commit is contained in:
Epictetus
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Mine Athena's DB for real AI-news engagement patterns.
|
||||
Goal: tell us WHAT KIND of AI news people consume/click, with numbers.
|
||||
Read-only against oracle.db."""
|
||||
import json, math, sqlite3, re, os
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
DB = os.path.join(os.path.dirname(__file__), "oracle.db")
|
||||
c = sqlite3.connect(DB)
|
||||
|
||||
rows = c.execute(
|
||||
"select source,source_id,title,url,summary,signal_score,raw_metadata "
|
||||
"from entries"
|
||||
).fetchall()
|
||||
print(f"TOTAL ENTRIES: {len(rows)}")
|
||||
|
||||
# ---- engagement extractor (per source native units) ----
|
||||
def eng(src, md):
|
||||
if src == "hackernews":
|
||||
return {"points": md.get("score",0), "comments": md.get("descendants",0),
|
||||
"composite": (md.get("score",0) or 0) + 2*(md.get("descendants",0) or 0)}
|
||||
if src == "reddit":
|
||||
return {"ups": md.get("ups",0), "comments": md.get("num_comments",0),
|
||||
"composite": (md.get("ups",0) or 0) + 2*(md.get("num_comments",0) or 0)}
|
||||
if src == "huggingface":
|
||||
return {"likes": md.get("likes",0), "downloads": md.get("downloads",0),
|
||||
"composite": (md.get("likes",0) or 0)*10 + (md.get("downloads",0) or 0)*0.01}
|
||||
if src == "github":
|
||||
return {"stars": md.get("stars",0), "stars_per_day": md.get("stars_per_day",0),
|
||||
"composite": (md.get("stars_per_day",0) or 0)*50 + (md.get("stars",0) or 0)*0.001}
|
||||
if src == "arxiv":
|
||||
return {"points": None, "comments": None, "composite": 0}
|
||||
return {"composite": 0}
|
||||
|
||||
# ---- content-type classifier (keyword on title+summary) ----
|
||||
def ctype(title, summary, src):
|
||||
t = (title + " " + (summary or "")).lower()
|
||||
# order matters: most specific first
|
||||
if src == "huggingface" or re.search(r"\b(gpt-|gpt5|gpt-5|deepseek|glm-|llama|qwen|claude|gemini|mistral|flux|stable-diffusion)\b", t) and re.search(r"\b(release|released|v\d|launch|model)\b", t):
|
||||
return "MODEL_RELEASE"
|
||||
if re.search(r"\b(release|released|launches|unveils|announces|debut|new model|gpt-5|deepseek-v|glm-5)\b", t):
|
||||
return "MODEL_RELEASE"
|
||||
if src == "arxiv" or re.search(r"\b(paper|study|benchmark|arxiv|proposes|learns?|novel|framework for|towards)\b", t):
|
||||
return "RESEARCH"
|
||||
if re.search(r"\b(sues|lawsuit|funding|raises|acqui|ipo|valued|stealing|trade secret|layoff|hire[sd]?|exec|ceo|openai|anthropic|google|meta|microsoft)\b", t) and not re.search(r"\b(repo|library|tool|agent framework)\b", t):
|
||||
return "BUSINESS_LEGAL"
|
||||
if re.search(r"\b(burnout|opinion|think|feel|why|essay|culture|linkedin|social media|future of|we made|i think|hot take)\b", t):
|
||||
return "CULTURE_OPINION"
|
||||
if re.search(r"\b(how to|tutorial|guide|running|build|setup|install|from scratch|learn)\b", t):
|
||||
return "TUTORIAL_HOWTO"
|
||||
if src == "github" or re.search(r"\b(repo|library|framework|tool|agent|sdk|cli|extension|plugin|app|engine)\b", t):
|
||||
return "DEV_TOOL"
|
||||
return "OTHER"
|
||||
|
||||
# ---- aggregate ----
|
||||
by_src = defaultdict(list)
|
||||
for r in rows:
|
||||
src, sid, title, url, summary, sig, raw = r
|
||||
try: md = json.loads(raw or "{}")
|
||||
except: md = {}
|
||||
e = eng(src, md)
|
||||
ct = ctype(title, summary, src)
|
||||
by_src[src].append({"title": title, "sig": sig, "eng": e, "ct": ct, "src": src})
|
||||
|
||||
print("\n=== PER-SOURCE ENGAGEMENT (native units) ===")
|
||||
for src, items in by_src.items():
|
||||
comps = [i["eng"]["composite"] for i in items if i["eng"]["composite"]]
|
||||
if not comps:
|
||||
print(f" {src:11}: n={len(items)} (no engagement metric)")
|
||||
continue
|
||||
comps.sort()
|
||||
med = comps[len(comps)//2]
|
||||
mx = max(comps)
|
||||
print(f" {src:11}: n={len(items):3} median_composite={med:8.1f} max={mx:10.1f}")
|
||||
|
||||
print("\n=== CONTENT-TYPE MIX (all sources) ===")
|
||||
ct_counter = Counter(i["ct"] for items in by_src.values() for i in items)
|
||||
for ct, n in ct_counter.most_common():
|
||||
print(f" {ct:16}: {n:3} ({100*n/len(rows):.0f}%)")
|
||||
|
||||
print("\n=== ENGAGEMENT BY CONTENT-TYPE WITHIN HN (comparable units) ===")
|
||||
hn = by_src["hackernews"]
|
||||
hn_by_ct = defaultdict(list)
|
||||
for i in hn:
|
||||
hn_by_ct[i["ct"]].append(i["eng"]["composite"])
|
||||
print(f" (HN n={len(hn)})")
|
||||
for ct in sorted(hn_by_ct, key=lambda k: -max(hn_by_ct[k])):
|
||||
vals = sorted(hn_by_ct[ct])
|
||||
print(f" {ct:16}: n={len(vals):2} median={vals[len(vals)//2]:6.0f} max={max(vals):6.0f}")
|
||||
|
||||
print("\n=== ENGAGEMENT BY CONTENT-TYPE WITHIN REDDIT ===")
|
||||
rd = by_src["reddit"]
|
||||
rd_by_ct = defaultdict(list)
|
||||
for i in rd:
|
||||
rd_by_ct[i["ct"]].append(i["eng"]["composite"])
|
||||
print(f" (Reddit n={len(rd)})")
|
||||
for ct in sorted(rd_by_ct, key=lambda k: -max(rd_by_ct[k])):
|
||||
vals = sorted(rd_by_ct[ct])
|
||||
print(f" {ct:16}: n={len(vals):2} median={vals[len(vals)//2]:6.0f} max={max(vals):6.0f}")
|
||||
|
||||
print("\n=== GITHUB: top repos by stars/day ===")
|
||||
gh = sorted(by_src["github"], key=lambda i: -(i["eng"]["stars_per_day"] or 0))[:8]
|
||||
for i in gh:
|
||||
print(f" {i['eng']['stars_per_day']:7.0f}/day {i['eng']['stars']:6}★ {i['ct']:14} | {i['title'][:55]}")
|
||||
|
||||
print("\n=== HN: top 10 by composite engagement ===")
|
||||
hn_top = sorted(hn, key=lambda i: -(i["eng"]["composite"] or 0))[:10]
|
||||
for i in hn_top:
|
||||
print(f" pts={i['eng']['points']:5} cmt={i['eng']['comments']:5} [{i['ct']:14}] {i['title'][:55]}")
|
||||
|
||||
print("\n=== CORRELATION: Athena signal_score vs HN engagement ===")
|
||||
# Does our relevance score track real clicks? (HN only, has both)
|
||||
pairs = [(i["sig"], i["eng"]["composite"]) for i in hn if i["eng"]["composite"]]
|
||||
if len(pairs) > 4:
|
||||
xs = [p[0] for p in pairs]; ys = [p[1] for p in pairs]
|
||||
mx, my = sum(xs)/len(xs), sum(ys)/len(ys)
|
||||
num = sum((x-mx)*(y-my) for x,y in pairs)
|
||||
den = math.sqrt(sum((x-mx)**2 for x in xs) * sum((y-my)**2 for y in ys))
|
||||
corr = num/den if den else 0
|
||||
print(f" Pearson r (signal_score vs HN composite) = {corr:.2f} (n={len(pairs)})")
|
||||
print(f" -> {'signal tracks engagement' if corr>0.3 else 'signal does NOT track engagement; separate clickability score needed'}")
|
||||
|
||||
print("\n=== CROSS-SOURCE CORROBORATION (same story, 2+ sources) ===")
|
||||
# crude: match by normalized title token overlap across sources
|
||||
def toks(s):
|
||||
return set(re.findall(r"[a-z0-9]{4,}", s.lower()))
|
||||
cross = 0
|
||||
all_items = [i for items in by_src.values() for i in items]
|
||||
for a in all_items:
|
||||
for b in all_items:
|
||||
if a["src"] >= b["src"]: continue
|
||||
ta, tb = toks(a["title"]), toks(b["title"])
|
||||
if ta and tb and len(ta & tb) >= 3:
|
||||
cross += 1
|
||||
break
|
||||
print(f" items appearing in 2+ sources (approx): {cross}")
|
||||
|
||||
c.close()
|
||||
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live re-fetch right now and compare to today's cron snapshot in oracle.db.
|
||||
Read-only against the DB (only reads). Does NOT store anything."""
|
||||
import json, math, sys, os, time
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from datetime import datetime, timezone
|
||||
import sqlite3
|
||||
|
||||
import importlib
|
||||
|
||||
SOURCES = ["github", "arxiv", "reddit", "hackernews", "huggingface"]
|
||||
|
||||
ADAPTER_CLASSES = {
|
||||
"github": lambda: importlib.import_module("adapters.github").GitHubAdapter(),
|
||||
"arxiv": lambda: importlib.import_module("adapters.arxiv").ArxivAdapter(),
|
||||
"reddit": lambda: importlib.import_module("adapters.reddit").RedditAdapter(),
|
||||
"hackernews": lambda: importlib.import_module("adapters.hackernews").HackerNewsAdapter(),
|
||||
"huggingface": lambda: importlib.import_module("adapters.huggingface").HuggingFaceAdapter(),
|
||||
}
|
||||
NOW = datetime.now(timezone.utc)
|
||||
TODAY = NOW.strftime("%Y-%m-%d")
|
||||
|
||||
# ---- DB: what cron already pulled today ----
|
||||
db = sqlite3.connect(os.path.join(os.path.dirname(__file__), "oracle.db"))
|
||||
db_rows = db.execute(
|
||||
"select source,source_id,title,url,signal_score,raw_metadata "
|
||||
"from entries where first_seen >= ?", (TODAY + "T00:00:00",)
|
||||
).fetchall()
|
||||
db_by_srcid = {}
|
||||
for r in db_rows:
|
||||
src, sid = r[0], r[1]
|
||||
db_by_srcid.setdefault(src, {})[sid] = r
|
||||
print(f"[DB today] {len(db_rows)} entries across {len(set(r[0] for r in db_rows))} sources")
|
||||
|
||||
# ---- LIVE re-fetch right now ----
|
||||
live = {} # source -> list of entries
|
||||
fail = {}
|
||||
for name in SOURCES:
|
||||
try:
|
||||
adapter = ADAPTER_CLASSES[name]()
|
||||
t0 = time.time()
|
||||
entries = adapter.fetch(limit=20)
|
||||
live[name] = entries
|
||||
print(f"[LIVE] {name:11} {len(entries):2} entries in {time.time()-t0:5.1f}s")
|
||||
except Exception as e:
|
||||
fail[name] = str(e)
|
||||
live[name] = []
|
||||
print(f"[LIVE] {name:11} FAILED: {e}")
|
||||
|
||||
def norm(metric, values):
|
||||
"""0-100 normalization via log scale for skewed engagement."""
|
||||
vals = [v for v in values if v is not None and v >= 0]
|
||||
if not vals:
|
||||
return {}
|
||||
mx = max(vals)
|
||||
out = {}
|
||||
for k, v in metric.items():
|
||||
if v is None or v <= 0:
|
||||
out[k] = 0.0
|
||||
else:
|
||||
out[k] = round(100 * math.log(1 + v) / math.log(1 + mx), 1) if mx > 0 else 0.0
|
||||
return out
|
||||
|
||||
# ---- engagement extraction per source ----
|
||||
def engagement(e):
|
||||
"""Return (raw_engagement, metrics_dict, virality_0_100) per source."""
|
||||
src = e["source"]
|
||||
md = json.loads(e.get("raw_metadata", "{}") or "{}")
|
||||
if src == "hackernews":
|
||||
score = md.get("score", 0) or 0
|
||||
desc = md.get("descendants", 0) or 0
|
||||
eng = score + desc * 2 # comments weighted as stronger virality signal
|
||||
metrics = {"points": score, "comments": desc}
|
||||
elif src == "reddit":
|
||||
ups = md.get("ups", 0) or 0
|
||||
comments = md.get("num_comments", 0) or 0
|
||||
eng = ups + comments * 2
|
||||
metrics = {"ups": ups, "comments": comments}
|
||||
elif src == "huggingface":
|
||||
likes = md.get("likes", 0) or 0
|
||||
dl = md.get("downloads", 0) or 0
|
||||
eng = likes * 10 + dl * 0.01 # downloads are huge; down-weight
|
||||
metrics = {"likes": likes, "downloads": dl}
|
||||
elif src == "github":
|
||||
spd = md.get("stars_per_day", 0) or 0
|
||||
stars = md.get("stars", 0) or 0
|
||||
eng = spd * 50 + stars * 0.001
|
||||
metrics = {"stars/day": round(spd, 1), "stars": stars}
|
||||
elif src == "arxiv":
|
||||
eng = 0 # no engagement metrics; virality N/A, treat as research signal only
|
||||
metrics = {"categories": ", ".join(md.get("categories", [])[:3])}
|
||||
else:
|
||||
eng = 0
|
||||
metrics = {}
|
||||
return eng, metrics
|
||||
|
||||
# ---- Build unified compare list ----
|
||||
unified = []
|
||||
for src, entries in live.items():
|
||||
dbset = db_by_srcid.get(src, {})
|
||||
for e in entries:
|
||||
sid = e.get("source_id")
|
||||
eng, metrics = engagement(e)
|
||||
is_new = sid not in dbset # NEW since cron
|
||||
unified.append({
|
||||
"source": src,
|
||||
"title": e.get("title", "")[:90],
|
||||
"url": e.get("url", ""),
|
||||
"signal": e.get("signal_score", 0),
|
||||
"engagement": eng,
|
||||
"metrics": metrics,
|
||||
"new": is_new,
|
||||
"sid": sid,
|
||||
})
|
||||
|
||||
# ---- Normalize virality across ALL live items (cross-source) ----
|
||||
all_eng = {u["sid"]: u["engagement"] for u in unified if u["engagement"] > 0}
|
||||
norm_map = norm({k: v for k, v in all_eng.items()}, list(all_eng.values()))
|
||||
for u in unified:
|
||||
u["virality"] = norm_map.get(u["sid"], 0.0)
|
||||
|
||||
# ---- RANK by virality (desc), tiebreak signal_score ----
|
||||
ranked = sorted(unified, key=lambda u: (u["virality"], u["signal"]), reverse=True)
|
||||
|
||||
print("\n" + "="*100)
|
||||
print(f"LIVE PULL @ {NOW.strftime('%Y-%m-%d %H:%M UTC')} | {len(ranked)} items | NEW since cron: {sum(1 for u in unified if u['new'])}")
|
||||
print("="*100)
|
||||
|
||||
# Full list
|
||||
print("\n### FULL RANKED LIST (by virality)")
|
||||
for i, u in enumerate(ranked, 1):
|
||||
tag = "NEW" if u["new"] else " "
|
||||
m = " ".join(f"{k}={v}" for k, v in u["metrics"].items())
|
||||
print(f"{i:2}. [{u['virality']:5.1f}] {tag} {u['source']:10} | {u['title']}")
|
||||
if m:
|
||||
print(f" {m} (signal {u['signal']:.2f})")
|
||||
|
||||
# Trending = NEW items ranked by virality
|
||||
trending = [u for u in ranked if u["new"]]
|
||||
print("\n### TRENDING NOW (NEW since today's cron pull, by virality)")
|
||||
if not trending:
|
||||
print(" (no new items — live pull matches today's snapshot exactly)")
|
||||
for i, u in enumerate(trending, 1):
|
||||
m = " ".join(f"{k}={v}" for k, v in u["metrics"].items())
|
||||
print(f"{i:2}. [{u['virality']:5.1f}] {u['source']:10} | {u['title']}")
|
||||
if m: print(f" {m}")
|
||||
|
||||
# Source-level engagement summary
|
||||
print("\n### PER-SOURCE VIRALITY CEILING (top live engagement)")
|
||||
by_src = {}
|
||||
for u in unified:
|
||||
by_src.setdefault(u["source"], []).append(u)
|
||||
for src in SOURCES:
|
||||
items = by_src.get(src, [])
|
||||
if not items:
|
||||
print(f" {src:11}: (no live data)"); continue
|
||||
top = max(items, key=lambda x: x["virality"])
|
||||
print(f" {src:11}: top_virality={top['virality']:.1f} items_live={len(items)} new={sum(1 for x in items if x['new'])}")
|
||||
|
||||
db.close()
|
||||
|
||||
# ---- Persist full list as markdown (downloadable) ----
|
||||
import os as _os
|
||||
out_md = f"# Athena Live Compare — {NOW.strftime('%Y-%m-%d %H:%M UTC')}\n\n"
|
||||
out_md += f"- **Live pull:** {len(ranked)} items (Reddit excluded — rate-limited)\n"
|
||||
out_md += f"- **NEW since today's 13:00 UTC cron:** {sum(1 for u in unified if u['new'])}\n"
|
||||
out_md += f"- **Caveat:** HF scores = cumulative traction, not 24h velocity. Velocity signals = GitHub stars/day, HN points/comments.\n\n"
|
||||
out_md += "## FULL RANKED LIST (by cross-source virality)\n\n"
|
||||
out_md += "| # | Virality | New | Source | Title | Key metric | Signal |\n"
|
||||
out_md += "|---|---|---|---|---|---|---|\n"
|
||||
for i, u in enumerate(ranked, 1):
|
||||
tag = "NEW" if u["new"] else ""
|
||||
m = ", ".join(f"{k}={v}" for k, v in u["metrics"].items())
|
||||
out_md += f"| {i} | {u['virality']:.1f} | {tag} | {u['source']} | {u['title']} | {m} | {u['signal']:.2f} |\n"
|
||||
out_path = _os.path.join(_os.path.dirname(__file__), f"live_compare_{NOW.strftime('%Y%m%d_%H%M')}.md")
|
||||
with open(out_path, "w") as f:
|
||||
f.write(out_md)
|
||||
print(f"\n[SAVED] {out_path}")
|
||||
|
||||
@@ -0,0 +1,318 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Reddit Idea Generator — Proof of Concept v5
|
||||
Uses Reddit RSS feeds (Atom XML). No browser needed.
|
||||
Trafilatura for clean text extraction. SQLite for storage.
|
||||
|
||||
Usage: python3 reddit_proof.py [count]
|
||||
Example: python3 reddit_proof.py 20
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
import sqlite3
|
||||
import os
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime, timezone
|
||||
from html import unescape
|
||||
|
||||
import trafilatura
|
||||
|
||||
DB_PATH = os.path.join(os.path.dirname(__file__), "oracle.db")
|
||||
SCHEMA_PATH = os.path.join(os.path.dirname(__file__), "schema.sql")
|
||||
|
||||
SUBREDDITS = [
|
||||
"MachineLearning", "artificial", "LocalLLaMA", "Startups",
|
||||
]
|
||||
|
||||
|
||||
def init_db():
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
with open(SCHEMA_PATH) as f:
|
||||
conn.executescript(f.read())
|
||||
conn.commit()
|
||||
return conn
|
||||
|
||||
|
||||
def fetch_rss(subreddit, sort="hot"):
|
||||
"""Fetch RSS feed for a subreddit. Returns parsed entries."""
|
||||
url = f"https://www.reddit.com/r/{subreddit}/{sort}/.rss?limit=50"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "oracle-reddit-proof/1.0"})
|
||||
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
xml_data = resp.read().decode("utf-8")
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 429:
|
||||
wait = 5 * (attempt + 1)
|
||||
print(f" 429 on r/{subreddit}, retry in {wait}s")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
print(f" RSS error r/{subreddit}: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
print(f" RSS error r/{subreddit}: {e}")
|
||||
return []
|
||||
else:
|
||||
print(f" r/{subreddit}: still rate limited, skip")
|
||||
return []
|
||||
|
||||
# Parse Atom XML — find all <entry> elements
|
||||
root = ET.fromstring(xml_data)
|
||||
entries = []
|
||||
|
||||
# Handle namespace: Atom uses http://www.w3.org/2005/Atom
|
||||
# But ET.findall with ns prefix requires registering the namespace
|
||||
# Simpler approach: strip namespace from tags and search directly
|
||||
for entry in root.iter():
|
||||
# Get local name (strip namespace)
|
||||
tag = entry.tag.split("}")[-1] if "}" in entry.tag else entry.tag
|
||||
|
||||
if tag == "entry":
|
||||
title = None
|
||||
link = None
|
||||
author = ""
|
||||
content = ""
|
||||
pub = ""
|
||||
eid = ""
|
||||
|
||||
for child in entry:
|
||||
ctag = child.tag.split("}")[-1]
|
||||
if ctag == "title":
|
||||
title = child.text
|
||||
elif ctag == "link":
|
||||
link = child.get("href", "")
|
||||
elif ctag == "author":
|
||||
name_el = child[0] if child else None
|
||||
if name_el:
|
||||
name_tag = name_el.tag.split("}")[-1]
|
||||
if name_tag == "name":
|
||||
author = name_el.text or ""
|
||||
elif ctag == "content":
|
||||
content = child.text or ""
|
||||
elif ctag == "published":
|
||||
pub = child.text or ""
|
||||
elif ctag == "id":
|
||||
eid = child.text or ""
|
||||
|
||||
if title and link:
|
||||
entries.append({
|
||||
"title": unescape(title.strip()),
|
||||
"url": link,
|
||||
"author": unescape(author.strip()),
|
||||
"content": content,
|
||||
"published": pub,
|
||||
"id": eid,
|
||||
"subreddit": subreddit,
|
||||
})
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
def clean_html_content(html):
|
||||
"""Extract readable text from Reddit's HTML content."""
|
||||
if not html:
|
||||
return ""
|
||||
text = re.sub(r"<!--.*?-->", "", html, flags=re.DOTALL)
|
||||
text = re.sub(r"<div[^>]*>", "\n", text)
|
||||
text = re.sub(r"</div>", "\n", text)
|
||||
text = re.sub(r"<br\s*/?>", "\n", text, flags=re.I)
|
||||
text = re.sub(r"<[^>]+>", "", text)
|
||||
text = unescape(text)
|
||||
text = re.sub(r"\n\s*\n+", "\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) > 1:
|
||||
count = int(sys.argv[1])
|
||||
else:
|
||||
count = 20
|
||||
|
||||
print(f"=== Reddit Idea Generator — Proof of Concept v5 ===")
|
||||
print(f" count: {count}")
|
||||
print()
|
||||
|
||||
conn = init_db()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Step 1: Fetch RSS
|
||||
print(f"[1/3] Fetching RSS feeds...")
|
||||
all_entries = []
|
||||
seen_ids = set()
|
||||
|
||||
for i, sub in enumerate(SUBREDDITS):
|
||||
entries = fetch_rss(sub)
|
||||
new = [e for e in entries if e["id"] not in seen_ids]
|
||||
seen_ids.update(e["id"] for e in new)
|
||||
all_entries.extend(new)
|
||||
if new:
|
||||
print(f" r/{sub}: {len(new)} entries")
|
||||
# Rate limit between subreddits
|
||||
if i < len(SUBREDDITS) - 1:
|
||||
time.sleep(3)
|
||||
|
||||
print(f" Total: {len(all_entries)} entries")
|
||||
|
||||
if not all_entries:
|
||||
print("\n No entries fetched. Reddit may be rate-limiting this IP.")
|
||||
print(" Try again later or use fewer subreddits.")
|
||||
sys.exit(1)
|
||||
|
||||
# Limit to count
|
||||
entries_to_store = all_entries[:count]
|
||||
print(f" Storing {len(entries_to_store)} entries")
|
||||
|
||||
# Step 2: Store
|
||||
stored = 0
|
||||
for entry in entries_to_store:
|
||||
post_id = entry["id"].replace("t3_", "")
|
||||
content_text = clean_html_content(entry["content"])
|
||||
|
||||
# Signal score — RSS hot feed already sorted by relevance
|
||||
# Use position-based scoring (higher rank = higher score)
|
||||
idx = entries_to_store.index(entry)
|
||||
score = max(10.0 - idx * 0.5, 1.0)
|
||||
|
||||
# Category tags
|
||||
category_tags = ["reddit"]
|
||||
sub = entry.get("subreddit", "").lower()
|
||||
if "machinelearning" in sub:
|
||||
category_tags.append("machine-learning")
|
||||
elif "artificial" in sub:
|
||||
category_tags.append("ai-general")
|
||||
elif "localllama" in sub:
|
||||
category_tags.append("local-llm")
|
||||
elif "startups" in sub:
|
||||
category_tags.append("startups")
|
||||
|
||||
# Post type from title markers
|
||||
title = entry.get("title", "")
|
||||
if " [P]" in title or " [p]" in title:
|
||||
category_tags.append("project")
|
||||
elif " [R]" in title or " [r]" in title:
|
||||
category_tags.append("research")
|
||||
elif " [D]" in title or " [d]" in title:
|
||||
category_tags.append("discussion")
|
||||
elif " [N]" in title or " [n]" in title:
|
||||
category_tags.append("news")
|
||||
else:
|
||||
category_tags.append("general")
|
||||
|
||||
# Clean title (remove [X] markers)
|
||||
clean_title = re.sub(r"\s*\[[A-Z]\]\s*$", "", title)
|
||||
|
||||
raw_meta = {
|
||||
"subreddit": entry["subreddit"],
|
||||
"author": entry["author"],
|
||||
"published": entry["published"],
|
||||
"text_length": len(content_text),
|
||||
}
|
||||
|
||||
source_id = post_id or entry["url"].split("/")[-1] or f"rss_{stored}"
|
||||
now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
try:
|
||||
cursor.execute("""
|
||||
INSERT OR REPLACE INTO entries
|
||||
(source, source_id, url, title, extracted_text, summary,
|
||||
category_tags, signal_score, raw_metadata, first_seen, last_updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
"reddit", source_id, entry["url"], clean_title,
|
||||
content_text,
|
||||
None, # summary — LLM later
|
||||
json.dumps(category_tags),
|
||||
score,
|
||||
json.dumps(raw_meta),
|
||||
now, now,
|
||||
))
|
||||
stored += 1
|
||||
except Exception as e:
|
||||
print(f" DB ERROR: {e}")
|
||||
|
||||
conn.commit()
|
||||
print(f" Stored {stored} entries")
|
||||
|
||||
# Step 3: Summary
|
||||
print(f"\n[3/3] Summary")
|
||||
cursor.execute("SELECT COUNT(*) FROM entries")
|
||||
total = cursor.fetchone()[0]
|
||||
print(f" Total entries in DB: {total}")
|
||||
cursor.execute("SELECT COUNT(*) FROM entries WHERE source='reddit'")
|
||||
reddit_count = cursor.fetchone()[0]
|
||||
print(f" Reddit entries: {reddit_count}")
|
||||
cursor.execute("SELECT AVG(signal_score) FROM entries WHERE source='reddit'")
|
||||
avg_score = cursor.fetchone()[0] or 0
|
||||
print(f" Avg signal score: {avg_score:.2f}")
|
||||
|
||||
# Subreddit distribution
|
||||
cursor.execute("""
|
||||
SELECT raw_metadata, COUNT(*) FROM entries
|
||||
WHERE source='reddit'
|
||||
GROUP BY raw_metadata
|
||||
ORDER BY COUNT(*) DESC
|
||||
""")
|
||||
print(f"\n Subreddit distribution:")
|
||||
for meta, cnt in cursor.fetchall():
|
||||
d = json.loads(meta)
|
||||
print(f" r/{d.get('subreddit', '?')}: {cnt}")
|
||||
|
||||
# Top 5
|
||||
print(f"\n Top 5 by signal score:")
|
||||
cursor.execute("""
|
||||
SELECT id, title, signal_score, raw_metadata, category_tags,
|
||||
LENGTH(extracted_text) as text_len
|
||||
FROM entries WHERE source='reddit'
|
||||
ORDER BY signal_score DESC
|
||||
LIMIT 5
|
||||
""")
|
||||
for row in cursor.fetchall():
|
||||
eid, title, score, meta, tags, txt_len = row
|
||||
meta_dict = json.loads(meta) if meta else {}
|
||||
print(f" [{eid}] score={score:.1f} text={txt_len}ch")
|
||||
print(f" {title[:90]}")
|
||||
print(f" r/{meta_dict.get('subreddit', '?')} "
|
||||
f"by {meta_dict.get('author', '?')}")
|
||||
|
||||
# Extraction quality
|
||||
print(f"\n Extraction quality (top entry):")
|
||||
cursor.execute("""
|
||||
SELECT title, extracted_text
|
||||
FROM entries WHERE source='reddit'
|
||||
ORDER BY signal_score DESC
|
||||
LIMIT 1
|
||||
""")
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
title, excerpt = row
|
||||
print(f" Title: {title[:80]}")
|
||||
print(f" Length: {len(excerpt) if excerpt else 0} chars")
|
||||
if excerpt:
|
||||
print(f" Preview:\n {excerpt[:400]}...")
|
||||
else:
|
||||
print(" (empty)")
|
||||
|
||||
# Check for garbled extractions
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) FROM entries
|
||||
WHERE source='reddit' AND LENGTH(extracted_text) < 100
|
||||
""")
|
||||
short_count = cursor.fetchone()[0]
|
||||
if short_count > 0:
|
||||
print(f"\n ⚠ {short_count}/{stored} entries have very short extractions (<100 chars)")
|
||||
print(" These are likely link-only posts or external links")
|
||||
|
||||
conn.close()
|
||||
print(f"\n Database: {DB_PATH}")
|
||||
print(" Done.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user