Initial commit: Oracle AI research pipeline (adapters, pipeline, summarize, query)
Source-controlled baseline before Phase 5 cron. Excludes oracle.db, logs/, and __pycache__ via .gitignore. Pipeline verified running clean end-to-end (run_log write confirmed before conn.close()).
This commit is contained in:
Epictetus
+318
@@ -0,0 +1,318 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Reddit Idea Generator — Proof of Concept v5
|
||||
Uses Reddit RSS feeds (Atom XML). No browser needed.
|
||||
Trafilatura for clean text extraction. SQLite for storage.
|
||||
|
||||
Usage: python3 reddit_proof.py [count]
|
||||
Example: python3 reddit_proof.py 20
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
import sqlite3
|
||||
import os
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime, timezone
|
||||
from html import unescape
|
||||
|
||||
import trafilatura
|
||||
|
||||
DB_PATH = os.path.join(os.path.dirname(__file__), "oracle.db")
|
||||
SCHEMA_PATH = os.path.join(os.path.dirname(__file__), "schema.sql")
|
||||
|
||||
SUBREDDITS = [
|
||||
"MachineLearning", "artificial", "LocalLLaMA", "Startups",
|
||||
]
|
||||
|
||||
|
||||
def init_db():
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
with open(SCHEMA_PATH) as f:
|
||||
conn.executescript(f.read())
|
||||
conn.commit()
|
||||
return conn
|
||||
|
||||
|
||||
def fetch_rss(subreddit, sort="hot"):
|
||||
"""Fetch RSS feed for a subreddit. Returns parsed entries."""
|
||||
url = f"https://www.reddit.com/r/{subreddit}/{sort}/.rss?limit=50"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "oracle-reddit-proof/1.0"})
|
||||
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
xml_data = resp.read().decode("utf-8")
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 429:
|
||||
wait = 5 * (attempt + 1)
|
||||
print(f" 429 on r/{subreddit}, retry in {wait}s")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
print(f" RSS error r/{subreddit}: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
print(f" RSS error r/{subreddit}: {e}")
|
||||
return []
|
||||
else:
|
||||
print(f" r/{subreddit}: still rate limited, skip")
|
||||
return []
|
||||
|
||||
# Parse Atom XML — find all <entry> elements
|
||||
root = ET.fromstring(xml_data)
|
||||
entries = []
|
||||
|
||||
# Handle namespace: Atom uses http://www.w3.org/2005/Atom
|
||||
# But ET.findall with ns prefix requires registering the namespace
|
||||
# Simpler approach: strip namespace from tags and search directly
|
||||
for entry in root.iter():
|
||||
# Get local name (strip namespace)
|
||||
tag = entry.tag.split("}")[-1] if "}" in entry.tag else entry.tag
|
||||
|
||||
if tag == "entry":
|
||||
title = None
|
||||
link = None
|
||||
author = ""
|
||||
content = ""
|
||||
pub = ""
|
||||
eid = ""
|
||||
|
||||
for child in entry:
|
||||
ctag = child.tag.split("}")[-1]
|
||||
if ctag == "title":
|
||||
title = child.text
|
||||
elif ctag == "link":
|
||||
link = child.get("href", "")
|
||||
elif ctag == "author":
|
||||
name_el = child[0] if child else None
|
||||
if name_el:
|
||||
name_tag = name_el.tag.split("}")[-1]
|
||||
if name_tag == "name":
|
||||
author = name_el.text or ""
|
||||
elif ctag == "content":
|
||||
content = child.text or ""
|
||||
elif ctag == "published":
|
||||
pub = child.text or ""
|
||||
elif ctag == "id":
|
||||
eid = child.text or ""
|
||||
|
||||
if title and link:
|
||||
entries.append({
|
||||
"title": unescape(title.strip()),
|
||||
"url": link,
|
||||
"author": unescape(author.strip()),
|
||||
"content": content,
|
||||
"published": pub,
|
||||
"id": eid,
|
||||
"subreddit": subreddit,
|
||||
})
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
def clean_html_content(html):
|
||||
"""Extract readable text from Reddit's HTML content."""
|
||||
if not html:
|
||||
return ""
|
||||
text = re.sub(r"<!--.*?-->", "", html, flags=re.DOTALL)
|
||||
text = re.sub(r"<div[^>]*>", "\n", text)
|
||||
text = re.sub(r"</div>", "\n", text)
|
||||
text = re.sub(r"<br\s*/?>", "\n", text, flags=re.I)
|
||||
text = re.sub(r"<[^>]+>", "", text)
|
||||
text = unescape(text)
|
||||
text = re.sub(r"\n\s*\n+", "\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) > 1:
|
||||
count = int(sys.argv[1])
|
||||
else:
|
||||
count = 20
|
||||
|
||||
print(f"=== Reddit Idea Generator — Proof of Concept v5 ===")
|
||||
print(f" count: {count}")
|
||||
print()
|
||||
|
||||
conn = init_db()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Step 1: Fetch RSS
|
||||
print(f"[1/3] Fetching RSS feeds...")
|
||||
all_entries = []
|
||||
seen_ids = set()
|
||||
|
||||
for i, sub in enumerate(SUBREDDITS):
|
||||
entries = fetch_rss(sub)
|
||||
new = [e for e in entries if e["id"] not in seen_ids]
|
||||
seen_ids.update(e["id"] for e in new)
|
||||
all_entries.extend(new)
|
||||
if new:
|
||||
print(f" r/{sub}: {len(new)} entries")
|
||||
# Rate limit between subreddits
|
||||
if i < len(SUBREDDITS) - 1:
|
||||
time.sleep(3)
|
||||
|
||||
print(f" Total: {len(all_entries)} entries")
|
||||
|
||||
if not all_entries:
|
||||
print("\n No entries fetched. Reddit may be rate-limiting this IP.")
|
||||
print(" Try again later or use fewer subreddits.")
|
||||
sys.exit(1)
|
||||
|
||||
# Limit to count
|
||||
entries_to_store = all_entries[:count]
|
||||
print(f" Storing {len(entries_to_store)} entries")
|
||||
|
||||
# Step 2: Store
|
||||
stored = 0
|
||||
for entry in entries_to_store:
|
||||
post_id = entry["id"].replace("t3_", "")
|
||||
content_text = clean_html_content(entry["content"])
|
||||
|
||||
# Signal score — RSS hot feed already sorted by relevance
|
||||
# Use position-based scoring (higher rank = higher score)
|
||||
idx = entries_to_store.index(entry)
|
||||
score = max(10.0 - idx * 0.5, 1.0)
|
||||
|
||||
# Category tags
|
||||
category_tags = ["reddit"]
|
||||
sub = entry.get("subreddit", "").lower()
|
||||
if "machinelearning" in sub:
|
||||
category_tags.append("machine-learning")
|
||||
elif "artificial" in sub:
|
||||
category_tags.append("ai-general")
|
||||
elif "localllama" in sub:
|
||||
category_tags.append("local-llm")
|
||||
elif "startups" in sub:
|
||||
category_tags.append("startups")
|
||||
|
||||
# Post type from title markers
|
||||
title = entry.get("title", "")
|
||||
if " [P]" in title or " [p]" in title:
|
||||
category_tags.append("project")
|
||||
elif " [R]" in title or " [r]" in title:
|
||||
category_tags.append("research")
|
||||
elif " [D]" in title or " [d]" in title:
|
||||
category_tags.append("discussion")
|
||||
elif " [N]" in title or " [n]" in title:
|
||||
category_tags.append("news")
|
||||
else:
|
||||
category_tags.append("general")
|
||||
|
||||
# Clean title (remove [X] markers)
|
||||
clean_title = re.sub(r"\s*\[[A-Z]\]\s*$", "", title)
|
||||
|
||||
raw_meta = {
|
||||
"subreddit": entry["subreddit"],
|
||||
"author": entry["author"],
|
||||
"published": entry["published"],
|
||||
"text_length": len(content_text),
|
||||
}
|
||||
|
||||
source_id = post_id or entry["url"].split("/")[-1] or f"rss_{stored}"
|
||||
now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
try:
|
||||
cursor.execute("""
|
||||
INSERT OR REPLACE INTO entries
|
||||
(source, source_id, url, title, extracted_text, summary,
|
||||
category_tags, signal_score, raw_metadata, first_seen, last_updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
"reddit", source_id, entry["url"], clean_title,
|
||||
content_text,
|
||||
None, # summary — LLM later
|
||||
json.dumps(category_tags),
|
||||
score,
|
||||
json.dumps(raw_meta),
|
||||
now, now,
|
||||
))
|
||||
stored += 1
|
||||
except Exception as e:
|
||||
print(f" DB ERROR: {e}")
|
||||
|
||||
conn.commit()
|
||||
print(f" Stored {stored} entries")
|
||||
|
||||
# Step 3: Summary
|
||||
print(f"\n[3/3] Summary")
|
||||
cursor.execute("SELECT COUNT(*) FROM entries")
|
||||
total = cursor.fetchone()[0]
|
||||
print(f" Total entries in DB: {total}")
|
||||
cursor.execute("SELECT COUNT(*) FROM entries WHERE source='reddit'")
|
||||
reddit_count = cursor.fetchone()[0]
|
||||
print(f" Reddit entries: {reddit_count}")
|
||||
cursor.execute("SELECT AVG(signal_score) FROM entries WHERE source='reddit'")
|
||||
avg_score = cursor.fetchone()[0] or 0
|
||||
print(f" Avg signal score: {avg_score:.2f}")
|
||||
|
||||
# Subreddit distribution
|
||||
cursor.execute("""
|
||||
SELECT raw_metadata, COUNT(*) FROM entries
|
||||
WHERE source='reddit'
|
||||
GROUP BY raw_metadata
|
||||
ORDER BY COUNT(*) DESC
|
||||
""")
|
||||
print(f"\n Subreddit distribution:")
|
||||
for meta, cnt in cursor.fetchall():
|
||||
d = json.loads(meta)
|
||||
print(f" r/{d.get('subreddit', '?')}: {cnt}")
|
||||
|
||||
# Top 5
|
||||
print(f"\n Top 5 by signal score:")
|
||||
cursor.execute("""
|
||||
SELECT id, title, signal_score, raw_metadata, category_tags,
|
||||
LENGTH(extracted_text) as text_len
|
||||
FROM entries WHERE source='reddit'
|
||||
ORDER BY signal_score DESC
|
||||
LIMIT 5
|
||||
""")
|
||||
for row in cursor.fetchall():
|
||||
eid, title, score, meta, tags, txt_len = row
|
||||
meta_dict = json.loads(meta) if meta else {}
|
||||
print(f" [{eid}] score={score:.1f} text={txt_len}ch")
|
||||
print(f" {title[:90]}")
|
||||
print(f" r/{meta_dict.get('subreddit', '?')} "
|
||||
f"by {meta_dict.get('author', '?')}")
|
||||
|
||||
# Extraction quality
|
||||
print(f"\n Extraction quality (top entry):")
|
||||
cursor.execute("""
|
||||
SELECT title, extracted_text
|
||||
FROM entries WHERE source='reddit'
|
||||
ORDER BY signal_score DESC
|
||||
LIMIT 1
|
||||
""")
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
title, excerpt = row
|
||||
print(f" Title: {title[:80]}")
|
||||
print(f" Length: {len(excerpt) if excerpt else 0} chars")
|
||||
if excerpt:
|
||||
print(f" Preview:\n {excerpt[:400]}...")
|
||||
else:
|
||||
print(" (empty)")
|
||||
|
||||
# Check for garbled extractions
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) FROM entries
|
||||
WHERE source='reddit' AND LENGTH(extracted_text) < 100
|
||||
""")
|
||||
short_count = cursor.fetchone()[0]
|
||||
if short_count > 0:
|
||||
print(f"\n ⚠ {short_count}/{stored} entries have very short extractions (<100 chars)")
|
||||
print(" These are likely link-only posts or external links")
|
||||
|
||||
conn.close()
|
||||
print(f"\n Database: {DB_PATH}")
|
||||
print(" Done.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user