athena-oracle/reddit_proof.py

#!/usr/bin/env python3
"""
Reddit Idea Generator — Proof of Concept v5
Uses Reddit RSS feeds (Atom XML). No browser needed.
Trafilatura for clean text extraction. SQLite for storage.

Usage: python3 reddit_proof.py [count]
Example: python3 reddit_proof.py 20
"""

import sys
import json
import re
import xml.etree.ElementTree as ET
import sqlite3
import os
import time
import urllib.request
import urllib.error
from datetime import datetime, timezone
from html import unescape

import trafilatura

DB_PATH = os.path.join(os.path.dirname(__file__), "oracle.db")
SCHEMA_PATH = os.path.join(os.path.dirname(__file__), "schema.sql")

SUBREDDITS = [
    "MachineLearning", "artificial", "LocalLLaMA", "Startups",
]


def init_db():
    conn = sqlite3.connect(DB_PATH)
    with open(SCHEMA_PATH) as f:
        conn.executescript(f.read())
    conn.commit()
    return conn


def fetch_rss(subreddit, sort="hot"):
    """Fetch RSS feed for a subreddit. Returns parsed entries."""
    url = f"https://www.reddit.com/r/{subreddit}/{sort}/.rss?limit=50"
    req = urllib.request.Request(url, headers={"User-Agent": "oracle-reddit-proof/1.0"})

    for attempt in range(3):
        try:
            with urllib.request.urlopen(req, timeout=15) as resp:
                xml_data = resp.read().decode("utf-8")
            break
        except urllib.error.HTTPError as e:
            if e.code == 429:
                wait = 5 * (attempt + 1)
                print(f"    429 on r/{subreddit}, retry in {wait}s")
                time.sleep(wait)
                continue
            print(f"    RSS error r/{subreddit}: {e}")
            return []
        except Exception as e:
            print(f"    RSS error r/{subreddit}: {e}")
            return []
    else:
        print(f"    r/{subreddit}: still rate limited, skip")
        return []

    # Parse Atom XML — find all <entry> elements
    root = ET.fromstring(xml_data)
    entries = []

    # Handle namespace: Atom uses http://www.w3.org/2005/Atom
    # But ET.findall with ns prefix requires registering the namespace
    # Simpler approach: strip namespace from tags and search directly
    for entry in root.iter():
        # Get local name (strip namespace)
        tag = entry.tag.split("}")[-1] if "}" in entry.tag else entry.tag

        if tag == "entry":
            title = None
            link = None
            author = ""
            content = ""
            pub = ""
            eid = ""

            for child in entry:
                ctag = child.tag.split("}")[-1]
                if ctag == "title":
                    title = child.text
                elif ctag == "link":
                    link = child.get("href", "")
                elif ctag == "author":
                    name_el = child[0] if child else None
                    if name_el:
                        name_tag = name_el.tag.split("}")[-1]
                        if name_tag == "name":
                            author = name_el.text or ""
                elif ctag == "content":
                    content = child.text or ""
                elif ctag == "published":
                    pub = child.text or ""
                elif ctag == "id":
                    eid = child.text or ""

            if title and link:
                entries.append({
                    "title": unescape(title.strip()),
                    "url": link,
                    "author": unescape(author.strip()),
                    "content": content,
                    "published": pub,
                    "id": eid,
                    "subreddit": subreddit,
                })

    return entries


def clean_html_content(html):
    """Extract readable text from Reddit's HTML content."""
    if not html:
        return ""
    text = re.sub(r"<!--.*?-->", "", html, flags=re.DOTALL)
    text = re.sub(r"<div[^>]*>", "\n", text)
    text = re.sub(r"</div>", "\n", text)
    text = re.sub(r"<br\s*/?>", "\n", text, flags=re.I)
    text = re.sub(r"<[^>]+>", "", text)
    text = unescape(text)
    text = re.sub(r"\n\s*\n+", "\n\n", text)
    return text.strip()


def main():
    if len(sys.argv) > 1:
        count = int(sys.argv[1])
    else:
        count = 20

    print(f"=== Reddit Idea Generator — Proof of Concept v5 ===")
    print(f" count: {count}")
    print()

    conn = init_db()
    cursor = conn.cursor()

    # Step 1: Fetch RSS
    print(f"[1/3] Fetching RSS feeds...")
    all_entries = []
    seen_ids = set()

    for i, sub in enumerate(SUBREDDITS):
        entries = fetch_rss(sub)
        new = [e for e in entries if e["id"] not in seen_ids]
        seen_ids.update(e["id"] for e in new)
        all_entries.extend(new)
        if new:
            print(f"  r/{sub}: {len(new)} entries")
        # Rate limit between subreddits
        if i < len(SUBREDDITS) - 1:
            time.sleep(3)

    print(f"  Total: {len(all_entries)} entries")

    if not all_entries:
        print("\n  No entries fetched. Reddit may be rate-limiting this IP.")
        print("  Try again later or use fewer subreddits.")
        sys.exit(1)

    # Limit to count
    entries_to_store = all_entries[:count]
    print(f"  Storing {len(entries_to_store)} entries")

    # Step 2: Store
    stored = 0
    for entry in entries_to_store:
        post_id = entry["id"].replace("t3_", "")
        content_text = clean_html_content(entry["content"])

        # Signal score — RSS hot feed already sorted by relevance
        # Use position-based scoring (higher rank = higher score)
        idx = entries_to_store.index(entry)
        score = max(10.0 - idx * 0.5, 1.0)

        # Category tags
        category_tags = ["reddit"]
        sub = entry.get("subreddit", "").lower()
        if "machinelearning" in sub:
            category_tags.append("machine-learning")
        elif "artificial" in sub:
            category_tags.append("ai-general")
        elif "localllama" in sub:
            category_tags.append("local-llm")
        elif "startups" in sub:
            category_tags.append("startups")

        # Post type from title markers
        title = entry.get("title", "")
        if " [P]" in title or " [p]" in title:
            category_tags.append("project")
        elif " [R]" in title or " [r]" in title:
            category_tags.append("research")
        elif " [D]" in title or " [d]" in title:
            category_tags.append("discussion")
        elif " [N]" in title or " [n]" in title:
            category_tags.append("news")
        else:
            category_tags.append("general")

        # Clean title (remove [X] markers)
        clean_title = re.sub(r"\s*\[[A-Z]\]\s*$", "", title)

        raw_meta = {
            "subreddit": entry["subreddit"],
            "author": entry["author"],
            "published": entry["published"],
            "text_length": len(content_text),
        }

        source_id = post_id or entry["url"].split("/")[-1] or f"rss_{stored}"
        now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")

        try:
            cursor.execute("""
                INSERT OR REPLACE INTO entries
                (source, source_id, url, title, extracted_text, summary,
                 category_tags, signal_score, raw_metadata, first_seen, last_updated)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """, (
                "reddit", source_id, entry["url"], clean_title,
                content_text,
                None,  # summary — LLM later
                json.dumps(category_tags),
                score,
                json.dumps(raw_meta),
                now, now,
            ))
            stored += 1
        except Exception as e:
            print(f"  DB ERROR: {e}")

    conn.commit()
    print(f"  Stored {stored} entries")

    # Step 3: Summary
    print(f"\n[3/3] Summary")
    cursor.execute("SELECT COUNT(*) FROM entries")
    total = cursor.fetchone()[0]
    print(f"  Total entries in DB: {total}")
    cursor.execute("SELECT COUNT(*) FROM entries WHERE source='reddit'")
    reddit_count = cursor.fetchone()[0]
    print(f"  Reddit entries: {reddit_count}")
    cursor.execute("SELECT AVG(signal_score) FROM entries WHERE source='reddit'")
    avg_score = cursor.fetchone()[0] or 0
    print(f"  Avg signal score: {avg_score:.2f}")

    # Subreddit distribution
    cursor.execute("""
        SELECT raw_metadata, COUNT(*) FROM entries
        WHERE source='reddit'
        GROUP BY raw_metadata
        ORDER BY COUNT(*) DESC
    """)
    print(f"\n  Subreddit distribution:")
    for meta, cnt in cursor.fetchall():
        d = json.loads(meta)
        print(f"    r/{d.get('subreddit', '?')}: {cnt}")

    # Top 5
    print(f"\n  Top 5 by signal score:")
    cursor.execute("""
        SELECT id, title, signal_score, raw_metadata, category_tags,
               LENGTH(extracted_text) as text_len
        FROM entries WHERE source='reddit'
        ORDER BY signal_score DESC
        LIMIT 5
    """)
    for row in cursor.fetchall():
        eid, title, score, meta, tags, txt_len = row
        meta_dict = json.loads(meta) if meta else {}
        print(f"    [{eid}] score={score:.1f} text={txt_len}ch")
        print(f"        {title[:90]}")
        print(f"        r/{meta_dict.get('subreddit', '?')} "
              f"by {meta_dict.get('author', '?')}")

    # Extraction quality
    print(f"\n  Extraction quality (top entry):")
    cursor.execute("""
        SELECT title, extracted_text
        FROM entries WHERE source='reddit'
        ORDER BY signal_score DESC
        LIMIT 1
    """)
    row = cursor.fetchone()
    if row:
        title, excerpt = row
        print(f"    Title: {title[:80]}")
        print(f"    Length: {len(excerpt) if excerpt else 0} chars")
        if excerpt:
            print(f"    Preview:\n    {excerpt[:400]}...")
        else:
            print("    (empty)")

    # Check for garbled extractions
    cursor.execute("""
        SELECT COUNT(*) FROM entries
        WHERE source='reddit' AND LENGTH(extracted_text) < 100
    """)
    short_count = cursor.fetchone()[0]
    if short_count > 0:
        print(f"\n  ⚠ {short_count}/{stored} entries have very short extractions (<100 chars)")
        print("    These are likely link-only posts or external links")

    conn.close()
    print(f"\n  Database: {DB_PATH}")
    print("  Done.")


if __name__ == "__main__":
    main()