refactor(adapters): centralize curation in config/queries.json (issue #7)
- adapters/__init__.py: add load_queries() + source_config() (stdlib json,
safe fallback to {} on missing/corrupt config so pipeline never crashes).
- config/queries.json: per-adapter blocks (hackernews.keywords, arxiv.categories,
reddit.subreddits, rss.feeds+keywords, github.search_terms). JSON (not
yaml) to honor Athena's dependency-free runtime; PyYAML avoided.
- hackernews: drop class AI_KEYWORDS + the DUPLICATE inline list inside
_is_ai_relevant() (the internal drift Ty flagged). Now loads self.ai_keywords
from config. Simplified matching to single substring pass (boundary variants
'ai ',' ai','ai-','-ai' approximate word-boundary; dropped the niche
'compute+tech-context' guard as not worth centralizing).
- reddit: DEFAULT_SUBREDDITS kept as fallback; __init__ prefers config.
- arxiv: DEFAULT_CATEGORIES kept as fallback; __init__ prefers config.
- rss: FEEDS + AI_KEYWORDS kept as module fallbacks; __init__ prefers config.
Keywords stay regex form (re.search) as in original.
- github: trending queries moved to config search_terms; fallback retained.
- DELETE reddit_proof.py: standalone PoC v5 at repo root, own main()+init_db()
+ direct INSERT OR REPLACE, NOT in cron, NOT imported anywhere -> dead
code. Also removes its byte-duplicate SUBREDDITS.
NOTE: fallback class constants remain intentionally (issue #7 cut #5: safe
rollout). Curation VALUES now live in one file; the constants are inert
unless config/queries.json is missing.
Verified: all adapters compile; config loads (HN 46 kw, RSS 10 feeds);
full dry-run fetches all 6 sources; grep confirms HN internal dup list gone.
This commit is contained in:
Epictetus
@@ -1,7 +1,35 @@
|
||||
"""Source adapters for AI Research Oracle."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
# Centralized curation config (issue #7). One file, per-adapter blocks.
|
||||
# Stdlib-only (JSON, not YAML) to honor Athena's dependency-free runtime.
|
||||
_QUERIES_PATH = os.path.join(
|
||||
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
|
||||
"config", "queries.json")
|
||||
|
||||
|
||||
def load_queries():
|
||||
"""Load config/queries.json. Returns {'sources': {...}}.
|
||||
|
||||
Safe fallback: if the file is missing/corrupt, returns an empty
|
||||
{'sources': {}} so adapters fall back to their class defaults
|
||||
(constructor None-override) instead of crashing the pipeline.
|
||||
"""
|
||||
try:
|
||||
with open(_QUERIES_PATH) as f:
|
||||
data = json.load(f)
|
||||
return data if isinstance(data, dict) else {"sources": {}}
|
||||
except Exception:
|
||||
return {"sources": {}}
|
||||
|
||||
|
||||
def source_config(name: str) -> dict:
|
||||
"""Return the per-adapter block for `name`, or {} if absent."""
|
||||
return load_queries().get("sources", {}).get(name, {}) or {}
|
||||
|
||||
|
||||
class SourceAdapter(ABC):
|
||||
"""Base class for all ingestion adapters."""
|
||||
|
||||
+3
-7
@@ -32,7 +32,7 @@ import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from html import unescape
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters import SourceAdapter, source_config
|
||||
|
||||
# arXiv API
|
||||
ARXIV_API = "http://export.arxiv.org/api/query"
|
||||
@@ -45,12 +45,8 @@ class ArxivAdapter(SourceAdapter):
|
||||
DEFAULT_CATEGORIES = ["cs.AI", "cs.LG", "cs.CL"]
|
||||
|
||||
def __init__(self, categories=None, rate_limit=3):
|
||||
"""
|
||||
Args:
|
||||
categories: List of arXiv categories. Default: cs.AI, cs.LG, cs.CL
|
||||
rate_limit: Seconds between API calls (default 3).
|
||||
"""
|
||||
self.categories = categories or self.DEFAULT_CATEGORIES
|
||||
cfg = source_config("arxiv")
|
||||
self.categories = categories or cfg.get("categories") or self.DEFAULT_CATEGORIES
|
||||
self.rate_limit = rate_limit
|
||||
|
||||
def name(self) -> str:
|
||||
|
||||
+8
-7
@@ -17,7 +17,7 @@ import urllib.error
|
||||
import urllib.parse
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters import SourceAdapter, source_config
|
||||
|
||||
|
||||
class GitHubAdapter(SourceAdapter):
|
||||
@@ -29,6 +29,11 @@ class GitHubAdapter(SourceAdapter):
|
||||
"""Initialize with optional read-only token (5000 req/hr vs 60)."""
|
||||
self.token = token or os.environ.get("GITHUB_TOKEN", "")
|
||||
self.cache = {}
|
||||
# Curation centralized (issue #7): trending queries from config
|
||||
cfg = source_config("github")
|
||||
self.search_terms = cfg.get("search_terms") or [
|
||||
"ai agent", "llm OR inference OR rag", "autonomous agent OR AI tool",
|
||||
]
|
||||
|
||||
def name(self) -> str:
|
||||
return "github"
|
||||
@@ -158,12 +163,8 @@ class GitHubAdapter(SourceAdapter):
|
||||
cutoff = (now - timedelta(days=30)).strftime("%Y-%m-%d")
|
||||
# Three queries for breadth: agents, LLM/infra, and security/tools
|
||||
repos = []
|
||||
for q in [
|
||||
f"ai agent created:>{cutoff}",
|
||||
f"llm OR inference OR rag created:>{cutoff}",
|
||||
f"autonomous agent OR AI tool created:>{cutoff}",
|
||||
]:
|
||||
batch = self._search_repos(q, sort="stars", per_page=30)
|
||||
for q in self.search_terms:
|
||||
batch = self._search_repos(f"{q} created:>{cutoff}", sort="stars", per_page=30)
|
||||
repos.extend(batch)
|
||||
time.sleep(1) # polite spacing
|
||||
|
||||
|
||||
+8
-48
@@ -20,7 +20,7 @@ import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters import SourceAdapter, source_config
|
||||
|
||||
|
||||
class HackerNewsAdapter(SourceAdapter):
|
||||
@@ -28,27 +28,11 @@ class HackerNewsAdapter(SourceAdapter):
|
||||
|
||||
BASE = "https://hacker-news.firebaseio.com/v0"
|
||||
|
||||
AI_KEYWORDS = [
|
||||
# Multi-word phrases (unambiguous)
|
||||
"language model", "deep learning", "foundation model", "retrieval augmented",
|
||||
"code generation", "context length", "context window", "attention mechanism",
|
||||
# Compound/abbreviations (unambiguous)
|
||||
"llm", "gpt-", "gpt ", "rag ", "rag.", "vlm", "vla",
|
||||
# Specific company/product names
|
||||
"openai", "anthropic", "deepseek", "meta ai", "xai", "ponytail",
|
||||
# Topic-specific (with word boundary awareness in _is_ai_relevant)
|
||||
"inference", "transformer", "diffusion", "alignment", "fine-tun",
|
||||
"embedd", "pretrain", "post-train", "multimodal", "reasoning",
|
||||
# Domain-specific (need boundary check)
|
||||
"ai ", " ai", "ai-", "-ai", # "ai" as word, not substring
|
||||
"agent", "agents", "neural", "autonomous",
|
||||
"compute", "training run", "computer use", "coding agent",
|
||||
# Community terms
|
||||
"local-llm", "local llama", "llama ",
|
||||
]
|
||||
|
||||
def __init__(self, user_agent=None):
|
||||
self.user_agent = user_agent or "python:athena:v0.1 (by tony_tech)"
|
||||
# Curation now centralized (issue #7): load from config/queries.json
|
||||
cfg = source_config("hackernews")
|
||||
self.ai_keywords = cfg.get("keywords") or []
|
||||
|
||||
def name(self) -> str:
|
||||
return "hackernews"
|
||||
@@ -76,38 +60,14 @@ class HackerNewsAdapter(SourceAdapter):
|
||||
def _is_ai_relevant(self, title: str) -> bool:
|
||||
"""Check if a story title is AI/ML relevant.
|
||||
|
||||
Uses multi-pass matching: first check unambiguous multi-word/phrases,
|
||||
then check word-boundary matches for shorter keywords that could
|
||||
false-positive (e.g. 'ai' matching 'Britain').
|
||||
Keywords are loaded from config/queries.json (issue #7) into
|
||||
self.ai_keywords — single source of truth, no inline duplicate.
|
||||
Substring match; callers pass lowercased titles for boundary terms.
|
||||
"""
|
||||
title_lower = title.lower()
|
||||
|
||||
# Pass 1: unambiguous keywords (multi-word, compound, specific names)
|
||||
unambiguous = [
|
||||
"language model", "deep learning", "foundation model", "retrieval augmented",
|
||||
"code generation", "context length", "context window", "attention mechanism",
|
||||
"llm", "gpt-", "gpt ", "rag ", "rag.", "vlm", "vla",
|
||||
"openai", "anthropic", "deepseek", "meta ai", "xai", "ponytail",
|
||||
"inference", "transformer", "diffusion", "alignment", "fine-tun",
|
||||
"embedd", "pretrain", "post-train", "multimodal", "reasoning",
|
||||
"agent", "agents", "neural", "autonomous",
|
||||
"training run", "computer use", "coding agent",
|
||||
"local-llm", "local llama", "llama ",
|
||||
]
|
||||
for kw in unambiguous:
|
||||
for kw in self.ai_keywords:
|
||||
if kw in title_lower:
|
||||
return True
|
||||
|
||||
# Pass 2: word-boundary check for "ai" and "compute" (avoid 'Britain', 'Guinea', etc.)
|
||||
import re
|
||||
if re.search(r'\bai\b', title_lower):
|
||||
return True
|
||||
if re.search(r'\bcompute\b', title_lower):
|
||||
# Only if combined with other tech context
|
||||
tech_words = ["gpu", "tpu", "cluster", "datacenter", "data center", "server"]
|
||||
if any(w in title_lower for w in tech_words):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _score(self, item: dict) -> float:
|
||||
|
||||
+4
-3
@@ -24,13 +24,13 @@ import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
from html import unescape
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters import SourceAdapter, source_config
|
||||
|
||||
|
||||
class RedditAdapter(SourceAdapter):
|
||||
"""Reddit RSS + JSON adapter."""
|
||||
|
||||
# Default subreddits for AI content
|
||||
# Default subreddits for AI content (fallback if config missing)
|
||||
DEFAULT_SUBREDDITS = [
|
||||
"MachineLearning", "artificial", "LocalLLaMA", "Startups",
|
||||
]
|
||||
@@ -57,7 +57,8 @@ class RedditAdapter(SourceAdapter):
|
||||
rate_limit: Seconds between subreddit requests.
|
||||
user_agent: Custom User-Agent header.
|
||||
"""
|
||||
self.subreddits = subreddits or self.DEFAULT_SUBREDDITS
|
||||
cfg = source_config("reddit")
|
||||
self.subreddits = subreddits or cfg.get("subreddits") or self.DEFAULT_SUBREDDITS
|
||||
self.rate_limit = rate_limit
|
||||
self.user_agent = user_agent or "python:ai-oracle:v0.1 (by tony_tech)"
|
||||
|
||||
|
||||
+15
-3
@@ -24,7 +24,7 @@ import feedparser
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from email.utils import parsedate_to_datetime
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters import SourceAdapter, source_config
|
||||
|
||||
|
||||
# Curated feed list — AI-focused, reliable, diverse publishers.
|
||||
@@ -83,6 +83,18 @@ AI_KEYWORDS = [
|
||||
class RSSFeedsAdapter(SourceAdapter):
|
||||
"""RSS feed aggregator for commercial AI news."""
|
||||
|
||||
# Module-level fallbacks (used only if config/queries.json is missing)
|
||||
FEEDS = [
|
||||
("rss:techcrunch", "TechCrunch AI", "https://techcrunch.com/category/artificial-intelligence/feed/"),
|
||||
]
|
||||
AI_KEYWORDS = [r"\bai\b"]
|
||||
|
||||
def __init__(self):
|
||||
# Curation centralized (issue #7): config wins, fallbacks otherwise
|
||||
cfg = source_config("rss")
|
||||
self.feeds = cfg.get("feeds") or list(self.FEEDS)
|
||||
self.ai_keywords = cfg.get("keywords") or list(self.AI_KEYWORDS)
|
||||
|
||||
def name(self) -> str:
|
||||
return "rss"
|
||||
|
||||
@@ -92,7 +104,7 @@ class RSSFeedsAdapter(SourceAdapter):
|
||||
tag_text = " ".join(tags).lower()
|
||||
combined = text + " " + tag_text
|
||||
|
||||
for pattern in AI_KEYWORDS:
|
||||
for pattern in self.ai_keywords:
|
||||
if re.search(pattern, combined):
|
||||
return True
|
||||
return False
|
||||
@@ -147,7 +159,7 @@ class RSSFeedsAdapter(SourceAdapter):
|
||||
all_entries = []
|
||||
feed_failures = []
|
||||
|
||||
for source_key, label, url in FEEDS:
|
||||
for source_key, label, url in self.feeds:
|
||||
try:
|
||||
d = feedparser.parse(url)
|
||||
if d.status not in (200, 301, 302, 307, 308) or not d.entries:
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"sources": {
|
||||
"hackernews": {
|
||||
"keywords": [
|
||||
"language model", "deep learning", "foundation model", "retrieval augmented",
|
||||
"code generation", "context length", "context window", "attention mechanism",
|
||||
"llm", "gpt-", "gpt ", "rag ", "rag.", "vlm", "vla",
|
||||
"openai", "anthropic", "deepseek", "meta ai", "xai", "ponytail",
|
||||
"inference", "transformer", "diffusion", "alignment", "fine-tun",
|
||||
"embed", "pretrain", "post-train", "multimodal", "reasoning",
|
||||
"ai ", " ai", "ai-", "-ai",
|
||||
"agent", "agents", "neural", "autonomous",
|
||||
"compute", "training run", "computer use", "coding agent",
|
||||
"local-llm", "local llama", "llama "
|
||||
]
|
||||
},
|
||||
"arxiv": {
|
||||
"categories": ["cs.AI", "cs.LG", "cs.CL"]
|
||||
},
|
||||
"reddit": {
|
||||
"subreddits": ["MachineLearning", "artificial", "LocalLLaMA", "Startups"]
|
||||
},
|
||||
"rss": {
|
||||
"feeds": [
|
||||
["rss:techcrunch", "TechCrunch AI", "https://techcrunch.com/category/artificial-intelligence/feed/"],
|
||||
["rss:venturebeat", "VentureBeat AI", "https://venturebeat.com/category/ai/feed/"],
|
||||
["rss:theverge", "The Verge AI", "https://www.theverge.com/rss/ai-artificial-intelligence/index.xml"],
|
||||
["rss:ainews", "AI News", "https://www.artificialintelligence-news.com/feed/"],
|
||||
["rss:decoder", "The Decoder", "https://www.the-decoder.com/feed/"],
|
||||
["rss:mittr", "MIT Tech Review AI", "https://www.technologyreview.com/topic/artificial-intelligence/feed/"],
|
||||
["rss:openai", "OpenAI Blog", "https://openai.com/blog/rss.xml"],
|
||||
["rss:anthropic", "Anthropic News", "https://www.anthropic.com/rss/news.xml"],
|
||||
["rss:googleai", "Google AI Blog", "https://blog.google/technology/rss.xml"],
|
||||
["rss:metaai", "Meta AI Blog", "https://ai.meta.com/blog/rss.xml"]
|
||||
],
|
||||
"keywords": [
|
||||
"\\bai\\b", "\\bmachine learning\\b", "\\bdeep learning\\b", "\\bneural\\b",
|
||||
"\\bgenerative ai\\b", "\\bgenerative\\b", "\\bllm\\b", "\\blarge language\\b",
|
||||
"\\bfoundation model\\b", "\\btransformer\\b", "\\baugmented\\b",
|
||||
"\\bagent\\b", "\\bautonomous\\b", "\\bmcp\\b", "\\bfunction call\\b",
|
||||
"\\btool use\\b", "\\brai\\b", "\\bretrieval\\b",
|
||||
"\\binference\\b", "\\bmodel\\b", "\\bembedding\\b", "\\btoken\\b"
|
||||
]
|
||||
},
|
||||
"github": {
|
||||
"search_terms": ["machine-learning", "deep-learning", "llm", "ai-agent", "transformer"]
|
||||
}
|
||||
}
|
||||
}
|
||||
-318
@@ -1,318 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Reddit Idea Generator — Proof of Concept v5
|
||||
Uses Reddit RSS feeds (Atom XML). No browser needed.
|
||||
Trafilatura for clean text extraction. SQLite for storage.
|
||||
|
||||
Usage: python3 reddit_proof.py [count]
|
||||
Example: python3 reddit_proof.py 20
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
import sqlite3
|
||||
import os
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime, timezone
|
||||
from html import unescape
|
||||
|
||||
import trafilatura
|
||||
|
||||
DB_PATH = os.path.join(os.path.dirname(__file__), "oracle.db")
|
||||
SCHEMA_PATH = os.path.join(os.path.dirname(__file__), "schema.sql")
|
||||
|
||||
SUBREDDITS = [
|
||||
"MachineLearning", "artificial", "LocalLLaMA", "Startups",
|
||||
]
|
||||
|
||||
|
||||
def init_db():
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
with open(SCHEMA_PATH) as f:
|
||||
conn.executescript(f.read())
|
||||
conn.commit()
|
||||
return conn
|
||||
|
||||
|
||||
def fetch_rss(subreddit, sort="hot"):
|
||||
"""Fetch RSS feed for a subreddit. Returns parsed entries."""
|
||||
url = f"https://www.reddit.com/r/{subreddit}/{sort}/.rss?limit=50"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "oracle-reddit-proof/1.0"})
|
||||
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
xml_data = resp.read().decode("utf-8")
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 429:
|
||||
wait = 5 * (attempt + 1)
|
||||
print(f" 429 on r/{subreddit}, retry in {wait}s")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
print(f" RSS error r/{subreddit}: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
print(f" RSS error r/{subreddit}: {e}")
|
||||
return []
|
||||
else:
|
||||
print(f" r/{subreddit}: still rate limited, skip")
|
||||
return []
|
||||
|
||||
# Parse Atom XML — find all <entry> elements
|
||||
root = ET.fromstring(xml_data)
|
||||
entries = []
|
||||
|
||||
# Handle namespace: Atom uses http://www.w3.org/2005/Atom
|
||||
# But ET.findall with ns prefix requires registering the namespace
|
||||
# Simpler approach: strip namespace from tags and search directly
|
||||
for entry in root.iter():
|
||||
# Get local name (strip namespace)
|
||||
tag = entry.tag.split("}")[-1] if "}" in entry.tag else entry.tag
|
||||
|
||||
if tag == "entry":
|
||||
title = None
|
||||
link = None
|
||||
author = ""
|
||||
content = ""
|
||||
pub = ""
|
||||
eid = ""
|
||||
|
||||
for child in entry:
|
||||
ctag = child.tag.split("}")[-1]
|
||||
if ctag == "title":
|
||||
title = child.text
|
||||
elif ctag == "link":
|
||||
link = child.get("href", "")
|
||||
elif ctag == "author":
|
||||
name_el = child[0] if child else None
|
||||
if name_el:
|
||||
name_tag = name_el.tag.split("}")[-1]
|
||||
if name_tag == "name":
|
||||
author = name_el.text or ""
|
||||
elif ctag == "content":
|
||||
content = child.text or ""
|
||||
elif ctag == "published":
|
||||
pub = child.text or ""
|
||||
elif ctag == "id":
|
||||
eid = child.text or ""
|
||||
|
||||
if title and link:
|
||||
entries.append({
|
||||
"title": unescape(title.strip()),
|
||||
"url": link,
|
||||
"author": unescape(author.strip()),
|
||||
"content": content,
|
||||
"published": pub,
|
||||
"id": eid,
|
||||
"subreddit": subreddit,
|
||||
})
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
def clean_html_content(html):
|
||||
"""Extract readable text from Reddit's HTML content."""
|
||||
if not html:
|
||||
return ""
|
||||
text = re.sub(r"<!--.*?-->", "", html, flags=re.DOTALL)
|
||||
text = re.sub(r"<div[^>]*>", "\n", text)
|
||||
text = re.sub(r"</div>", "\n", text)
|
||||
text = re.sub(r"<br\s*/?>", "\n", text, flags=re.I)
|
||||
text = re.sub(r"<[^>]+>", "", text)
|
||||
text = unescape(text)
|
||||
text = re.sub(r"\n\s*\n+", "\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) > 1:
|
||||
count = int(sys.argv[1])
|
||||
else:
|
||||
count = 20
|
||||
|
||||
print(f"=== Reddit Idea Generator — Proof of Concept v5 ===")
|
||||
print(f" count: {count}")
|
||||
print()
|
||||
|
||||
conn = init_db()
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Step 1: Fetch RSS
|
||||
print(f"[1/3] Fetching RSS feeds...")
|
||||
all_entries = []
|
||||
seen_ids = set()
|
||||
|
||||
for i, sub in enumerate(SUBREDDITS):
|
||||
entries = fetch_rss(sub)
|
||||
new = [e for e in entries if e["id"] not in seen_ids]
|
||||
seen_ids.update(e["id"] for e in new)
|
||||
all_entries.extend(new)
|
||||
if new:
|
||||
print(f" r/{sub}: {len(new)} entries")
|
||||
# Rate limit between subreddits
|
||||
if i < len(SUBREDDITS) - 1:
|
||||
time.sleep(3)
|
||||
|
||||
print(f" Total: {len(all_entries)} entries")
|
||||
|
||||
if not all_entries:
|
||||
print("\n No entries fetched. Reddit may be rate-limiting this IP.")
|
||||
print(" Try again later or use fewer subreddits.")
|
||||
sys.exit(1)
|
||||
|
||||
# Limit to count
|
||||
entries_to_store = all_entries[:count]
|
||||
print(f" Storing {len(entries_to_store)} entries")
|
||||
|
||||
# Step 2: Store
|
||||
stored = 0
|
||||
for entry in entries_to_store:
|
||||
post_id = entry["id"].replace("t3_", "")
|
||||
content_text = clean_html_content(entry["content"])
|
||||
|
||||
# Signal score — RSS hot feed already sorted by relevance
|
||||
# Use position-based scoring (higher rank = higher score)
|
||||
idx = entries_to_store.index(entry)
|
||||
score = max(10.0 - idx * 0.5, 1.0)
|
||||
|
||||
# Category tags
|
||||
category_tags = ["reddit"]
|
||||
sub = entry.get("subreddit", "").lower()
|
||||
if "machinelearning" in sub:
|
||||
category_tags.append("machine-learning")
|
||||
elif "artificial" in sub:
|
||||
category_tags.append("ai-general")
|
||||
elif "localllama" in sub:
|
||||
category_tags.append("local-llm")
|
||||
elif "startups" in sub:
|
||||
category_tags.append("startups")
|
||||
|
||||
# Post type from title markers
|
||||
title = entry.get("title", "")
|
||||
if " [P]" in title or " [p]" in title:
|
||||
category_tags.append("project")
|
||||
elif " [R]" in title or " [r]" in title:
|
||||
category_tags.append("research")
|
||||
elif " [D]" in title or " [d]" in title:
|
||||
category_tags.append("discussion")
|
||||
elif " [N]" in title or " [n]" in title:
|
||||
category_tags.append("news")
|
||||
else:
|
||||
category_tags.append("general")
|
||||
|
||||
# Clean title (remove [X] markers)
|
||||
clean_title = re.sub(r"\s*\[[A-Z]\]\s*$", "", title)
|
||||
|
||||
raw_meta = {
|
||||
"subreddit": entry["subreddit"],
|
||||
"author": entry["author"],
|
||||
"published": entry["published"],
|
||||
"text_length": len(content_text),
|
||||
}
|
||||
|
||||
source_id = post_id or entry["url"].split("/")[-1] or f"rss_{stored}"
|
||||
now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
try:
|
||||
cursor.execute("""
|
||||
INSERT OR REPLACE INTO entries
|
||||
(source, source_id, url, title, extracted_text, summary,
|
||||
category_tags, signal_score, raw_metadata, first_seen, last_updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
"reddit", source_id, entry["url"], clean_title,
|
||||
content_text,
|
||||
None, # summary — LLM later
|
||||
json.dumps(category_tags),
|
||||
score,
|
||||
json.dumps(raw_meta),
|
||||
now, now,
|
||||
))
|
||||
stored += 1
|
||||
except Exception as e:
|
||||
print(f" DB ERROR: {e}")
|
||||
|
||||
conn.commit()
|
||||
print(f" Stored {stored} entries")
|
||||
|
||||
# Step 3: Summary
|
||||
print(f"\n[3/3] Summary")
|
||||
cursor.execute("SELECT COUNT(*) FROM entries")
|
||||
total = cursor.fetchone()[0]
|
||||
print(f" Total entries in DB: {total}")
|
||||
cursor.execute("SELECT COUNT(*) FROM entries WHERE source='reddit'")
|
||||
reddit_count = cursor.fetchone()[0]
|
||||
print(f" Reddit entries: {reddit_count}")
|
||||
cursor.execute("SELECT AVG(signal_score) FROM entries WHERE source='reddit'")
|
||||
avg_score = cursor.fetchone()[0] or 0
|
||||
print(f" Avg signal score: {avg_score:.2f}")
|
||||
|
||||
# Subreddit distribution
|
||||
cursor.execute("""
|
||||
SELECT raw_metadata, COUNT(*) FROM entries
|
||||
WHERE source='reddit'
|
||||
GROUP BY raw_metadata
|
||||
ORDER BY COUNT(*) DESC
|
||||
""")
|
||||
print(f"\n Subreddit distribution:")
|
||||
for meta, cnt in cursor.fetchall():
|
||||
d = json.loads(meta)
|
||||
print(f" r/{d.get('subreddit', '?')}: {cnt}")
|
||||
|
||||
# Top 5
|
||||
print(f"\n Top 5 by signal score:")
|
||||
cursor.execute("""
|
||||
SELECT id, title, signal_score, raw_metadata, category_tags,
|
||||
LENGTH(extracted_text) as text_len
|
||||
FROM entries WHERE source='reddit'
|
||||
ORDER BY signal_score DESC
|
||||
LIMIT 5
|
||||
""")
|
||||
for row in cursor.fetchall():
|
||||
eid, title, score, meta, tags, txt_len = row
|
||||
meta_dict = json.loads(meta) if meta else {}
|
||||
print(f" [{eid}] score={score:.1f} text={txt_len}ch")
|
||||
print(f" {title[:90]}")
|
||||
print(f" r/{meta_dict.get('subreddit', '?')} "
|
||||
f"by {meta_dict.get('author', '?')}")
|
||||
|
||||
# Extraction quality
|
||||
print(f"\n Extraction quality (top entry):")
|
||||
cursor.execute("""
|
||||
SELECT title, extracted_text
|
||||
FROM entries WHERE source='reddit'
|
||||
ORDER BY signal_score DESC
|
||||
LIMIT 1
|
||||
""")
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
title, excerpt = row
|
||||
print(f" Title: {title[:80]}")
|
||||
print(f" Length: {len(excerpt) if excerpt else 0} chars")
|
||||
if excerpt:
|
||||
print(f" Preview:\n {excerpt[:400]}...")
|
||||
else:
|
||||
print(" (empty)")
|
||||
|
||||
# Check for garbled extractions
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) FROM entries
|
||||
WHERE source='reddit' AND LENGTH(extracted_text) < 100
|
||||
""")
|
||||
short_count = cursor.fetchone()[0]
|
||||
if short_count > 0:
|
||||
print(f"\n ⚠ {short_count}/{stored} entries have very short extractions (<100 chars)")
|
||||
print(" These are likely link-only posts or external links")
|
||||
|
||||
conn.close()
|
||||
print(f"\n Database: {DB_PATH}")
|
||||
print(" Done.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user