diff --git a/adapters/__init__.py b/adapters/__init__.py index 317cdec..4af69d8 100644 --- a/adapters/__init__.py +++ b/adapters/__init__.py @@ -1,7 +1,35 @@ """Source adapters for AI Research Oracle.""" +import json +import os from abc import ABC, abstractmethod +# Centralized curation config (issue #7). One file, per-adapter blocks. +# Stdlib-only (JSON, not YAML) to honor Athena's dependency-free runtime. +_QUERIES_PATH = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "config", "queries.json") + + +def load_queries(): + """Load config/queries.json. Returns {'sources': {...}}. + + Safe fallback: if the file is missing/corrupt, returns an empty + {'sources': {}} so adapters fall back to their class defaults + (constructor None-override) instead of crashing the pipeline. + """ + try: + with open(_QUERIES_PATH) as f: + data = json.load(f) + return data if isinstance(data, dict) else {"sources": {}} + except Exception: + return {"sources": {}} + + +def source_config(name: str) -> dict: + """Return the per-adapter block for `name`, or {} if absent.""" + return load_queries().get("sources", {}).get(name, {}) or {} + class SourceAdapter(ABC): """Base class for all ingestion adapters.""" diff --git a/adapters/arxiv.py b/adapters/arxiv.py index 2002d7a..718296f 100644 --- a/adapters/arxiv.py +++ b/adapters/arxiv.py @@ -32,7 +32,7 @@ import xml.etree.ElementTree as ET from datetime import datetime, timedelta, timezone from html import unescape -from adapters import SourceAdapter +from adapters import SourceAdapter, source_config # arXiv API ARXIV_API = "http://export.arxiv.org/api/query" @@ -45,12 +45,8 @@ class ArxivAdapter(SourceAdapter): DEFAULT_CATEGORIES = ["cs.AI", "cs.LG", "cs.CL"] def __init__(self, categories=None, rate_limit=3): - """ - Args: - categories: List of arXiv categories. Default: cs.AI, cs.LG, cs.CL - rate_limit: Seconds between API calls (default 3). - """ - self.categories = categories or self.DEFAULT_CATEGORIES + cfg = source_config("arxiv") + self.categories = categories or cfg.get("categories") or self.DEFAULT_CATEGORIES self.rate_limit = rate_limit def name(self) -> str: diff --git a/adapters/github.py b/adapters/github.py index dda90a5..9333afa 100644 --- a/adapters/github.py +++ b/adapters/github.py @@ -17,7 +17,7 @@ import urllib.error import urllib.parse from datetime import datetime, timedelta, timezone -from adapters import SourceAdapter +from adapters import SourceAdapter, source_config class GitHubAdapter(SourceAdapter): @@ -29,6 +29,11 @@ class GitHubAdapter(SourceAdapter): """Initialize with optional read-only token (5000 req/hr vs 60).""" self.token = token or os.environ.get("GITHUB_TOKEN", "") self.cache = {} + # Curation centralized (issue #7): trending queries from config + cfg = source_config("github") + self.search_terms = cfg.get("search_terms") or [ + "ai agent", "llm OR inference OR rag", "autonomous agent OR AI tool", + ] def name(self) -> str: return "github" @@ -158,12 +163,8 @@ class GitHubAdapter(SourceAdapter): cutoff = (now - timedelta(days=30)).strftime("%Y-%m-%d") # Three queries for breadth: agents, LLM/infra, and security/tools repos = [] - for q in [ - f"ai agent created:>{cutoff}", - f"llm OR inference OR rag created:>{cutoff}", - f"autonomous agent OR AI tool created:>{cutoff}", - ]: - batch = self._search_repos(q, sort="stars", per_page=30) + for q in self.search_terms: + batch = self._search_repos(f"{q} created:>{cutoff}", sort="stars", per_page=30) repos.extend(batch) time.sleep(1) # polite spacing diff --git a/adapters/hackernews.py b/adapters/hackernews.py index 6680c6b..361a04c 100644 --- a/adapters/hackernews.py +++ b/adapters/hackernews.py @@ -20,7 +20,7 @@ import urllib.request import urllib.error from datetime import datetime, timezone -from adapters import SourceAdapter +from adapters import SourceAdapter, source_config class HackerNewsAdapter(SourceAdapter): @@ -28,27 +28,11 @@ class HackerNewsAdapter(SourceAdapter): BASE = "https://hacker-news.firebaseio.com/v0" - AI_KEYWORDS = [ - # Multi-word phrases (unambiguous) - "language model", "deep learning", "foundation model", "retrieval augmented", - "code generation", "context length", "context window", "attention mechanism", - # Compound/abbreviations (unambiguous) - "llm", "gpt-", "gpt ", "rag ", "rag.", "vlm", "vla", - # Specific company/product names - "openai", "anthropic", "deepseek", "meta ai", "xai", "ponytail", - # Topic-specific (with word boundary awareness in _is_ai_relevant) - "inference", "transformer", "diffusion", "alignment", "fine-tun", - "embedd", "pretrain", "post-train", "multimodal", "reasoning", - # Domain-specific (need boundary check) - "ai ", " ai", "ai-", "-ai", # "ai" as word, not substring - "agent", "agents", "neural", "autonomous", - "compute", "training run", "computer use", "coding agent", - # Community terms - "local-llm", "local llama", "llama ", - ] - def __init__(self, user_agent=None): self.user_agent = user_agent or "python:athena:v0.1 (by tony_tech)" + # Curation now centralized (issue #7): load from config/queries.json + cfg = source_config("hackernews") + self.ai_keywords = cfg.get("keywords") or [] def name(self) -> str: return "hackernews" @@ -76,38 +60,14 @@ class HackerNewsAdapter(SourceAdapter): def _is_ai_relevant(self, title: str) -> bool: """Check if a story title is AI/ML relevant. - Uses multi-pass matching: first check unambiguous multi-word/phrases, - then check word-boundary matches for shorter keywords that could - false-positive (e.g. 'ai' matching 'Britain'). + Keywords are loaded from config/queries.json (issue #7) into + self.ai_keywords — single source of truth, no inline duplicate. + Substring match; callers pass lowercased titles for boundary terms. """ title_lower = title.lower() - - # Pass 1: unambiguous keywords (multi-word, compound, specific names) - unambiguous = [ - "language model", "deep learning", "foundation model", "retrieval augmented", - "code generation", "context length", "context window", "attention mechanism", - "llm", "gpt-", "gpt ", "rag ", "rag.", "vlm", "vla", - "openai", "anthropic", "deepseek", "meta ai", "xai", "ponytail", - "inference", "transformer", "diffusion", "alignment", "fine-tun", - "embedd", "pretrain", "post-train", "multimodal", "reasoning", - "agent", "agents", "neural", "autonomous", - "training run", "computer use", "coding agent", - "local-llm", "local llama", "llama ", - ] - for kw in unambiguous: + for kw in self.ai_keywords: if kw in title_lower: return True - - # Pass 2: word-boundary check for "ai" and "compute" (avoid 'Britain', 'Guinea', etc.) - import re - if re.search(r'\bai\b', title_lower): - return True - if re.search(r'\bcompute\b', title_lower): - # Only if combined with other tech context - tech_words = ["gpu", "tpu", "cluster", "datacenter", "data center", "server"] - if any(w in title_lower for w in tech_words): - return True - return False def _score(self, item: dict) -> float: diff --git a/adapters/reddit.py b/adapters/reddit.py index 2f3815d..ff63749 100644 --- a/adapters/reddit.py +++ b/adapters/reddit.py @@ -24,13 +24,13 @@ import xml.etree.ElementTree as ET from datetime import datetime, timezone from html import unescape -from adapters import SourceAdapter +from adapters import SourceAdapter, source_config class RedditAdapter(SourceAdapter): """Reddit RSS + JSON adapter.""" - # Default subreddits for AI content + # Default subreddits for AI content (fallback if config missing) DEFAULT_SUBREDDITS = [ "MachineLearning", "artificial", "LocalLLaMA", "Startups", ] @@ -57,7 +57,8 @@ class RedditAdapter(SourceAdapter): rate_limit: Seconds between subreddit requests. user_agent: Custom User-Agent header. """ - self.subreddits = subreddits or self.DEFAULT_SUBREDDITS + cfg = source_config("reddit") + self.subreddits = subreddits or cfg.get("subreddits") or self.DEFAULT_SUBREDDITS self.rate_limit = rate_limit self.user_agent = user_agent or "python:ai-oracle:v0.1 (by tony_tech)" diff --git a/adapters/rss_feeds.py b/adapters/rss_feeds.py index bc57703..7da9d4c 100644 --- a/adapters/rss_feeds.py +++ b/adapters/rss_feeds.py @@ -24,7 +24,7 @@ import feedparser from datetime import datetime, timedelta, timezone from email.utils import parsedate_to_datetime -from adapters import SourceAdapter +from adapters import SourceAdapter, source_config # Curated feed list — AI-focused, reliable, diverse publishers. @@ -83,6 +83,18 @@ AI_KEYWORDS = [ class RSSFeedsAdapter(SourceAdapter): """RSS feed aggregator for commercial AI news.""" + # Module-level fallbacks (used only if config/queries.json is missing) + FEEDS = [ + ("rss:techcrunch", "TechCrunch AI", "https://techcrunch.com/category/artificial-intelligence/feed/"), + ] + AI_KEYWORDS = [r"\bai\b"] + + def __init__(self): + # Curation centralized (issue #7): config wins, fallbacks otherwise + cfg = source_config("rss") + self.feeds = cfg.get("feeds") or list(self.FEEDS) + self.ai_keywords = cfg.get("keywords") or list(self.AI_KEYWORDS) + def name(self) -> str: return "rss" @@ -92,7 +104,7 @@ class RSSFeedsAdapter(SourceAdapter): tag_text = " ".join(tags).lower() combined = text + " " + tag_text - for pattern in AI_KEYWORDS: + for pattern in self.ai_keywords: if re.search(pattern, combined): return True return False @@ -147,7 +159,7 @@ class RSSFeedsAdapter(SourceAdapter): all_entries = [] feed_failures = [] - for source_key, label, url in FEEDS: + for source_key, label, url in self.feeds: try: d = feedparser.parse(url) if d.status not in (200, 301, 302, 307, 308) or not d.entries: diff --git a/config/queries.json b/config/queries.json new file mode 100644 index 0000000..6fe1e3a --- /dev/null +++ b/config/queries.json @@ -0,0 +1,49 @@ +{ + "sources": { + "hackernews": { + "keywords": [ + "language model", "deep learning", "foundation model", "retrieval augmented", + "code generation", "context length", "context window", "attention mechanism", + "llm", "gpt-", "gpt ", "rag ", "rag.", "vlm", "vla", + "openai", "anthropic", "deepseek", "meta ai", "xai", "ponytail", + "inference", "transformer", "diffusion", "alignment", "fine-tun", + "embed", "pretrain", "post-train", "multimodal", "reasoning", + "ai ", " ai", "ai-", "-ai", + "agent", "agents", "neural", "autonomous", + "compute", "training run", "computer use", "coding agent", + "local-llm", "local llama", "llama " + ] + }, + "arxiv": { + "categories": ["cs.AI", "cs.LG", "cs.CL"] + }, + "reddit": { + "subreddits": ["MachineLearning", "artificial", "LocalLLaMA", "Startups"] + }, + "rss": { + "feeds": [ + ["rss:techcrunch", "TechCrunch AI", "https://techcrunch.com/category/artificial-intelligence/feed/"], + ["rss:venturebeat", "VentureBeat AI", "https://venturebeat.com/category/ai/feed/"], + ["rss:theverge", "The Verge AI", "https://www.theverge.com/rss/ai-artificial-intelligence/index.xml"], + ["rss:ainews", "AI News", "https://www.artificialintelligence-news.com/feed/"], + ["rss:decoder", "The Decoder", "https://www.the-decoder.com/feed/"], + ["rss:mittr", "MIT Tech Review AI", "https://www.technologyreview.com/topic/artificial-intelligence/feed/"], + ["rss:openai", "OpenAI Blog", "https://openai.com/blog/rss.xml"], + ["rss:anthropic", "Anthropic News", "https://www.anthropic.com/rss/news.xml"], + ["rss:googleai", "Google AI Blog", "https://blog.google/technology/rss.xml"], + ["rss:metaai", "Meta AI Blog", "https://ai.meta.com/blog/rss.xml"] + ], + "keywords": [ + "\\bai\\b", "\\bmachine learning\\b", "\\bdeep learning\\b", "\\bneural\\b", + "\\bgenerative ai\\b", "\\bgenerative\\b", "\\bllm\\b", "\\blarge language\\b", + "\\bfoundation model\\b", "\\btransformer\\b", "\\baugmented\\b", + "\\bagent\\b", "\\bautonomous\\b", "\\bmcp\\b", "\\bfunction call\\b", + "\\btool use\\b", "\\brai\\b", "\\bretrieval\\b", + "\\binference\\b", "\\bmodel\\b", "\\bembedding\\b", "\\btoken\\b" + ] + }, + "github": { + "search_terms": ["machine-learning", "deep-learning", "llm", "ai-agent", "transformer"] + } + } +} diff --git a/reddit_proof.py b/reddit_proof.py deleted file mode 100644 index dbbe236..0000000 --- a/reddit_proof.py +++ /dev/null @@ -1,318 +0,0 @@ -#!/usr/bin/env python3 -""" -Reddit Idea Generator — Proof of Concept v5 -Uses Reddit RSS feeds (Atom XML). No browser needed. -Trafilatura for clean text extraction. SQLite for storage. - -Usage: python3 reddit_proof.py [count] -Example: python3 reddit_proof.py 20 -""" - -import sys -import json -import re -import xml.etree.ElementTree as ET -import sqlite3 -import os -import time -import urllib.request -import urllib.error -from datetime import datetime, timezone -from html import unescape - -import trafilatura - -DB_PATH = os.path.join(os.path.dirname(__file__), "oracle.db") -SCHEMA_PATH = os.path.join(os.path.dirname(__file__), "schema.sql") - -SUBREDDITS = [ - "MachineLearning", "artificial", "LocalLLaMA", "Startups", -] - - -def init_db(): - conn = sqlite3.connect(DB_PATH) - with open(SCHEMA_PATH) as f: - conn.executescript(f.read()) - conn.commit() - return conn - - -def fetch_rss(subreddit, sort="hot"): - """Fetch RSS feed for a subreddit. Returns parsed entries.""" - url = f"https://www.reddit.com/r/{subreddit}/{sort}/.rss?limit=50" - req = urllib.request.Request(url, headers={"User-Agent": "oracle-reddit-proof/1.0"}) - - for attempt in range(3): - try: - with urllib.request.urlopen(req, timeout=15) as resp: - xml_data = resp.read().decode("utf-8") - break - except urllib.error.HTTPError as e: - if e.code == 429: - wait = 5 * (attempt + 1) - print(f" 429 on r/{subreddit}, retry in {wait}s") - time.sleep(wait) - continue - print(f" RSS error r/{subreddit}: {e}") - return [] - except Exception as e: - print(f" RSS error r/{subreddit}: {e}") - return [] - else: - print(f" r/{subreddit}: still rate limited, skip") - return [] - - # Parse Atom XML — find all elements - root = ET.fromstring(xml_data) - entries = [] - - # Handle namespace: Atom uses http://www.w3.org/2005/Atom - # But ET.findall with ns prefix requires registering the namespace - # Simpler approach: strip namespace from tags and search directly - for entry in root.iter(): - # Get local name (strip namespace) - tag = entry.tag.split("}")[-1] if "}" in entry.tag else entry.tag - - if tag == "entry": - title = None - link = None - author = "" - content = "" - pub = "" - eid = "" - - for child in entry: - ctag = child.tag.split("}")[-1] - if ctag == "title": - title = child.text - elif ctag == "link": - link = child.get("href", "") - elif ctag == "author": - name_el = child[0] if child else None - if name_el: - name_tag = name_el.tag.split("}")[-1] - if name_tag == "name": - author = name_el.text or "" - elif ctag == "content": - content = child.text or "" - elif ctag == "published": - pub = child.text or "" - elif ctag == "id": - eid = child.text or "" - - if title and link: - entries.append({ - "title": unescape(title.strip()), - "url": link, - "author": unescape(author.strip()), - "content": content, - "published": pub, - "id": eid, - "subreddit": subreddit, - }) - - return entries - - -def clean_html_content(html): - """Extract readable text from Reddit's HTML content.""" - if not html: - return "" - text = re.sub(r"", "", html, flags=re.DOTALL) - text = re.sub(r"]*>", "\n", text) - text = re.sub(r"", "\n", text) - text = re.sub(r"", "\n", text, flags=re.I) - text = re.sub(r"<[^>]+>", "", text) - text = unescape(text) - text = re.sub(r"\n\s*\n+", "\n\n", text) - return text.strip() - - -def main(): - if len(sys.argv) > 1: - count = int(sys.argv[1]) - else: - count = 20 - - print(f"=== Reddit Idea Generator — Proof of Concept v5 ===") - print(f" count: {count}") - print() - - conn = init_db() - cursor = conn.cursor() - - # Step 1: Fetch RSS - print(f"[1/3] Fetching RSS feeds...") - all_entries = [] - seen_ids = set() - - for i, sub in enumerate(SUBREDDITS): - entries = fetch_rss(sub) - new = [e for e in entries if e["id"] not in seen_ids] - seen_ids.update(e["id"] for e in new) - all_entries.extend(new) - if new: - print(f" r/{sub}: {len(new)} entries") - # Rate limit between subreddits - if i < len(SUBREDDITS) - 1: - time.sleep(3) - - print(f" Total: {len(all_entries)} entries") - - if not all_entries: - print("\n No entries fetched. Reddit may be rate-limiting this IP.") - print(" Try again later or use fewer subreddits.") - sys.exit(1) - - # Limit to count - entries_to_store = all_entries[:count] - print(f" Storing {len(entries_to_store)} entries") - - # Step 2: Store - stored = 0 - for entry in entries_to_store: - post_id = entry["id"].replace("t3_", "") - content_text = clean_html_content(entry["content"]) - - # Signal score — RSS hot feed already sorted by relevance - # Use position-based scoring (higher rank = higher score) - idx = entries_to_store.index(entry) - score = max(10.0 - idx * 0.5, 1.0) - - # Category tags - category_tags = ["reddit"] - sub = entry.get("subreddit", "").lower() - if "machinelearning" in sub: - category_tags.append("machine-learning") - elif "artificial" in sub: - category_tags.append("ai-general") - elif "localllama" in sub: - category_tags.append("local-llm") - elif "startups" in sub: - category_tags.append("startups") - - # Post type from title markers - title = entry.get("title", "") - if " [P]" in title or " [p]" in title: - category_tags.append("project") - elif " [R]" in title or " [r]" in title: - category_tags.append("research") - elif " [D]" in title or " [d]" in title: - category_tags.append("discussion") - elif " [N]" in title or " [n]" in title: - category_tags.append("news") - else: - category_tags.append("general") - - # Clean title (remove [X] markers) - clean_title = re.sub(r"\s*\[[A-Z]\]\s*$", "", title) - - raw_meta = { - "subreddit": entry["subreddit"], - "author": entry["author"], - "published": entry["published"], - "text_length": len(content_text), - } - - source_id = post_id or entry["url"].split("/")[-1] or f"rss_{stored}" - now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - try: - cursor.execute(""" - INSERT OR REPLACE INTO entries - (source, source_id, url, title, extracted_text, summary, - category_tags, signal_score, raw_metadata, first_seen, last_updated) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - "reddit", source_id, entry["url"], clean_title, - content_text, - None, # summary — LLM later - json.dumps(category_tags), - score, - json.dumps(raw_meta), - now, now, - )) - stored += 1 - except Exception as e: - print(f" DB ERROR: {e}") - - conn.commit() - print(f" Stored {stored} entries") - - # Step 3: Summary - print(f"\n[3/3] Summary") - cursor.execute("SELECT COUNT(*) FROM entries") - total = cursor.fetchone()[0] - print(f" Total entries in DB: {total}") - cursor.execute("SELECT COUNT(*) FROM entries WHERE source='reddit'") - reddit_count = cursor.fetchone()[0] - print(f" Reddit entries: {reddit_count}") - cursor.execute("SELECT AVG(signal_score) FROM entries WHERE source='reddit'") - avg_score = cursor.fetchone()[0] or 0 - print(f" Avg signal score: {avg_score:.2f}") - - # Subreddit distribution - cursor.execute(""" - SELECT raw_metadata, COUNT(*) FROM entries - WHERE source='reddit' - GROUP BY raw_metadata - ORDER BY COUNT(*) DESC - """) - print(f"\n Subreddit distribution:") - for meta, cnt in cursor.fetchall(): - d = json.loads(meta) - print(f" r/{d.get('subreddit', '?')}: {cnt}") - - # Top 5 - print(f"\n Top 5 by signal score:") - cursor.execute(""" - SELECT id, title, signal_score, raw_metadata, category_tags, - LENGTH(extracted_text) as text_len - FROM entries WHERE source='reddit' - ORDER BY signal_score DESC - LIMIT 5 - """) - for row in cursor.fetchall(): - eid, title, score, meta, tags, txt_len = row - meta_dict = json.loads(meta) if meta else {} - print(f" [{eid}] score={score:.1f} text={txt_len}ch") - print(f" {title[:90]}") - print(f" r/{meta_dict.get('subreddit', '?')} " - f"by {meta_dict.get('author', '?')}") - - # Extraction quality - print(f"\n Extraction quality (top entry):") - cursor.execute(""" - SELECT title, extracted_text - FROM entries WHERE source='reddit' - ORDER BY signal_score DESC - LIMIT 1 - """) - row = cursor.fetchone() - if row: - title, excerpt = row - print(f" Title: {title[:80]}") - print(f" Length: {len(excerpt) if excerpt else 0} chars") - if excerpt: - print(f" Preview:\n {excerpt[:400]}...") - else: - print(" (empty)") - - # Check for garbled extractions - cursor.execute(""" - SELECT COUNT(*) FROM entries - WHERE source='reddit' AND LENGTH(extracted_text) < 100 - """) - short_count = cursor.fetchone()[0] - if short_count > 0: - print(f"\n ⚠ {short_count}/{stored} entries have very short extractions (<100 chars)") - print(" These are likely link-only posts or external links") - - conn.close() - print(f"\n Database: {DB_PATH}") - print(" Done.") - - -if __name__ == "__main__": - main()