adapters/__init__.py

@@ -1,7 +1,35 @@
 """Source adapters for AI Research Oracle."""
 
+import json
+import os
 from abc import ABC, abstractmethod
 
+# Centralized curation config (issue #7). One file, per-adapter blocks.
+# Stdlib-only (JSON, not YAML) to honor Athena's dependency-free runtime.
+_QUERIES_PATH = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+    "config", "queries.json")
+
+
+def load_queries():
+    """Load config/queries.json. Returns {'sources': {...}}.
+
+    Safe fallback: if the file is missing/corrupt, returns an empty
+    {'sources': {}} so adapters fall back to their class defaults
+    (constructor None-override) instead of crashing the pipeline.
+    """
+    try:
+        with open(_QUERIES_PATH) as f:
+            data = json.load(f)
+        return data if isinstance(data, dict) else {"sources": {}}
+    except Exception:
+        return {"sources": {}}
+
+
+def source_config(name: str) -> dict:
+    """Return the per-adapter block for `name`, or {} if absent."""
+    return load_queries().get("sources", {}).get(name, {}) or {}
+
 
 class SourceAdapter(ABC):
     """Base class for all ingestion adapters."""

adapters/arxiv.py

@@ -32,7 +32,7 @@ import xml.etree.ElementTree as ET
 from datetime import datetime, timedelta, timezone
 from html import unescape
 
-from adapters import SourceAdapter
+from adapters import SourceAdapter, source_config
 
 # arXiv API
 ARXIV_API = "http://export.arxiv.org/api/query"
@@ -45,12 +45,8 @@ class ArxivAdapter(SourceAdapter):
     DEFAULT_CATEGORIES = ["cs.AI", "cs.LG", "cs.CL"]
 
     def __init__(self, categories=None, rate_limit=3):
-        """
+        cfg = source_config("arxiv")
-        Args:
+        self.categories = categories or cfg.get("categories") or self.DEFAULT_CATEGORIES
-            categories: List of arXiv categories. Default: cs.AI, cs.LG, cs.CL
-            rate_limit: Seconds between API calls (default 3).
-        """
-        self.categories = categories or self.DEFAULT_CATEGORIES
         self.rate_limit = rate_limit
 
     def name(self) -> str:

adapters/github.py

@@ -17,7 +17,7 @@ import urllib.error
 import urllib.parse
 from datetime import datetime, timedelta, timezone
 
-from adapters import SourceAdapter
+from adapters import SourceAdapter, source_config
 
 
 class GitHubAdapter(SourceAdapter):
@@ -29,6 +29,11 @@ class GitHubAdapter(SourceAdapter):
         """Initialize with optional read-only token (5000 req/hr vs 60)."""
         self.token = token or os.environ.get("GITHUB_TOKEN", "")
         self.cache = {}
+        # Curation centralized (issue #7): trending queries from config
+        cfg = source_config("github")
+        self.search_terms = cfg.get("search_terms") or [
+            "ai agent", "llm OR inference OR rag", "autonomous agent OR AI tool",
+        ]
 
     def name(self) -> str:
         return "github"
@@ -158,12 +163,8 @@ class GitHubAdapter(SourceAdapter):
             cutoff = (now - timedelta(days=30)).strftime("%Y-%m-%d")
             # Three queries for breadth: agents, LLM/infra, and security/tools
             repos = []
-            for q in [
+            for q in self.search_terms:
-                f"ai agent created:>{cutoff}",
+                batch = self._search_repos(f"{q} created:>{cutoff}", sort="stars", per_page=30)
-                f"llm OR inference OR rag created:>{cutoff}",
-                f"autonomous agent OR AI tool created:>{cutoff}",
-            ]:
-                batch = self._search_repos(q, sort="stars", per_page=30)
                 repos.extend(batch)
                 time.sleep(1)  # polite spacing
 

adapters/hackernews.py

@@ -20,7 +20,7 @@ import urllib.request
 import urllib.error
 from datetime import datetime, timezone
 
-from adapters import SourceAdapter
+from adapters import SourceAdapter, source_config
 
 
 class HackerNewsAdapter(SourceAdapter):
@@ -28,27 +28,11 @@ class HackerNewsAdapter(SourceAdapter):
 
     BASE = "https://hacker-news.firebaseio.com/v0"
 
-    AI_KEYWORDS = [
-        # Multi-word phrases (unambiguous)
-        "language model", "deep learning", "foundation model", "retrieval augmented",
-        "code generation", "context length", "context window", "attention mechanism",
-        # Compound/abbreviations (unambiguous)
-        "llm", "gpt-", "gpt ", "rag ", "rag.", "vlm", "vla",
-        # Specific company/product names
-        "openai", "anthropic", "deepseek", "meta ai", "xai", "ponytail",
-        # Topic-specific (with word boundary awareness in _is_ai_relevant)
-        "inference", "transformer", "diffusion", "alignment", "fine-tun",
-        "embedd", "pretrain", "post-train", "multimodal", "reasoning",
-        # Domain-specific (need boundary check)
-        "ai ", " ai", "ai-", "-ai",  # "ai" as word, not substring
-        "agent", "agents", "neural", "autonomous",
-        "compute", "training run", "computer use", "coding agent",
-        # Community terms
-        "local-llm", "local llama", "llama ",
-    ]
-
     def __init__(self, user_agent=None):
         self.user_agent = user_agent or "python:athena:v0.1 (by tony_tech)"
+        # Curation now centralized (issue #7): load from config/queries.json
+        cfg = source_config("hackernews")
+        self.ai_keywords = cfg.get("keywords") or []
 
     def name(self) -> str:
         return "hackernews"
@@ -76,38 +60,14 @@ class HackerNewsAdapter(SourceAdapter):
     def _is_ai_relevant(self, title: str) -> bool:
         """Check if a story title is AI/ML relevant.
 
-        Uses multi-pass matching: first check unambiguous multi-word/phrases,
+        Keywords are loaded from config/queries.json (issue #7) into
-        then check word-boundary matches for shorter keywords that could
+        self.ai_keywords — single source of truth, no inline duplicate.
-        false-positive (e.g. 'ai' matching 'Britain').
+        Substring match; callers pass lowercased titles for boundary terms.
         """
         title_lower = title.lower()
-
+        for kw in self.ai_keywords:
-        # Pass 1: unambiguous keywords (multi-word, compound, specific names)
-        unambiguous = [
-            "language model", "deep learning", "foundation model", "retrieval augmented",
-            "code generation", "context length", "context window", "attention mechanism",
-            "llm", "gpt-", "gpt ", "rag ", "rag.", "vlm", "vla",
-            "openai", "anthropic", "deepseek", "meta ai", "xai", "ponytail",
-            "inference", "transformer", "diffusion", "alignment", "fine-tun",
-            "embedd", "pretrain", "post-train", "multimodal", "reasoning",
-            "agent", "agents", "neural", "autonomous",
-            "training run", "computer use", "coding agent",
-            "local-llm", "local llama", "llama ",
-        ]
-        for kw in unambiguous:
             if kw in title_lower:
                 return True
-
-        # Pass 2: word-boundary check for "ai" and "compute" (avoid 'Britain', 'Guinea', etc.)
-        import re
-        if re.search(r'\bai\b', title_lower):
-            return True
-        if re.search(r'\bcompute\b', title_lower):
-            # Only if combined with other tech context
-            tech_words = ["gpu", "tpu", "cluster", "datacenter", "data center", "server"]
-            if any(w in title_lower for w in tech_words):
-                return True
-
         return False
 
     def _score(self, item: dict) -> float:

adapters/reddit.py

@@ -24,13 +24,13 @@ import xml.etree.ElementTree as ET
 from datetime import datetime, timezone
 from html import unescape
 
-from adapters import SourceAdapter
+from adapters import SourceAdapter, source_config
 
 
 class RedditAdapter(SourceAdapter):
     """Reddit RSS + JSON adapter."""
 
-    # Default subreddits for AI content
+    # Default subreddits for AI content (fallback if config missing)
     DEFAULT_SUBREDDITS = [
         "MachineLearning", "artificial", "LocalLLaMA", "Startups",
     ]
@@ -57,7 +57,8 @@ class RedditAdapter(SourceAdapter):
             rate_limit: Seconds between subreddit requests.
             user_agent: Custom User-Agent header.
         """
-        self.subreddits = subreddits or self.DEFAULT_SUBREDDITS
+        cfg = source_config("reddit")
+        self.subreddits = subreddits or cfg.get("subreddits") or self.DEFAULT_SUBREDDITS
         self.rate_limit = rate_limit
         self.user_agent = user_agent or "python:ai-oracle:v0.1 (by tony_tech)"
 

adapters/rss_feeds.py

@@ -24,7 +24,7 @@ import feedparser
 from datetime import datetime, timedelta, timezone
 from email.utils import parsedate_to_datetime
 
-from adapters import SourceAdapter
+from adapters import SourceAdapter, source_config
 
 
 # Curated feed list — AI-focused, reliable, diverse publishers.
@@ -83,6 +83,18 @@ AI_KEYWORDS = [
 class RSSFeedsAdapter(SourceAdapter):
     """RSS feed aggregator for commercial AI news."""
 
+    # Module-level fallbacks (used only if config/queries.json is missing)
+    FEEDS = [
+        ("rss:techcrunch", "TechCrunch AI", "https://techcrunch.com/category/artificial-intelligence/feed/"),
+    ]
+    AI_KEYWORDS = [r"\bai\b"]
+
+    def __init__(self):
+        # Curation centralized (issue #7): config wins, fallbacks otherwise
+        cfg = source_config("rss")
+        self.feeds = cfg.get("feeds") or list(self.FEEDS)
+        self.ai_keywords = cfg.get("keywords") or list(self.AI_KEYWORDS)
+
     def name(self) -> str:
         return "rss"
 
@@ -92,7 +104,7 @@ class RSSFeedsAdapter(SourceAdapter):
         tag_text = " ".join(tags).lower()
         combined = text + " " + tag_text
 
-        for pattern in AI_KEYWORDS:
+        for pattern in self.ai_keywords:
             if re.search(pattern, combined):
                 return True
         return False
@@ -147,7 +159,7 @@ class RSSFeedsAdapter(SourceAdapter):
         all_entries = []
         feed_failures = []
 
-        for source_key, label, url in FEEDS:
+        for source_key, label, url in self.feeds:
             try:
                 d = feedparser.parse(url)
                 if d.status not in (200, 301, 302, 307, 308) or not d.entries:

config/queries.json

@@ -0,0 +1,49 @@
+{
+  "sources": {
+    "hackernews": {
+      "keywords": [
+        "language model", "deep learning", "foundation model", "retrieval augmented",
+        "code generation", "context length", "context window", "attention mechanism",
+        "llm", "gpt-", "gpt ", "rag ", "rag.", "vlm", "vla",
+        "openai", "anthropic", "deepseek", "meta ai", "xai", "ponytail",
+        "inference", "transformer", "diffusion", "alignment", "fine-tun",
+        "embed", "pretrain", "post-train", "multimodal", "reasoning",
+        "ai ", " ai", "ai-", "-ai",
+        "agent", "agents", "neural", "autonomous",
+        "compute", "training run", "computer use", "coding agent",
+        "local-llm", "local llama", "llama "
+      ]
+    },
+    "arxiv": {
+      "categories": ["cs.AI", "cs.LG", "cs.CL"]
+    },
+    "reddit": {
+      "subreddits": ["MachineLearning", "artificial", "LocalLLaMA", "Startups"]
+    },
+    "rss": {
+      "feeds": [
+        ["rss:techcrunch", "TechCrunch AI", "https://techcrunch.com/category/artificial-intelligence/feed/"],
+        ["rss:venturebeat", "VentureBeat AI", "https://venturebeat.com/category/ai/feed/"],
+        ["rss:theverge", "The Verge AI", "https://www.theverge.com/rss/ai-artificial-intelligence/index.xml"],
+        ["rss:ainews", "AI News", "https://www.artificialintelligence-news.com/feed/"],
+        ["rss:decoder", "The Decoder", "https://www.the-decoder.com/feed/"],
+        ["rss:mittr", "MIT Tech Review AI", "https://www.technologyreview.com/topic/artificial-intelligence/feed/"],
+        ["rss:openai", "OpenAI Blog", "https://openai.com/blog/rss.xml"],
+        ["rss:anthropic", "Anthropic News", "https://www.anthropic.com/rss/news.xml"],
+        ["rss:googleai", "Google AI Blog", "https://blog.google/technology/rss.xml"],
+        ["rss:metaai", "Meta AI Blog", "https://ai.meta.com/blog/rss.xml"]
+      ],
+      "keywords": [
+        "\\bai\\b", "\\bmachine learning\\b", "\\bdeep learning\\b", "\\bneural\\b",
+        "\\bgenerative ai\\b", "\\bgenerative\\b", "\\bllm\\b", "\\blarge language\\b",
+        "\\bfoundation model\\b", "\\btransformer\\b", "\\baugmented\\b",
+        "\\bagent\\b", "\\bautonomous\\b", "\\bmcp\\b", "\\bfunction call\\b",
+        "\\btool use\\b", "\\brai\\b", "\\bretrieval\\b",
+        "\\binference\\b", "\\bmodel\\b", "\\bembedding\\b", "\\btoken\\b"
+      ]
+    },
+    "github": {
+      "search_terms": ["machine-learning", "deep-learning", "llm", "ai-agent", "transformer"]
+    }
+  }
+}

reddit_proof.py

@@ -1,318 +0,0 @@
-#!/usr/bin/env python3
-"""
-Reddit Idea Generator — Proof of Concept v5
-Uses Reddit RSS feeds (Atom XML). No browser needed.
-Trafilatura for clean text extraction. SQLite for storage.
-
-Usage: python3 reddit_proof.py [count]
-Example: python3 reddit_proof.py 20
-"""
-
-import sys
-import json
-import re
-import xml.etree.ElementTree as ET
-import sqlite3
-import os
-import time
-import urllib.request
-import urllib.error
-from datetime import datetime, timezone
-from html import unescape
-
-import trafilatura
-
-DB_PATH = os.path.join(os.path.dirname(__file__), "oracle.db")
-SCHEMA_PATH = os.path.join(os.path.dirname(__file__), "schema.sql")
-
-SUBREDDITS = [
-    "MachineLearning", "artificial", "LocalLLaMA", "Startups",
-]
-
-
-def init_db():
-    conn = sqlite3.connect(DB_PATH)
-    with open(SCHEMA_PATH) as f:
-        conn.executescript(f.read())
-    conn.commit()
-    return conn
-
-
-def fetch_rss(subreddit, sort="hot"):
-    """Fetch RSS feed for a subreddit. Returns parsed entries."""
-    url = f"https://www.reddit.com/r/{subreddit}/{sort}/.rss?limit=50"
-    req = urllib.request.Request(url, headers={"User-Agent": "oracle-reddit-proof/1.0"})
-
-    for attempt in range(3):
-        try:
-            with urllib.request.urlopen(req, timeout=15) as resp:
-                xml_data = resp.read().decode("utf-8")
-            break
-        except urllib.error.HTTPError as e:
-            if e.code == 429:
-                wait = 5 * (attempt + 1)
-                print(f"    429 on r/{subreddit}, retry in {wait}s")
-                time.sleep(wait)
-                continue
-            print(f"    RSS error r/{subreddit}: {e}")
-            return []
-        except Exception as e:
-            print(f"    RSS error r/{subreddit}: {e}")
-            return []
-    else:
-        print(f"    r/{subreddit}: still rate limited, skip")
-        return []
-
-    # Parse Atom XML — find all <entry> elements
-    root = ET.fromstring(xml_data)
-    entries = []
-
-    # Handle namespace: Atom uses http://www.w3.org/2005/Atom
-    # But ET.findall with ns prefix requires registering the namespace
-    # Simpler approach: strip namespace from tags and search directly
-    for entry in root.iter():
-        # Get local name (strip namespace)
-        tag = entry.tag.split("}")[-1] if "}" in entry.tag else entry.tag
-
-        if tag == "entry":
-            title = None
-            link = None
-            author = ""
-            content = ""
-            pub = ""
-            eid = ""
-
-            for child in entry:
-                ctag = child.tag.split("}")[-1]
-                if ctag == "title":
-                    title = child.text
-                elif ctag == "link":
-                    link = child.get("href", "")
-                elif ctag == "author":
-                    name_el = child[0] if child else None
-                    if name_el:
-                        name_tag = name_el.tag.split("}")[-1]
-                        if name_tag == "name":
-                            author = name_el.text or ""
-                elif ctag == "content":
-                    content = child.text or ""
-                elif ctag == "published":
-                    pub = child.text or ""
-                elif ctag == "id":
-                    eid = child.text or ""
-
-            if title and link:
-                entries.append({
-                    "title": unescape(title.strip()),
-                    "url": link,
-                    "author": unescape(author.strip()),
-                    "content": content,
-                    "published": pub,
-                    "id": eid,
-                    "subreddit": subreddit,
-                })
-
-    return entries
-
-
-def clean_html_content(html):
-    """Extract readable text from Reddit's HTML content."""
-    if not html:
-        return ""
-    text = re.sub(r"<!--.*?-->", "", html, flags=re.DOTALL)
-    text = re.sub(r"<div[^>]*>", "\n", text)
-    text = re.sub(r"</div>", "\n", text)
-    text = re.sub(r"<br\s*/?>", "\n", text, flags=re.I)
-    text = re.sub(r"<[^>]+>", "", text)
-    text = unescape(text)
-    text = re.sub(r"\n\s*\n+", "\n\n", text)
-    return text.strip()
-
-
-def main():
-    if len(sys.argv) > 1:
-        count = int(sys.argv[1])
-    else:
-        count = 20
-
-    print(f"=== Reddit Idea Generator — Proof of Concept v5 ===")
-    print(f" count: {count}")
-    print()
-
-    conn = init_db()
-    cursor = conn.cursor()
-
-    # Step 1: Fetch RSS
-    print(f"[1/3] Fetching RSS feeds...")
-    all_entries = []
-    seen_ids = set()
-
-    for i, sub in enumerate(SUBREDDITS):
-        entries = fetch_rss(sub)
-        new = [e for e in entries if e["id"] not in seen_ids]
-        seen_ids.update(e["id"] for e in new)
-        all_entries.extend(new)
-        if new:
-            print(f"  r/{sub}: {len(new)} entries")
-        # Rate limit between subreddits
-        if i < len(SUBREDDITS) - 1:
-            time.sleep(3)
-
-    print(f"  Total: {len(all_entries)} entries")
-
-    if not all_entries:
-        print("\n  No entries fetched. Reddit may be rate-limiting this IP.")
-        print("  Try again later or use fewer subreddits.")
-        sys.exit(1)
-
-    # Limit to count
-    entries_to_store = all_entries[:count]
-    print(f"  Storing {len(entries_to_store)} entries")
-
-    # Step 2: Store
-    stored = 0
-    for entry in entries_to_store:
-        post_id = entry["id"].replace("t3_", "")
-        content_text = clean_html_content(entry["content"])
-
-        # Signal score — RSS hot feed already sorted by relevance
-        # Use position-based scoring (higher rank = higher score)
-        idx = entries_to_store.index(entry)
-        score = max(10.0 - idx * 0.5, 1.0)
-
-        # Category tags
-        category_tags = ["reddit"]
-        sub = entry.get("subreddit", "").lower()
-        if "machinelearning" in sub:
-            category_tags.append("machine-learning")
-        elif "artificial" in sub:
-            category_tags.append("ai-general")
-        elif "localllama" in sub:
-            category_tags.append("local-llm")
-        elif "startups" in sub:
-            category_tags.append("startups")
-
-        # Post type from title markers
-        title = entry.get("title", "")
-        if " [P]" in title or " [p]" in title:
-            category_tags.append("project")
-        elif " [R]" in title or " [r]" in title:
-            category_tags.append("research")
-        elif " [D]" in title or " [d]" in title:
-            category_tags.append("discussion")
-        elif " [N]" in title or " [n]" in title:
-            category_tags.append("news")
-        else:
-            category_tags.append("general")
-
-        # Clean title (remove [X] markers)
-        clean_title = re.sub(r"\s*\[[A-Z]\]\s*$", "", title)
-
-        raw_meta = {
-            "subreddit": entry["subreddit"],
-            "author": entry["author"],
-            "published": entry["published"],
-            "text_length": len(content_text),
-        }
-
-        source_id = post_id or entry["url"].split("/")[-1] or f"rss_{stored}"
-        now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-        try:
-            cursor.execute("""
-                INSERT OR REPLACE INTO entries
-                (source, source_id, url, title, extracted_text, summary,
-                 category_tags, signal_score, raw_metadata, first_seen, last_updated)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-            """, (
-                "reddit", source_id, entry["url"], clean_title,
-                content_text,
-                None,  # summary — LLM later
-                json.dumps(category_tags),
-                score,
-                json.dumps(raw_meta),
-                now, now,
-            ))
-            stored += 1
-        except Exception as e:
-            print(f"  DB ERROR: {e}")
-
-    conn.commit()
-    print(f"  Stored {stored} entries")
-
-    # Step 3: Summary
-    print(f"\n[3/3] Summary")
-    cursor.execute("SELECT COUNT(*) FROM entries")
-    total = cursor.fetchone()[0]
-    print(f"  Total entries in DB: {total}")
-    cursor.execute("SELECT COUNT(*) FROM entries WHERE source='reddit'")
-    reddit_count = cursor.fetchone()[0]
-    print(f"  Reddit entries: {reddit_count}")
-    cursor.execute("SELECT AVG(signal_score) FROM entries WHERE source='reddit'")
-    avg_score = cursor.fetchone()[0] or 0
-    print(f"  Avg signal score: {avg_score:.2f}")
-
-    # Subreddit distribution
-    cursor.execute("""
-        SELECT raw_metadata, COUNT(*) FROM entries
-        WHERE source='reddit'
-        GROUP BY raw_metadata
-        ORDER BY COUNT(*) DESC
-    """)
-    print(f"\n  Subreddit distribution:")
-    for meta, cnt in cursor.fetchall():
-        d = json.loads(meta)
-        print(f"    r/{d.get('subreddit', '?')}: {cnt}")
-
-    # Top 5
-    print(f"\n  Top 5 by signal score:")
-    cursor.execute("""
-        SELECT id, title, signal_score, raw_metadata, category_tags,
-               LENGTH(extracted_text) as text_len
-        FROM entries WHERE source='reddit'
-        ORDER BY signal_score DESC
-        LIMIT 5
-    """)
-    for row in cursor.fetchall():
-        eid, title, score, meta, tags, txt_len = row
-        meta_dict = json.loads(meta) if meta else {}
-        print(f"    [{eid}] score={score:.1f} text={txt_len}ch")
-        print(f"        {title[:90]}")
-        print(f"        r/{meta_dict.get('subreddit', '?')} "
-              f"by {meta_dict.get('author', '?')}")
-
-    # Extraction quality
-    print(f"\n  Extraction quality (top entry):")
-    cursor.execute("""
-        SELECT title, extracted_text
-        FROM entries WHERE source='reddit'
-        ORDER BY signal_score DESC
-        LIMIT 1
-    """)
-    row = cursor.fetchone()
-    if row:
-        title, excerpt = row
-        print(f"    Title: {title[:80]}")
-        print(f"    Length: {len(excerpt) if excerpt else 0} chars")
-        if excerpt:
-            print(f"    Preview:\n    {excerpt[:400]}...")
-        else:
-            print("    (empty)")
-
-    # Check for garbled extractions
-    cursor.execute("""
-        SELECT COUNT(*) FROM entries
-        WHERE source='reddit' AND LENGTH(extracted_text) < 100
-    """)
-    short_count = cursor.fetchone()[0]
-    if short_count > 0:
-        print(f"\n  ⚠ {short_count}/{stored} entries have very short extractions (<100 chars)")
-        print("    These are likely link-only posts or external links")
-
-    conn.close()
-    print(f"\n  Database: {DB_PATH}")
-    print("  Done.")
-
-
-if __name__ == "__main__":
-    main()