Tony_tech/athena-oracle
3
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

refactor(adapters): centralize curation in config/queries.json (issue #7)

Browse Source 
- adapters/__init__.py: add load_queries() + source_config() (stdlib json,
  safe fallback to {} on missing/corrupt config so pipeline never crashes).
- config/queries.json: per-adapter blocks (hackernews.keywords, arxiv.categories,
  reddit.subreddits, rss.feeds+keywords, github.search_terms). JSON (not
  yaml) to honor Athena's dependency-free runtime; PyYAML avoided.
- hackernews: drop class AI_KEYWORDS + the DUPLICATE inline list inside
  _is_ai_relevant() (the internal drift Ty flagged). Now loads self.ai_keywords
  from config. Simplified matching to single substring pass (boundary variants
  'ai ',' ai','ai-','-ai' approximate word-boundary; dropped the niche
  'compute+tech-context' guard as not worth centralizing).
- reddit: DEFAULT_SUBREDDITS kept as fallback; __init__ prefers config.
- arxiv: DEFAULT_CATEGORIES kept as fallback; __init__ prefers config.
- rss: FEEDS + AI_KEYWORDS kept as module fallbacks; __init__ prefers config.
  Keywords stay regex form (re.search) as in original.
- github: trending queries moved to config search_terms; fallback retained.
- DELETE reddit_proof.py: standalone PoC v5 at repo root, own main()+init_db()
  + direct INSERT OR REPLACE, NOT in cron, NOT imported anywhere -> dead
  code. Also removes its byte-duplicate SUBREDDITS.

NOTE: fallback class constants remain intentionally (issue #7 cut #5: safe
rollout). Curation VALUES now live in one file; the constants are inert
unless config/queries.json is missing.

Verified: all adapters compile; config loads (HN 46 kw, RSS 10 feeds);
full dry-run fetches all 6 sources; grep confirms HN internal dup list gone.
This commit is contained in:
Epictetus
2026-07-10 17:42:06 +00:00
parent 7ee1af3d7b
commit a9fbe27178
8 changed files with 115 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files
adapters/__init__.py
+28
View File
@@ -1,7 +1,35 @@
"""Source adapters for AI Research Oracle."""
import json
import os
from abc import ABC, abstractmethod
# Centralized curation config (issue #7). One file, per-adapter blocks.
# Stdlib-only (JSON, not YAML) to honor Athena's dependency-free runtime.
_QUERIES_PATH = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"config", "queries.json")
def load_queries():
"""Load config/queries.json. Returns {'sources': {...}}.
Safe fallback: if the file is missing/corrupt, returns an empty
{'sources': {}} so adapters fall back to their class defaults
(constructor None-override) instead of crashing the pipeline.
"""
try:
with open(_QUERIES_PATH) as f:
data = json.load(f)
return data if isinstance(data, dict) else {"sources": {}}
except Exception:
return {"sources": {}}
def source_config(name: str) -> dict:
"""Return the per-adapter block for `name`, or {} if absent."""
return load_queries().get("sources", {}).get(name, {}) or {}
class SourceAdapter(ABC):
"""Base class for all ingestion adapters."""
adapters/arxiv.py
+3 -7
View File
@@ -32,7 +32,7 @@ import xml.etree.ElementTree as ET
from datetime import datetime, timedelta, timezone
from html import unescape
from adapters import SourceAdapter
from adapters import SourceAdapter, source_config
# arXiv API
ARXIV_API = "http://export.arxiv.org/api/query"
@@ -45,12 +45,8 @@ class ArxivAdapter(SourceAdapter):
DEFAULT_CATEGORIES = ["cs.AI", "cs.LG", "cs.CL"]
def __init__(self, categories=None, rate_limit=3):
"""
Args:
categories: List of arXiv categories. Default: cs.AI, cs.LG, cs.CL
rate_limit: Seconds between API calls (default 3).
"""
self.categories = categories or self.DEFAULT_CATEGORIES
cfg = source_config("arxiv")
self.categories = categories or cfg.get("categories") or self.DEFAULT_CATEGORIES
self.rate_limit = rate_limit
def name(self) -> str:
adapters/github.py
+8 -7
View File
@@ -17,7 +17,7 @@ import urllib.error
import urllib.parse
from datetime import datetime, timedelta, timezone
from adapters import SourceAdapter
from adapters import SourceAdapter, source_config
class GitHubAdapter(SourceAdapter):
@@ -29,6 +29,11 @@ class GitHubAdapter(SourceAdapter):
"""Initialize with optional read-only token (5000 req/hr vs 60)."""
self.token = token or os.environ.get("GITHUB_TOKEN", "")
self.cache = {}
# Curation centralized (issue #7): trending queries from config
cfg = source_config("github")
self.search_terms = cfg.get("search_terms") or [
"ai agent", "llm OR inference OR rag", "autonomous agent OR AI tool",
]
def name(self) -> str:
return "github"
@@ -158,12 +163,8 @@ class GitHubAdapter(SourceAdapter):
cutoff = (now - timedelta(days=30)).strftime("%Y-%m-%d")
# Three queries for breadth: agents, LLM/infra, and security/tools
repos = []
for q in [
f"ai agent created:>{cutoff}",
f"llm OR inference OR rag created:>{cutoff}",
f"autonomous agent OR AI tool created:>{cutoff}",
]:
batch = self._search_repos(q, sort="stars", per_page=30)
for q in self.search_terms:
batch = self._search_repos(f"{q} created:>{cutoff}", sort="stars", per_page=30)
repos.extend(batch)
time.sleep(1) # polite spacing
adapters/hackernews.py
+8 -48
View File
@@ -20,7 +20,7 @@ import urllib.request
import urllib.error
from datetime import datetime, timezone
from adapters import SourceAdapter
from adapters import SourceAdapter, source_config
class HackerNewsAdapter(SourceAdapter):
@@ -28,27 +28,11 @@ class HackerNewsAdapter(SourceAdapter):
BASE = "https://hacker-news.firebaseio.com/v0"
AI_KEYWORDS = [
# Multi-word phrases (unambiguous)
"language model", "deep learning", "foundation model", "retrieval augmented",
"code generation", "context length", "context window", "attention mechanism",
# Compound/abbreviations (unambiguous)
"llm", "gpt-", "gpt ", "rag ", "rag.", "vlm", "vla",
# Specific company/product names
"openai", "anthropic", "deepseek", "meta ai", "xai", "ponytail",
# Topic-specific (with word boundary awareness in _is_ai_relevant)
"inference", "transformer", "diffusion", "alignment", "fine-tun",
"embedd", "pretrain", "post-train", "multimodal", "reasoning",
# Domain-specific (need boundary check)
"ai ", " ai", "ai-", "-ai", # "ai" as word, not substring
"agent", "agents", "neural", "autonomous",
"compute", "training run", "computer use", "coding agent",
# Community terms
"local-llm", "local llama", "llama ",
]
def __init__(self, user_agent=None):
self.user_agent = user_agent or "python:athena:v0.1 (by tony_tech)"
# Curation now centralized (issue #7): load from config/queries.json
cfg = source_config("hackernews")
self.ai_keywords = cfg.get("keywords") or []
def name(self) -> str:
return "hackernews"
@@ -76,38 +60,14 @@ class HackerNewsAdapter(SourceAdapter):
def _is_ai_relevant(self, title: str) -> bool:
"""Check if a story title is AI/ML relevant.
Uses multi-pass matching: first check unambiguous multi-word/phrases,
then check word-boundary matches for shorter keywords that could
false-positive (e.g. 'ai' matching 'Britain').
Keywords are loaded from config/queries.json (issue #7) into
self.ai_keywords — single source of truth, no inline duplicate.
Substring match; callers pass lowercased titles for boundary terms.
"""
title_lower = title.lower()
# Pass 1: unambiguous keywords (multi-word, compound, specific names)
unambiguous = [
"language model", "deep learning", "foundation model", "retrieval augmented",
"code generation", "context length", "context window", "attention mechanism",
"llm", "gpt-", "gpt ", "rag ", "rag.", "vlm", "vla",
"openai", "anthropic", "deepseek", "meta ai", "xai", "ponytail",
"inference", "transformer", "diffusion", "alignment", "fine-tun",
"embedd", "pretrain", "post-train", "multimodal", "reasoning",
"agent", "agents", "neural", "autonomous",
"training run", "computer use", "coding agent",
"local-llm", "local llama", "llama ",
]
for kw in unambiguous:
for kw in self.ai_keywords:
if kw in title_lower:
return True
# Pass 2: word-boundary check for "ai" and "compute" (avoid 'Britain', 'Guinea', etc.)
import re
if re.search(r'\bai\b', title_lower):
return True
if re.search(r'\bcompute\b', title_lower):
# Only if combined with other tech context
tech_words = ["gpu", "tpu", "cluster", "datacenter", "data center", "server"]
if any(w in title_lower for w in tech_words):
return True
return False
def _score(self, item: dict) -> float:
adapters/reddit.py
+4 -3
View File
@@ -24,13 +24,13 @@ import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from html import unescape
from adapters import SourceAdapter
from adapters import SourceAdapter, source_config
class RedditAdapter(SourceAdapter):
"""Reddit RSS + JSON adapter."""
# Default subreddits for AI content
# Default subreddits for AI content (fallback if config missing)
DEFAULT_SUBREDDITS = [
"MachineLearning", "artificial", "LocalLLaMA", "Startups",
]
@@ -57,7 +57,8 @@ class RedditAdapter(SourceAdapter):
rate_limit: Seconds between subreddit requests.
user_agent: Custom User-Agent header.
"""
self.subreddits = subreddits or self.DEFAULT_SUBREDDITS
cfg = source_config("reddit")
self.subreddits = subreddits or cfg.get("subreddits") or self.DEFAULT_SUBREDDITS
self.rate_limit = rate_limit
self.user_agent = user_agent or "python:ai-oracle:v0.1 (by tony_tech)"
adapters/rss_feeds.py
+15 -3
View File
@@ -24,7 +24,7 @@ import feedparser
from datetime import datetime, timedelta, timezone
from email.utils import parsedate_to_datetime
from adapters import SourceAdapter
from adapters import SourceAdapter, source_config
# Curated feed list — AI-focused, reliable, diverse publishers.
@@ -83,6 +83,18 @@ AI_KEYWORDS = [
class RSSFeedsAdapter(SourceAdapter):
"""RSS feed aggregator for commercial AI news."""
# Module-level fallbacks (used only if config/queries.json is missing)
FEEDS = [
("rss:techcrunch", "TechCrunch AI", "https://techcrunch.com/category/artificial-intelligence/feed/"),
]
AI_KEYWORDS = [r"\bai\b"]
def __init__(self):
# Curation centralized (issue #7): config wins, fallbacks otherwise
cfg = source_config("rss")
self.feeds = cfg.get("feeds") or list(self.FEEDS)
self.ai_keywords = cfg.get("keywords") or list(self.AI_KEYWORDS)
def name(self) -> str:
return "rss"
@@ -92,7 +104,7 @@ class RSSFeedsAdapter(SourceAdapter):
tag_text = " ".join(tags).lower()
combined = text + " " + tag_text
for pattern in AI_KEYWORDS:
for pattern in self.ai_keywords:
if re.search(pattern, combined):
return True
return False
@@ -147,7 +159,7 @@ class RSSFeedsAdapter(SourceAdapter):
all_entries = []
feed_failures = []
for source_key, label, url in FEEDS:
for source_key, label, url in self.feeds:
try:
d = feedparser.parse(url)
if d.status not in (200, 301, 302, 307, 308) or not d.entries: