Epictetus
a9fbe27178
- adapters/__init__.py: add load_queries() + source_config() (stdlib json,
safe fallback to {} on missing/corrupt config so pipeline never crashes).
- config/queries.json: per-adapter blocks (hackernews.keywords, arxiv.categories,
reddit.subreddits, rss.feeds+keywords, github.search_terms). JSON (not
yaml) to honor Athena's dependency-free runtime; PyYAML avoided.
- hackernews: drop class AI_KEYWORDS + the DUPLICATE inline list inside
_is_ai_relevant() (the internal drift Ty flagged). Now loads self.ai_keywords
from config. Simplified matching to single substring pass (boundary variants
'ai ',' ai','ai-','-ai' approximate word-boundary; dropped the niche
'compute+tech-context' guard as not worth centralizing).
- reddit: DEFAULT_SUBREDDITS kept as fallback; __init__ prefers config.
- arxiv: DEFAULT_CATEGORIES kept as fallback; __init__ prefers config.
- rss: FEEDS + AI_KEYWORDS kept as module fallbacks; __init__ prefers config.
Keywords stay regex form (re.search) as in original.
- github: trending queries moved to config search_terms; fallback retained.
- DELETE reddit_proof.py: standalone PoC v5 at repo root, own main()+init_db()
+ direct INSERT OR REPLACE, NOT in cron, NOT imported anywhere -> dead
code. Also removes its byte-duplicate SUBREDDITS.
NOTE: fallback class constants remain intentionally (issue #7 cut #5: safe
rollout). Curation VALUES now live in one file; the constants are inert
unless config/queries.json is missing.
Verified: all adapters compile; config loads (HN 46 kw, RSS 10 feeds);
full dry-run fetches all 6 sources; grep confirms HN internal dup list gone.
46 lines
1.4 KiB
Python
46 lines
1.4 KiB
Python
"""Source adapters for AI Research Oracle."""
|
|
|
|
import json
|
|
import os
|
|
from abc import ABC, abstractmethod
|
|
|
|
# Centralized curation config (issue #7). One file, per-adapter blocks.
|
|
# Stdlib-only (JSON, not YAML) to honor Athena's dependency-free runtime.
|
|
_QUERIES_PATH = os.path.join(
|
|
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
|
|
"config", "queries.json")
|
|
|
|
|
|
def load_queries():
|
|
"""Load config/queries.json. Returns {'sources': {...}}.
|
|
|
|
Safe fallback: if the file is missing/corrupt, returns an empty
|
|
{'sources': {}} so adapters fall back to their class defaults
|
|
(constructor None-override) instead of crashing the pipeline.
|
|
"""
|
|
try:
|
|
with open(_QUERIES_PATH) as f:
|
|
data = json.load(f)
|
|
return data if isinstance(data, dict) else {"sources": {}}
|
|
except Exception:
|
|
return {"sources": {}}
|
|
|
|
|
|
def source_config(name: str) -> dict:
|
|
"""Return the per-adapter block for `name`, or {} if absent."""
|
|
return load_queries().get("sources", {}).get(name, {}) or {}
|
|
|
|
|
|
class SourceAdapter(ABC):
|
|
"""Base class for all ingestion adapters."""
|
|
|
|
@abstractmethod
|
|
def name(self) -> str:
|
|
"""Source name: 'github', 'arxiv', 'reddit'."""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def fetch(self, query: str = "", limit: int = 20) -> list[dict]:
|
|
"""Return entries matching DB schema fields."""
|
|
pass
|