refactor(adapters): centralize curation in config/queries.json (issue #7)
- adapters/__init__.py: add load_queries() + source_config() (stdlib json,
safe fallback to {} on missing/corrupt config so pipeline never crashes).
- config/queries.json: per-adapter blocks (hackernews.keywords, arxiv.categories,
reddit.subreddits, rss.feeds+keywords, github.search_terms). JSON (not
yaml) to honor Athena's dependency-free runtime; PyYAML avoided.
- hackernews: drop class AI_KEYWORDS + the DUPLICATE inline list inside
_is_ai_relevant() (the internal drift Ty flagged). Now loads self.ai_keywords
from config. Simplified matching to single substring pass (boundary variants
'ai ',' ai','ai-','-ai' approximate word-boundary; dropped the niche
'compute+tech-context' guard as not worth centralizing).
- reddit: DEFAULT_SUBREDDITS kept as fallback; __init__ prefers config.
- arxiv: DEFAULT_CATEGORIES kept as fallback; __init__ prefers config.
- rss: FEEDS + AI_KEYWORDS kept as module fallbacks; __init__ prefers config.
Keywords stay regex form (re.search) as in original.
- github: trending queries moved to config search_terms; fallback retained.
- DELETE reddit_proof.py: standalone PoC v5 at repo root, own main()+init_db()
+ direct INSERT OR REPLACE, NOT in cron, NOT imported anywhere -> dead
code. Also removes its byte-duplicate SUBREDDITS.
NOTE: fallback class constants remain intentionally (issue #7 cut #5: safe
rollout). Curation VALUES now live in one file; the constants are inert
unless config/queries.json is missing.
Verified: all adapters compile; config loads (HN 46 kw, RSS 10 feeds);
full dry-run fetches all 6 sources; grep confirms HN internal dup list gone.
This commit is contained in:
+8
-48
@@ -20,7 +20,7 @@ import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters import SourceAdapter, source_config
|
||||
|
||||
|
||||
class HackerNewsAdapter(SourceAdapter):
|
||||
@@ -28,27 +28,11 @@ class HackerNewsAdapter(SourceAdapter):
|
||||
|
||||
BASE = "https://hacker-news.firebaseio.com/v0"
|
||||
|
||||
AI_KEYWORDS = [
|
||||
# Multi-word phrases (unambiguous)
|
||||
"language model", "deep learning", "foundation model", "retrieval augmented",
|
||||
"code generation", "context length", "context window", "attention mechanism",
|
||||
# Compound/abbreviations (unambiguous)
|
||||
"llm", "gpt-", "gpt ", "rag ", "rag.", "vlm", "vla",
|
||||
# Specific company/product names
|
||||
"openai", "anthropic", "deepseek", "meta ai", "xai", "ponytail",
|
||||
# Topic-specific (with word boundary awareness in _is_ai_relevant)
|
||||
"inference", "transformer", "diffusion", "alignment", "fine-tun",
|
||||
"embedd", "pretrain", "post-train", "multimodal", "reasoning",
|
||||
# Domain-specific (need boundary check)
|
||||
"ai ", " ai", "ai-", "-ai", # "ai" as word, not substring
|
||||
"agent", "agents", "neural", "autonomous",
|
||||
"compute", "training run", "computer use", "coding agent",
|
||||
# Community terms
|
||||
"local-llm", "local llama", "llama ",
|
||||
]
|
||||
|
||||
def __init__(self, user_agent=None):
|
||||
self.user_agent = user_agent or "python:athena:v0.1 (by tony_tech)"
|
||||
# Curation now centralized (issue #7): load from config/queries.json
|
||||
cfg = source_config("hackernews")
|
||||
self.ai_keywords = cfg.get("keywords") or []
|
||||
|
||||
def name(self) -> str:
|
||||
return "hackernews"
|
||||
@@ -76,38 +60,14 @@ class HackerNewsAdapter(SourceAdapter):
|
||||
def _is_ai_relevant(self, title: str) -> bool:
|
||||
"""Check if a story title is AI/ML relevant.
|
||||
|
||||
Uses multi-pass matching: first check unambiguous multi-word/phrases,
|
||||
then check word-boundary matches for shorter keywords that could
|
||||
false-positive (e.g. 'ai' matching 'Britain').
|
||||
Keywords are loaded from config/queries.json (issue #7) into
|
||||
self.ai_keywords — single source of truth, no inline duplicate.
|
||||
Substring match; callers pass lowercased titles for boundary terms.
|
||||
"""
|
||||
title_lower = title.lower()
|
||||
|
||||
# Pass 1: unambiguous keywords (multi-word, compound, specific names)
|
||||
unambiguous = [
|
||||
"language model", "deep learning", "foundation model", "retrieval augmented",
|
||||
"code generation", "context length", "context window", "attention mechanism",
|
||||
"llm", "gpt-", "gpt ", "rag ", "rag.", "vlm", "vla",
|
||||
"openai", "anthropic", "deepseek", "meta ai", "xai", "ponytail",
|
||||
"inference", "transformer", "diffusion", "alignment", "fine-tun",
|
||||
"embedd", "pretrain", "post-train", "multimodal", "reasoning",
|
||||
"agent", "agents", "neural", "autonomous",
|
||||
"training run", "computer use", "coding agent",
|
||||
"local-llm", "local llama", "llama ",
|
||||
]
|
||||
for kw in unambiguous:
|
||||
for kw in self.ai_keywords:
|
||||
if kw in title_lower:
|
||||
return True
|
||||
|
||||
# Pass 2: word-boundary check for "ai" and "compute" (avoid 'Britain', 'Guinea', etc.)
|
||||
import re
|
||||
if re.search(r'\bai\b', title_lower):
|
||||
return True
|
||||
if re.search(r'\bcompute\b', title_lower):
|
||||
# Only if combined with other tech context
|
||||
tech_words = ["gpu", "tpu", "cluster", "datacenter", "data center", "server"]
|
||||
if any(w in title_lower for w in tech_words):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _score(self, item: dict) -> float:
|
||||
|
||||
Reference in New Issue
Block a user