Add Hacker News adapter + wire into pipeline

- adapters/hackernews.py: HN Firebase API adapter with AI keyword filtering
- Word-boundary matching to avoid substring traps (Britain/Guinea)
- Score: log(points) + log(comments), actual HN scores
- Wired into ENABLED_SOURCES + verification in pipeline.py
- Live test: 10 AI/ML stories fetched, all clean
This commit is contained in:
Epictetus
2026-07-08 05:32:28 +00:00
parent 729760fb27
commit 6396b66b45
2 changed files with 344 additions and 1 deletions
+23 -1
View File
@@ -31,10 +31,11 @@ ADAPTERS = {
"github": lambda: __import__("adapters.github", fromlist=["GitHubAdapter"]).GitHubAdapter(),
"arxiv": lambda: __import__("adapters.arxiv", fromlist=["ArxivAdapter"]).ArxivAdapter(),
"reddit": lambda: __import__("adapters.reddit", fromlist=["RedditAdapter"]).RedditAdapter(),
"hackernews": lambda: __import__("adapters.hackernews", fromlist=["HackerNewsAdapter"]).HackerNewsAdapter(),
}
# Default enabled sources
ENABLED_SOURCES = ["github", "arxiv", "reddit"]
ENABLED_SOURCES = ["github", "arxiv", "reddit", "hackernews"]
def init_db(db_path: str, schema_path: str) -> sqlite3.Connection:
@@ -144,6 +145,27 @@ def verify_entries(conn: sqlite3.Connection, source: str, sample_size: int = 3):
except Exception as e:
print(f" ? [{eid}] {title[:60]}... verify failed: {e}")
elif src == "hackernews":
hn_id = meta.get("id", "")
if hn_id:
try:
req = urllib.request.Request(
f"https://hacker-news.firebaseio.com/v0/item/{hn_id}.json",
headers={"User-Agent": "ai-oracle/0.1"},
)
with urllib.request.urlopen(req, timeout=10) as resp:
live = json.loads(resp.read().decode("utf-8"))
live_score = live.get("score", 0)
db_score = meta.get("score", 0)
drift = abs(live_score - db_score)
if drift <= 20:
print(f" ✓ [{eid}] {title[:60]}... hn_score={db_score} drift={drift}")
passed += 1
else:
print(f" ? [{eid}] {title[:60]}... hn_score={db_score} vs live={live_score} DRIFT={drift}")
except Exception as e:
print(f" ? [{eid}] {title[:60]}... verify failed: {e}")
elif src == "reddit":
# For Reddit, just check the URL is reachable (no easy verification of scores via RSS)
if url: