Add Hacker News adapter + wire into pipeline
- adapters/hackernews.py: HN Firebase API adapter with AI keyword filtering - Word-boundary matching to avoid substring traps (Britain/Guinea) - Score: log(points) + log(comments), actual HN scores - Wired into ENABLED_SOURCES + verification in pipeline.py - Live test: 10 AI/ML stories fetched, all clean
This commit is contained in:
+23
-1
@@ -31,10 +31,11 @@ ADAPTERS = {
|
||||
"github": lambda: __import__("adapters.github", fromlist=["GitHubAdapter"]).GitHubAdapter(),
|
||||
"arxiv": lambda: __import__("adapters.arxiv", fromlist=["ArxivAdapter"]).ArxivAdapter(),
|
||||
"reddit": lambda: __import__("adapters.reddit", fromlist=["RedditAdapter"]).RedditAdapter(),
|
||||
"hackernews": lambda: __import__("adapters.hackernews", fromlist=["HackerNewsAdapter"]).HackerNewsAdapter(),
|
||||
}
|
||||
|
||||
# Default enabled sources
|
||||
ENABLED_SOURCES = ["github", "arxiv", "reddit"]
|
||||
ENABLED_SOURCES = ["github", "arxiv", "reddit", "hackernews"]
|
||||
|
||||
|
||||
def init_db(db_path: str, schema_path: str) -> sqlite3.Connection:
|
||||
@@ -144,6 +145,27 @@ def verify_entries(conn: sqlite3.Connection, source: str, sample_size: int = 3):
|
||||
except Exception as e:
|
||||
print(f" ? [{eid}] {title[:60]}... verify failed: {e}")
|
||||
|
||||
elif src == "hackernews":
|
||||
hn_id = meta.get("id", "")
|
||||
if hn_id:
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"https://hacker-news.firebaseio.com/v0/item/{hn_id}.json",
|
||||
headers={"User-Agent": "ai-oracle/0.1"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
live = json.loads(resp.read().decode("utf-8"))
|
||||
live_score = live.get("score", 0)
|
||||
db_score = meta.get("score", 0)
|
||||
drift = abs(live_score - db_score)
|
||||
if drift <= 20:
|
||||
print(f" ✓ [{eid}] {title[:60]}... hn_score={db_score} drift={drift}")
|
||||
passed += 1
|
||||
else:
|
||||
print(f" ? [{eid}] {title[:60]}... hn_score={db_score} vs live={live_score} DRIFT={drift}")
|
||||
except Exception as e:
|
||||
print(f" ? [{eid}] {title[:60]}... verify failed: {e}")
|
||||
|
||||
elif src == "reddit":
|
||||
# For Reddit, just check the URL is reachable (no easy verification of scores via RSS)
|
||||
if url:
|
||||
|
||||
Reference in New Issue
Block a user