Add RSS Feeds adapter (6th source) — commercial AI news from 10 feeds
Bug fix: date parsing was ISO-only (RFC 2822 feeds filtered out). Added email.utils.parsedate_to_datetime() fallback for RSS dates. 10 feeds: TechCrunch AI, VentureBeat AI, The Verge AI, AI News, The Decoder, MIT Tech Review AI, OpenAI Blog, Anthropic, Google AI, Meta AI. 3 dead feeds (Anthropic/Google/Meta 404), 7 working. Score: authority (primary blogs 1.5x, industry 1.3x) × recency decay (48h half-life) Age cutoff: 14 days. Score type: estimated. Wired into pipeline.py ENABLED_SOURCES. Added RSS URL verification. 3/3 spot-checks passed (all URLs reachable, HTTP 200). Commit: 62031be→c24a89f
This commit is contained in:
+17
-1
@@ -33,10 +33,11 @@ ADAPTERS = {
|
||||
"reddit": lambda: __import__("adapters.reddit", fromlist=["RedditAdapter"]).RedditAdapter(),
|
||||
"hackernews": lambda: __import__("adapters.hackernews", fromlist=["HackerNewsAdapter"]).HackerNewsAdapter(),
|
||||
"huggingface": lambda: __import__("adapters.huggingface", fromlist=["HuggingFaceAdapter"]).HuggingFaceAdapter(),
|
||||
"rss": lambda: __import__("adapters.rss_feeds", fromlist=["RSSFeedsAdapter"]).RSSFeedsAdapter(),
|
||||
}
|
||||
|
||||
# Default enabled sources
|
||||
ENABLED_SOURCES = ["github", "arxiv", "reddit", "hackernews", "huggingface"]
|
||||
ENABLED_SOURCES = ["github", "arxiv", "reddit", "hackernews", "huggingface", "rss"]
|
||||
|
||||
|
||||
def init_db(db_path: str, schema_path: str) -> sqlite3.Connection:
|
||||
@@ -203,6 +204,21 @@ def verify_entries(conn: sqlite3.Connection, source: str, sample_size: int = 3):
|
||||
except Exception as e:
|
||||
print(f" ? [{eid}] {title[:60]}... verify failed: {e}")
|
||||
|
||||
elif src == "rss":
|
||||
# For RSS, verify the article URL is reachable
|
||||
if url:
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "ai-oracle/0.1"})
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
status = resp.status
|
||||
if status in (200, 301, 302):
|
||||
print(f" ✓ [{eid}] {title[:60]}... URL reachable (HTTP {status})")
|
||||
passed += 1
|
||||
else:
|
||||
print(f" ? [{eid}] {title[:60]}... HTTP {status}")
|
||||
except Exception as e:
|
||||
print(f" ? [{eid}] {title[:60]}... verify failed: {e}")
|
||||
|
||||
time.sleep(0.5) # polite spacing
|
||||
|
||||
return passed > 0
|
||||
|
||||
Reference in New Issue
Block a user