#!/usr/bin/env python3 """ RSS Feeds adapter for Athena. Ingests commercial AI news from curated RSS feeds — no auth, no scraping, reliable structured data. Fills the "commercial AI news" gap: corporate moves, funding, product launches, partnerships, policy. Genuinely different signal from GitHub/arXiv/Reddit/HN/HF. Feeds selected for AI relevance, reliability, and freshness. Techmeme skipped (requires JS rendering). Axios/The Decoder kept despite 3xx status codes (feedparser follows redirects fine). Usage: python3 adapters/rss_feeds.py """ import json import os import re import time import sqlite3 import feedparser from datetime import datetime, timedelta, timezone from email.utils import parsedate_to_datetime from adapters import SourceAdapter from adapters._store import true_first_seen, upsert_entries # Curated feed list — AI-focused, reliable, diverse publishers. # Format: (source_key, label, url) # Each source_key becomes the `source` field in the DB. FEEDS = [ # Industry news ("rss:techcrunch", "TechCrunch AI", "https://techcrunch.com/category/artificial-intelligence/feed/"), ("rss:venturebeat", "VentureBeat AI", "https://venturebeat.com/category/ai/feed/"), ("rss:theverge", "The Verge AI", "https://www.theverge.com/rss/ai-artificial-intelligence/index.xml"), ("rss:ainews", "AI News", "https://www.artificialintelligence-news.com/feed/"), # Analysis ("rss:decoder", "The Decoder", "https://www.the-decoder.com/feed/"), ("rss:mittr", "MIT Tech Review AI", "https://www.technologyreview.com/topic/artificial-intelligence/feed/"), # Company blogs (primary signals for launches) # NOTE: anthropic/googleai/metaai RSS feeds are DEAD (404 as of 2026-07-12). # Replaced with working equivalents: DeepMind RSS, MIT Tech Review, The Decoder. ("rss:openai", "OpenAI Blog", "https://openai.com/blog/rss.xml"), ("rss:deepmind", "Google DeepMind Blog", "https://deepmind.google/blog/rss.xml"), ("rss:mittr", "MIT Tech Review AI", "https://www.technologyreview.com/topic/artificial-intelligence/feed/"), ("rss:decoder", "The Decoder", "https://www.the-decoder.com/feed/"), ] # AI relevance keywords for filtering — word-boundary matching # to avoid false positives ("Great Britain" != "AI") AI_KEYWORDS = [ # Core AI terms r"\bai\b", r"\bmachine learning\b", r"\bdeep learning\b", r"\bneural\b", r"\bgenerative ai\b", r"\bgenerative\b", r"\bllm\b", r"\blarge language\b", r"\bfoundation model\b", r"\btransformer\b", r"\baugmented\b", # Agents & automation r"\bagent\b", r"\bautonomous\b", r"\bmcp\b", r"\bfunction call\b", r"\btool use\b", r"\brai\b", r"\bretrieval\b", # Models & inference r"\binference\b", r"\bmodel\b", r"\bembedding\b", r"\btoken\b", r"\bfine-tune\b", r"\btraining\b", r"\bquantiz\b", # Specific models (current landscape) r"\bclaude\b", r"\bgpt\b", r"\bgemini\b", r"\bllama\b", r"\bdeepseek\b", r"\bqwen\b", r"\bmistral\b", r"\bcodex\b", r"\bchatgpt\b", r"\bsora\b", r"\bkimi\b", r"\bglm\b", r"\bminimax\b", # Companies r"\bopenai\b", r"\banthropic\b", r"\bgoogle\b", r"\bmeta\b", r"\bmicrosoft\b", r"\bnvidia\b", r"\bamazon\b", r"\baws\b", r"\bsamba(?=\w*nova\b)", r"\bcohere\b", r"\baider\b", ] class RSSFeedsAdapter(SourceAdapter): """RSS feed aggregator for commercial AI news.""" def name(self) -> str: return "rss" def _is_ai_relevant(self, title: str, summary: str, tags: list) -> bool: """Check if an entry is AI-relevant using word-boundary keyword matching.""" text = (title + " " + summary).lower() tag_text = " ".join(tags).lower() combined = text + " " + tag_text for pattern in AI_KEYWORDS: if re.search(pattern, combined): return True return False def _score(self, entry: dict, source_label: str, age_hours: float) -> float: """Score: source authority + recency. Estimated (no native popularity metric). Source authority weights (based on AI signal quality): - OpenAI/Anthropic/Google/Meta blogs: 1.5 (primary signals) - TechCrunch/VentureBeat: 1.3 (industry news) - The Verge/AI News/The Decoder/MIT TR: 1.0 (general coverage) Recency decay: exponential half-life of 48 hours. """ # Source authority authority = 1.0 primary_blogs = ("openai", "anthropic", "googleai", "metaai") industry_news = ("techcrunch", "venturebeat") source_key = entry.get("source_key", "") if source_key in primary_blogs: authority = 1.5 elif source_key in industry_news: authority = 1.3 # Recency decay: half-life = 48 hours import math recency = 2 ** (-age_hours / 48.0) # 1.0 at t=0, 0.5 at 48h, 0.25 at 96h # Tag bonus: more tags = more signal tag_count = len(entry.get("tags", [])) tag_bonus = min(tag_count * 0.1, 0.5) # max +0.5 score = (authority * recency * 4.0) + tag_bonus return min(round(score, 2), 10.0) def _tags(self, entry: dict) -> list: """Generate category tags from feed entry metadata.""" tags = ["rss", entry.get("source_key", "rss:unknown")] # Add feed-provided tags for t in entry.get("tags", []): clean = re.sub(r"[^a-z0-9-]", "-", t.lower()) tags.append(f"tag:{clean}") return tags def fetch(self, query: str = "", limit: int = 20) -> list[dict]: """Fetch entries from all configured RSS feeds. Returns the top `limit` entries across all feeds, sorted by score. Only AI-relevant entries are included. """ now = datetime.now(timezone.utc) all_entries = [] feed_failures = [] for source_key, label, url in FEEDS: try: d = feedparser.parse(url) if d.status not in (200, 301, 302, 307, 308) or not d.entries: feed_failures.append(f"{source_key}: status={d.status}, entries=0") continue for entry in d.entries: if not self._is_ai_relevant( entry.get("title", ""), entry.get("summary", ""), [t.get("term", "") for t in entry.get("tags", [])], ): continue # Parse publish time — handle RFC 2822 (most RSS) and ISO 8601 published = entry.get("published", entry.get("updated", "")) age_hours = 999 # default: old if published: try: pub_dt = parsedate_to_datetime(published) except (ValueError, TypeError): try: pub_dt = datetime.fromisoformat(published.replace("Z", "+00:00")) except (ValueError, TypeError): pass else: age_hours = max((now - pub_dt).total_seconds() / 3600, 0.1) # Skip entries older than 14 days if age_hours > 14 * 24: continue tags = [t.get("term", "") for t in entry.get("tags", [])] entry_dict = { "source_key": source_key, "source_label": label, "age_hours": age_hours, "tags": tags, } score = self._score(entry_dict, label, age_hours) entry_dict["score"] = score now_iso = now.strftime("%Y-%m-%dT%H:%M:%SZ") first_seen = true_first_seen( {"published": published}, "rss", now_iso) all_entries.append({ "source": "rss", "source_id": f"{source_key}:{entry.get('id', entry.get('link', ''))[-30:]}", "url": entry.get("link", ""), "title": entry.get("title", ""), "extracted_text": entry.get("summary", ""), "summary": None, # LLM later "category_tags": json.dumps(self._tags(entry_dict)), "signal_score": score, "raw_metadata": json.dumps({ "source_key": source_key, "source_label": label, "author": entry.get("author", ""), "published": published, "age_hours": round(age_hours, 1), "tags": tags, "score_type": "estimated", }), "first_seen": first_seen, "last_updated": now_iso, }) time.sleep(0.5) # polite spacing except Exception as e: feed_failures.append(f"{source_key}: {e}") if feed_failures: print(f" ⚠ Feed failures: {', '.join(feed_failures)}") # Sort by score descending, take top N all_entries.sort(key=lambda e: e["signal_score"], reverse=True) return all_entries[:limit] if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description="RSS feeds adapter for Athena") parser.add_argument("--limit", type=int, default=20, help="Max entries") parser.add_argument("--dry-run", action="store_true", help="Don't store in DB") parser.add_argument("--db", default=os.path.join(os.path.dirname(__file__), "..", "oracle.db")) parser.add_argument("--schema", default=os.path.join(os.path.dirname(__file__), "..", "schema.sql")) args = parser.parse_args() print(f"=== RSS Feeds Adapter ({len(FEEDS)} feeds) ===") adapter = RSSFeedsAdapter() entries = adapter.fetch(limit=args.limit * 2) # fetch extra for relevance filtering entries = entries[:args.limit] print(f"\n Fetched {len(entries)} AI-relevant entries from {len(FEEDS)} feeds") print() if entries: print(f" {'Rank':<4} {'Score':<7} {'Age':<8} {'Source':<16} {'Title'}") print(" " + "-" * 90) for i, e in enumerate(entries[:15], 1): meta = json.loads(e["raw_metadata"]) print(f" {i:<4} {e['signal_score']:<7.2f} {meta['age_hours']:<8.1f}h {meta['source_label']:<16} {e['title'][:70]}") if not args.dry_run: conn = sqlite3.connect(args.db) if os.path.exists(args.schema): with open(args.schema) as f: conn.executescript(f.read()) conn.commit() cur = conn.cursor() stored = upsert_entries(conn, entries) conn.commit() conn.close() print(f"\n Stored {stored} entries") print(f"\n Done.")