Initial commit: Oracle AI research pipeline (adapters, pipeline, summarize, query)
Source-controlled baseline before Phase 5 cron. Excludes oracle.db, logs/, and __pycache__ via .gitignore. Pipeline verified running clean end-to-end (run_log write confirmed before conn.close()).
This commit is contained in:
Epictetus
@@ -0,0 +1,17 @@
|
||||
"""Source adapters for AI Research Oracle."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class SourceAdapter(ABC):
|
||||
"""Base class for all ingestion adapters."""
|
||||
|
||||
@abstractmethod
|
||||
def name(self) -> str:
|
||||
"""Source name: 'github', 'arxiv', 'reddit'."""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def fetch(self, query: str = "", limit: int = 20) -> list[dict]:
|
||||
"""Return entries matching DB schema fields."""
|
||||
pass
|
||||
@@ -0,0 +1,527 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
arXiv adapter for AI Research Oracle.
|
||||
Fetches recent AI/ML papers via arXiv API (Atom XML).
|
||||
No scraping — uses the official API endpoint.
|
||||
|
||||
Rate limits: 1 req/3s (be polite). arXiv enforces this aggressively.
|
||||
|
||||
============================================================================
|
||||
SCORING DESIGN PRINCIPLE (do not violate when copying this to other adapters)
|
||||
============================================================================
|
||||
Structural metadata (recency, author count, abstract length, category
|
||||
diversity) is a WEAK signal present in EVERY paper regardless of topic. It
|
||||
must NEVER dominate the final score. Content-relevance signals (AI keyword
|
||||
density in the abstract, methodology-contribution detection) MUST carry the
|
||||
weight. A paper that merely USES an existing AI tool as incidental methodology
|
||||
scores lower than one that CONTRIBUTES a new method — detection is by verb
|
||||
(creative "we propose/introduce" vs evaluative "we evaluate/using"), not by
|
||||
domain. Applied-domain papers are TAGGED for filtering, NEVER penalized: for a
|
||||
startup-idea oracle, novel AI applied to a vertical is desired signal.
|
||||
(See _score() and _methodology_claim_score() for the implementation.)
|
||||
============================================================================
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from html import unescape
|
||||
|
||||
from adapters import SourceAdapter
|
||||
|
||||
# arXiv API
|
||||
ARXIV_API = "http://export.arxiv.org/api/query"
|
||||
|
||||
|
||||
class ArxivAdapter(SourceAdapter):
|
||||
"""arXiv API adapter."""
|
||||
|
||||
# Default categories to scan
|
||||
DEFAULT_CATEGORIES = ["cs.AI", "cs.LG", "cs.CL"]
|
||||
|
||||
def __init__(self, categories=None, rate_limit=3):
|
||||
"""
|
||||
Args:
|
||||
categories: List of arXiv categories. Default: cs.AI, cs.LG, cs.CL
|
||||
rate_limit: Seconds between API calls (default 3).
|
||||
"""
|
||||
self.categories = categories or self.DEFAULT_CATEGORIES
|
||||
self.rate_limit = rate_limit
|
||||
|
||||
def name(self) -> str:
|
||||
return "arxiv"
|
||||
|
||||
def _parse_atom(self, xml_data: str) -> list[dict]:
|
||||
"""Parse arXiv Atom XML into raw paper dicts."""
|
||||
root = ET.fromstring(xml_data)
|
||||
|
||||
# Handle namespaces — arXiv uses multiple
|
||||
# Walk all elements and extract what we need
|
||||
papers = []
|
||||
|
||||
for entry in root.iter():
|
||||
tag = entry.tag.split("}")[-1] if "}" in entry.tag else entry.tag
|
||||
if tag == "entry":
|
||||
paper = {
|
||||
"id": "",
|
||||
"title": "",
|
||||
"summary": "",
|
||||
"published": "",
|
||||
"updated": "",
|
||||
"authors": [],
|
||||
"categories": [],
|
||||
"comment": "",
|
||||
"journal_ref": "",
|
||||
"doi": "",
|
||||
"link": "",
|
||||
}
|
||||
|
||||
for child in entry:
|
||||
ctag = child.tag.split("}")[-1]
|
||||
|
||||
if ctag == "id":
|
||||
# arXiv ID like http://arxiv.org/abs/cs.AI/2607.00123
|
||||
paper["id"] = child.text or ""
|
||||
# Also extract clean ID
|
||||
if "abs/" in paper["id"]:
|
||||
paper["arxiv_id"] = paper["id"].split("abs/")[-1]
|
||||
|
||||
elif ctag == "title":
|
||||
paper["title"] = unescape((child.text or "").strip())
|
||||
|
||||
elif ctag == "summary":
|
||||
paper["summary"] = unescape((child.text or "").strip())
|
||||
|
||||
elif ctag == "published":
|
||||
paper["published"] = child.text or ""
|
||||
|
||||
elif ctag == "updated":
|
||||
paper["updated"] = child.text or ""
|
||||
|
||||
elif ctag == "author":
|
||||
for ac in child:
|
||||
a_tag = ac.tag.split("}")[-1]
|
||||
if a_tag == "name":
|
||||
paper["authors"].append(unescape((ac.text or "").strip()))
|
||||
|
||||
elif ctag == "category":
|
||||
term = child.get("term", "")
|
||||
if term:
|
||||
paper["categories"].append(term)
|
||||
|
||||
elif ctag == "arxiv":
|
||||
# arXiv-specific: comment, journal_ref, doi
|
||||
sub_tag = child.tag.split("}")[-1]
|
||||
if sub_tag == "comment":
|
||||
paper["comment"] = unescape((child.text or "").strip())
|
||||
elif sub_tag == "journal_ref":
|
||||
paper["journal_ref"] = unescape((child.text or "").strip())
|
||||
elif sub_tag == "doi":
|
||||
paper["doi"] = unescape((child.text or "").strip())
|
||||
|
||||
elif ctag == "link":
|
||||
href = child.get("href", "")
|
||||
# Prefer the abstract page link
|
||||
if "abs/" in href and not paper["link"]:
|
||||
paper["link"] = href
|
||||
elif "pdf/" in href:
|
||||
paper["pdf_link"] = href
|
||||
|
||||
# Clean up arxiv_id if not extracted from ID field
|
||||
if "arxiv_id" not in paper and "abs/" in paper.get("id", ""):
|
||||
paper["arxiv_id"] = paper["id"].split("abs/")[-1]
|
||||
|
||||
if paper["title"] and paper["summary"]:
|
||||
papers.append(paper)
|
||||
|
||||
return papers
|
||||
|
||||
def _request(self, query: str, max_results: int = 20, sort_by="submittedDate") -> list[dict]:
|
||||
"""Make an arXiv API request."""
|
||||
url = (
|
||||
f"{ARXIV_API}"
|
||||
f"?search_query={urllib.parse.quote(query)}"
|
||||
f"&sortBy={sort_by}"
|
||||
f"&sortOrder=descending"
|
||||
f"&max_results={max_results}"
|
||||
)
|
||||
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "ai-oracle/0.1"})
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
xml_data = resp.read().decode("utf-8")
|
||||
return self._parse_atom(xml_data)
|
||||
except urllib.error.HTTPError as e:
|
||||
print(f" HTTP {e.code} for arXiv query")
|
||||
return []
|
||||
except Exception as e:
|
||||
print(f" arXiv request error: {e}")
|
||||
return []
|
||||
|
||||
def _score(self, paper: dict, age_days: float) -> float:
|
||||
"""Score based on AI-methodology relevance, not structural metadata.
|
||||
|
||||
arXiv has no upvotes/stars. The key signal is whether the paper is a
|
||||
genuine AI/ML *contribution* (new architecture, framework, method) vs.
|
||||
merely *using* an existing AI tool as incidental methodology.
|
||||
|
||||
Structural signals (recency, author count, category diversity) are
|
||||
weak and capped low so they cannot dominate the score — any paper has
|
||||
them regardless of topic. The relevance signals (methodology claim +
|
||||
AI keyword density) carry the weight.
|
||||
|
||||
Applied-domain papers (healthcare, finance, biology) are TAGGED for
|
||||
filtering but NOT penalized — for a startup-idea oracle, "novel AI
|
||||
technique applied to a vertical" is exactly the signal we want.
|
||||
"""
|
||||
# --- Structural signals (weak, capped low) ---
|
||||
# Recency: newer = slightly higher, but max 2.0 (was 5.0)
|
||||
try:
|
||||
pub_str = paper.get("published", "")
|
||||
pub_dt = datetime.fromisoformat(pub_str.replace("Z", "+00:00"))
|
||||
now = datetime.now(timezone.utc)
|
||||
age_hours = (now - pub_dt).total_seconds() / 3600
|
||||
recency = max(0, 2.0 - age_hours / 48.0)
|
||||
except (ValueError, TypeError):
|
||||
recency = max(0, 2.0 - age_days * 0.1)
|
||||
|
||||
# Author count: collaborative work is a weak signal, max 0.5 (was 2.0)
|
||||
author_count = len(paper.get("authors", []))
|
||||
author_bonus = min(author_count * 0.05, 0.5)
|
||||
|
||||
# Category diversity: cross-domain is mildly interesting, max 0.3
|
||||
cats = paper.get("categories", [])
|
||||
diversity_bonus = min(len(cats) * 0.1, 0.3)
|
||||
|
||||
# Journal/DOI: published = validated, small bonus
|
||||
journal_bonus = 0.5 if paper.get("journal_ref") else 0.0
|
||||
doi_bonus = 0.2 if paper.get("doi") else 0.0
|
||||
|
||||
# Comment field (page count etc.): tiny bonus
|
||||
comment = paper.get("comment", "")
|
||||
comment_bonus = min(len(comment) / 300.0, 0.5) if comment else 0.0
|
||||
|
||||
# --- Relevance signals (carry the weight) ---
|
||||
title_lower = paper.get("title", "").lower()
|
||||
abstract = paper.get("summary", "")
|
||||
abstract_lower = abstract.lower()
|
||||
|
||||
# AI keyword density in abstract (not just title)
|
||||
ai_keywords = [
|
||||
"agent", "llm", "large language", "gpt", "transformer",
|
||||
"neural", "deep learning", "machine learning", "reinforcement",
|
||||
"diffusion", "multimodal", "embedding", "attention", "rag",
|
||||
"retrieval", "fine-tun", "pretrain", "self-supervised",
|
||||
"foundation model", "reasoning", "alignment", "policy gradient",
|
||||
"graph neural", "vision-language", "vla", "vlm",
|
||||
]
|
||||
ai_kw_hits = sum(1 for kw in ai_keywords if kw in abstract_lower)
|
||||
# Density matters more than single mention; saturate at ~6 hits
|
||||
ai_keyword_bonus = min(ai_kw_hits * 0.35, 2.0)
|
||||
|
||||
# Title keyword bonus (small)
|
||||
title_kw_list = ["agent", "llm", "reasoning", "verif", "multimodal",
|
||||
"embodied", "robot", "alignment", "autonomous"]
|
||||
title_kw_hits = sum(1 for kw in title_kw_list if kw in title_lower)
|
||||
title_kw_bonus = min(title_kw_hits * 0.2, 0.5)
|
||||
|
||||
# AI methodology claim: does the abstract propose/invent something?
|
||||
methodology_bonus = self._methodology_claim_score(abstract_lower)
|
||||
|
||||
# --- Domain tag (no penalty) ---
|
||||
combined_text = f"{title_lower} {abstract_lower}"
|
||||
applied_domains = {
|
||||
"applied:healthcare": ["gastric", "biopsy", "pathology", "clinical",
|
||||
"patient", "diagnosis", "medical imaging", "h. pylori",
|
||||
"cancer screening", "oncology", "neurology", "cardiology"],
|
||||
"applied:finance": ["stock market", "trading", "portfolio", "forex",
|
||||
"cryptocurrency", "fintech", "credit scoring", "fraud detection"],
|
||||
"applied:biology": ["protein folding", "gene expression", "genome",
|
||||
"molecular", "bioinformatics", "cell type", "organism"],
|
||||
}
|
||||
primary_domain = None
|
||||
for domain, domain_keywords in applied_domains.items():
|
||||
hits = sum(1 for kw in domain_keywords if kw in combined_text)
|
||||
if hits >= 2: # need at least 2 domain keywords to trigger
|
||||
primary_domain = domain
|
||||
break
|
||||
if primary_domain:
|
||||
paper["_applied_domain"] = primary_domain
|
||||
|
||||
# Total: structural (capped ~3.5) + relevance (capped ~5.0)
|
||||
structural = recency + author_bonus + diversity_bonus + journal_bonus + doi_bonus + comment_bonus
|
||||
relevance = ai_keyword_bonus + title_kw_bonus + methodology_bonus
|
||||
score = structural + relevance
|
||||
|
||||
return min(round(score, 2), 10.0)
|
||||
|
||||
def _methodology_claim_score(self, abstract_lower: str) -> float:
|
||||
"""Detect whether the abstract proposes a NEW AI method vs. using an existing one.
|
||||
|
||||
The core distinction is the VERB: creative verbs (propose/introduce/
|
||||
develop/design) signal a contribution; evaluative verbs (evaluate/
|
||||
conduct/retrospective/using/based on) signal applying an existing tool.
|
||||
|
||||
Check weak (evaluative) patterns FIRST — if the paper is an evaluation
|
||||
or application of an existing system, it gets 0.5 even if it also says
|
||||
'we present'. Only if no weak pattern matches do we look for a genuine
|
||||
contribution claim.
|
||||
|
||||
Returns 0.0 (no claim / unclear), 0.5 (uses existing tool),
|
||||
2.0 (novel method), or 2.5 (explicit proposal of new AI system).
|
||||
"""
|
||||
# Weak FIRST: evaluative / application language
|
||||
weak_markers = ["we evaluate", "we conducted", "we conduct", "retrospective",
|
||||
"case study", "pilot study", "using", "we leverage",
|
||||
"we applied", "based on", "we report a", "we present a retrospective",
|
||||
"an evaluation of", "we benchmark", "empirical study"]
|
||||
for wm in weak_markers:
|
||||
if wm in abstract_lower:
|
||||
return 0.5
|
||||
|
||||
# Strong: genuinely creative verbs + AI noun within 60 chars
|
||||
proposal_verbs = ["we propose", "we introduce", "we develop", "we design",
|
||||
"we formulate", "we construct", "we build", "we present a novel",
|
||||
"we present a new"]
|
||||
ai_nouns = ["model", "architecture", "framework", "method", "approach",
|
||||
"agent", "system", "network", "algorithm", "pipeline",
|
||||
"llm", "transformer", "policy", "graph", "orchestration",
|
||||
"harness", "memory", "solver", "planner"]
|
||||
for verb in proposal_verbs:
|
||||
idx = abstract_lower.find(verb)
|
||||
if idx != -1:
|
||||
window = abstract_lower[idx:idx + 60]
|
||||
if any(noun in window for noun in ai_nouns):
|
||||
return 2.5
|
||||
|
||||
# Strong-ish: 'novel'/'new' + AI architecture/framework/model within 40 chars
|
||||
for marker in ["novel", "new"]:
|
||||
idx = abstract_lower.find(marker)
|
||||
scan = 0
|
||||
while idx != -1 and scan < 5:
|
||||
window = abstract_lower[idx:idx + 40]
|
||||
if any(noun in window for noun in
|
||||
["architecture", "framework", "model", "method",
|
||||
"approach", "agent", "system", "harness"]):
|
||||
return 2.0
|
||||
idx = abstract_lower.find(marker, idx + 1)
|
||||
scan += 1
|
||||
|
||||
return 0.0
|
||||
|
||||
def _tags(self, paper: dict) -> list:
|
||||
"""Generate category tags from arXiv metadata."""
|
||||
tags = ["arxiv"]
|
||||
|
||||
# Add arXiv categories
|
||||
for cat in paper.get("categories", [])[:5]:
|
||||
tags.append(f"cat:{cat}")
|
||||
|
||||
# Primary signal categories
|
||||
cats_lower = [c.lower() for c in paper.get("categories", [])]
|
||||
if "cs.ai" in cats_lower:
|
||||
tags.append("ai-general")
|
||||
if "cs.lg" in cats_lower:
|
||||
tags.append("machine-learning")
|
||||
if "cs.cl" in cats_lower:
|
||||
tags.append("nlp")
|
||||
if "cs.cv" in cats_lower:
|
||||
tags.append("computer-vision")
|
||||
if "cs.ro" in cats_lower:
|
||||
tags.append("robotics")
|
||||
if "cs.se" in cats_lower:
|
||||
tags.append("software-engineering")
|
||||
|
||||
# Check if it's a survey/tutorial
|
||||
title_lower = paper.get("title", "").lower()
|
||||
summary_lower = paper.get("summary", "").lower()
|
||||
if any(kw in summary_lower for kw in ["survey", "tutorial", "overview of", "review of"]):
|
||||
tags.append("survey")
|
||||
|
||||
# Check for preprint vs published
|
||||
if paper.get("journal_ref"):
|
||||
tags.append("published")
|
||||
else:
|
||||
tags.append("preprint")
|
||||
|
||||
# Applied-domain tag
|
||||
if paper.get("_applied_domain"):
|
||||
tags.append(paper["_applied_domain"])
|
||||
|
||||
return tags
|
||||
|
||||
def fetch(self, query: str = "", limit: int = 20) -> list[dict]:
|
||||
"""
|
||||
Fetch papers from arXiv.
|
||||
|
||||
If query is empty, fetch recent papers from configured categories.
|
||||
If query is provided, search for it.
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
all_papers = []
|
||||
|
||||
if query:
|
||||
print(f" Searching arXiv: '{query}'")
|
||||
papers = self._request(query, max_results=limit, sort_by="submittedDate")
|
||||
all_papers.extend(papers)
|
||||
else:
|
||||
# Fetch from each configured category
|
||||
for cat in self.categories:
|
||||
q = f"cat:{cat}"
|
||||
cat_papers = self._request(q, max_results=limit, sort_by="submittedDate")
|
||||
all_papers.extend(cat_papers)
|
||||
time.sleep(self.rate_limit)
|
||||
|
||||
# Deduplicate by arxiv_id
|
||||
seen = set()
|
||||
unique = []
|
||||
for p in all_papers:
|
||||
pid = p.get("arxiv_id", p.get("id", ""))
|
||||
if pid and pid not in seen:
|
||||
seen.add(pid)
|
||||
unique.append(p)
|
||||
all_papers = unique
|
||||
|
||||
# Sort by published date (newest first), take top limit
|
||||
all_papers.sort(
|
||||
key=lambda p: p.get("published", ""),
|
||||
reverse=True,
|
||||
)
|
||||
all_papers = all_papers[:limit]
|
||||
|
||||
entries = []
|
||||
for paper in all_papers:
|
||||
# Calculate age
|
||||
published = paper.get("published", "")
|
||||
try:
|
||||
pub_dt = datetime.fromisoformat(published.replace("Z", "+00:00"))
|
||||
age_days = (now - pub_dt).days
|
||||
except (ValueError, TypeError):
|
||||
age_days = 0
|
||||
|
||||
score = self._score(paper, age_days)
|
||||
tags = self._tags(paper)
|
||||
|
||||
# Source ID: arxiv_id
|
||||
source_id = paper.get("arxiv_id", paper.get("id", "")).split("/")[-1]
|
||||
|
||||
# URL: abstract page
|
||||
url = paper.get("link", "")
|
||||
if not url and paper.get("arxiv_id"):
|
||||
url = f"https://arxiv.org/abs/{paper['arxiv_id']}"
|
||||
|
||||
# Title (clean — remove trailing category markers)
|
||||
title = re.sub(r"\s*\([A-Za-z0-9., ]*\)\s*$", "", paper.get("title", "")).strip()
|
||||
|
||||
# Extracted text: summary (abstract) — already clean
|
||||
extracted_text = paper.get("summary", "")
|
||||
|
||||
# Structured metadata
|
||||
raw_meta = {
|
||||
"arxiv_id": paper.get("arxiv_id", ""),
|
||||
"authors": paper.get("authors", []),
|
||||
"author_count": len(paper.get("authors", [])),
|
||||
"categories": paper.get("categories", []),
|
||||
"published": paper.get("published", ""),
|
||||
"updated": paper.get("updated", ""),
|
||||
"comment": paper.get("comment", ""),
|
||||
"journal_ref": paper.get("journal_ref", ""),
|
||||
"doi": paper.get("doi", ""),
|
||||
"pdf_link": paper.get("pdf_link", ""),
|
||||
"age_days": age_days,
|
||||
"abstract_length": len(extracted_text),
|
||||
"score_type": "estimated", # arXiv has no upvotes
|
||||
}
|
||||
# Store applied-domain info if detected
|
||||
if paper.get("_applied_domain"):
|
||||
raw_meta["applied_domain"] = paper["_applied_domain"]
|
||||
|
||||
now_str = now.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
entries.append({
|
||||
"source": "arxiv",
|
||||
"source_id": source_id,
|
||||
"url": url,
|
||||
"title": title,
|
||||
"extracted_text": extracted_text,
|
||||
"summary": None,
|
||||
"category_tags": json.dumps(tags),
|
||||
"signal_score": score,
|
||||
"raw_metadata": json.dumps(raw_meta),
|
||||
"first_seen": now_str,
|
||||
"last_updated": now_str,
|
||||
})
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
import sqlite3
|
||||
import urllib.parse
|
||||
|
||||
parser = argparse.ArgumentParser(description="arXiv adapter for AI Research Oracle")
|
||||
parser.add_argument("--query", default="", help="Search query (empty = recent categories)")
|
||||
parser.add_argument("--limit", type=int, default=20, help="Max entries")
|
||||
parser.add_argument("--db", default=os.path.join(os.path.dirname(__file__), "..", "oracle.db"), help="SQLite DB")
|
||||
parser.add_argument("--schema", default=os.path.join(os.path.dirname(__file__), "..", "schema.sql"), help="Schema file")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Don't store in DB")
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"=== arXiv Adapter ===")
|
||||
print(f" Query: {args.query or '(recent AI categories)'}")
|
||||
print(f" Limit: {args.limit}")
|
||||
print()
|
||||
|
||||
adapter = ArxivAdapter()
|
||||
entries = adapter.fetch(query=args.query, limit=args.limit)
|
||||
|
||||
print(f" Fetched {len(entries)} entries")
|
||||
|
||||
if not args.dry_run:
|
||||
conn = sqlite3.connect(args.db)
|
||||
if os.path.exists(args.schema):
|
||||
with open(args.schema) as f:
|
||||
conn.executescript(f.read())
|
||||
conn.commit()
|
||||
|
||||
cur = conn.cursor()
|
||||
stored = 0
|
||||
for entry in entries:
|
||||
try:
|
||||
cur.execute("""
|
||||
INSERT OR REPLACE INTO entries
|
||||
(source, source_id, url, title, extracted_text, summary,
|
||||
category_tags, signal_score, raw_metadata, first_seen, last_updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
entry["source"], entry["source_id"], entry["url"], entry["title"],
|
||||
entry["extracted_text"], entry["summary"],
|
||||
entry["category_tags"], entry["signal_score"],
|
||||
entry["raw_metadata"], entry["first_seen"], entry["last_updated"],
|
||||
))
|
||||
stored += 1
|
||||
except Exception as e:
|
||||
print(f" DB error: {e}")
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f" Stored {stored} entries")
|
||||
|
||||
# Print top 5
|
||||
print(f"\n Top entries:")
|
||||
for i, e in enumerate(entries[:5]):
|
||||
meta = json.loads(e["raw_metadata"]) if isinstance(e["raw_metadata"], str) else e["raw_metadata"]
|
||||
authors = meta.get("authors", [])
|
||||
author_str = f"{authors[0]} et al." if len(authors) > 2 else ", ".join(authors[:2])
|
||||
print(f" [{i+1}] score={e['signal_score']:.2f} authors={author_str}")
|
||||
print(f" {e['title'][:90]}")
|
||||
print(f" {e['url']}")
|
||||
print(f" abstract={meta.get('abstract_length', 0)}ch")
|
||||
|
||||
print(f"\n Done.")
|
||||
@@ -0,0 +1,311 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
GitHub adapter for AI Research Oracle.
|
||||
Fetches repos via GitHub REST API — no scraping, no trending page.
|
||||
Definition of "trending": created or pushed in last N days, sorted by stars.
|
||||
|
||||
Rate limits: 60 req/hr unauthenticated.
|
||||
Strategy: cache results, deduplicate, budget calls.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from adapters import SourceAdapter
|
||||
|
||||
|
||||
class GitHubAdapter(SourceAdapter):
|
||||
"""GitHub REST API adapter."""
|
||||
|
||||
BASE = "https://api.github.com"
|
||||
|
||||
def __init__(self, token: str = None):
|
||||
"""Initialize with optional read-only token (5000 req/hr vs 60)."""
|
||||
self.token = token or os.environ.get("GITHUB_TOKEN", "")
|
||||
self.cache = {}
|
||||
|
||||
def name(self) -> str:
|
||||
return "github"
|
||||
|
||||
def _headers(self):
|
||||
headers = {
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
"User-Agent": "ai-oracle/0.1",
|
||||
}
|
||||
if self.token:
|
||||
headers["Authorization"] = f"token {self.token}"
|
||||
return headers
|
||||
|
||||
def _request(self, url: str, max_retries: int = 2) -> dict | list | None:
|
||||
"""Make a GET request with retry on 403 (rate limit)."""
|
||||
req = urllib.request.Request(url, headers=self._headers())
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
# Check rate limit headers
|
||||
remaining = int(resp.headers.get("X-RateLimit-Remaining", 0))
|
||||
if remaining <= 5:
|
||||
print(f" ⚠ Rate limit low ({remaining} remaining), stopping")
|
||||
break
|
||||
|
||||
return data
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 403:
|
||||
# Rate limited — reset time is in headers
|
||||
reset = int(e.headers.get("X-RateLimit-Reset", 0))
|
||||
if reset:
|
||||
wait = max(reset - int(time.time()), 0) + 1
|
||||
print(f" ⚠ Rate limited, wait {wait}s")
|
||||
else:
|
||||
wait = 30 * (attempt + 1)
|
||||
print(f" 403 on attempt {attempt + 1}, retry in {wait}s")
|
||||
time.sleep(min(wait, 300)) # cap at 5 min
|
||||
continue
|
||||
print(f" HTTP {e.code} for {url}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f" Request error: {e}")
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
def _search_repos(self, query: str, sort: str = "stars", order: str = "desc", per_page: int = 30) -> list:
|
||||
"""Search repositories via GitHub API."""
|
||||
url = f"{self.BASE}/search/repositories?q={urllib.parse.quote(query)}&sort={sort}&order={order}&per_page={per_page}"
|
||||
result = self._request(url)
|
||||
if isinstance(result, dict) and "items" in result:
|
||||
return result["items"]
|
||||
return []
|
||||
|
||||
def _get_readme(self, repo_url: str) -> str:
|
||||
"""Fetch README content for a repo."""
|
||||
if repo_url in self.cache:
|
||||
return self.cache[repo_url]
|
||||
|
||||
result = self._request(repo_url, max_retries=1)
|
||||
if result and isinstance(result, dict):
|
||||
import base64
|
||||
content = result.get("content", "")
|
||||
encoding = result.get("encoding", "base64")
|
||||
if encoding == "base64" and content:
|
||||
try:
|
||||
decoded = base64.b64decode(content).decode("utf-8", errors="replace")
|
||||
# Truncate to 8000 chars to keep context manageable
|
||||
self.cache[repo_url] = decoded[:8000]
|
||||
return decoded[:8000]
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
def _score(self, repo: dict, age_days: float) -> float:
|
||||
"""Score: stars weighted by recency. Higher = better signal."""
|
||||
stars = repo.get("stargazers_count", 0)
|
||||
# Normalize: log scale on stars, decay by age
|
||||
import math
|
||||
star_score = min(math.log1p(stars) / 2.0, 10.0) # log(1000) ≈ 6.9 → ~3.5
|
||||
# Recency bonus: newer repos get a slight boost
|
||||
recency_bonus = max(0, 1.0 - age_days / 30.0) * 1.5 # up to +1.5 for very recent
|
||||
return min(star_score + recency_bonus, 10.0)
|
||||
|
||||
def _tags(self, repo: dict) -> list:
|
||||
"""Generate category tags from repo metadata."""
|
||||
tags = ["github"]
|
||||
lang = repo.get("language", "")
|
||||
if lang:
|
||||
tags.append(f"lang:{lang.lower()}")
|
||||
topics = repo.get("topics", [])
|
||||
for t in topics[:5]:
|
||||
tags.append(f"topic:{t}")
|
||||
# Check description for AI/ML signals
|
||||
desc = (repo.get("description") or "").lower()
|
||||
if any(kw in desc for kw in ["agent", "agents"]):
|
||||
tags.append("agents")
|
||||
if any(kw in desc for kw in ["llm", "large language"]):
|
||||
tags.append("llm")
|
||||
if any(kw in desc for kw in ["rag", "retrieval"]):
|
||||
tags.append("rag")
|
||||
return tags
|
||||
|
||||
def fetch(self, query: str = "", limit: int = 20) -> list[dict]:
|
||||
"""
|
||||
Fetch repos from GitHub.
|
||||
|
||||
If query is empty, fetch "trending" = recent high-star AI repos.
|
||||
If query is provided, search for it.
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
if query:
|
||||
print(f" Searching: '{query}'")
|
||||
repos = self._search_repos(query, sort="stars", per_page=min(limit * 2, 100))
|
||||
else:
|
||||
# "Trending" = AI repos created in last 14 days, sorted by stars
|
||||
cutoff = (now - timedelta(days=14)).strftime("%Y-%m-%d")
|
||||
# Two queries to get breadth
|
||||
repos = []
|
||||
for q in ["ai agents created:>=2026-06-01", "llm inference created:>=2026-06-01"]:
|
||||
batch = self._search_repos(q, sort="stars", per_page=50)
|
||||
repos.extend(batch)
|
||||
time.sleep(1) # polite spacing
|
||||
|
||||
# Deduplicate by full_name
|
||||
seen = set()
|
||||
unique = []
|
||||
for r in repos:
|
||||
fn = r.get("full_name", "")
|
||||
if fn not in seen:
|
||||
seen.add(fn)
|
||||
unique.append(r)
|
||||
repos = unique
|
||||
|
||||
# Sort by stars descending, take top limit
|
||||
repos.sort(key=lambda r: r.get("stargazers_count", 0), reverse=True)
|
||||
repos = repos[:limit]
|
||||
|
||||
entries = []
|
||||
readme_budget = min(10, limit) # Only fetch README for top 10 to stay under rate limit
|
||||
for idx, repo in enumerate(repos):
|
||||
# Calculate age
|
||||
created = repo.get("created_at")
|
||||
if created:
|
||||
try:
|
||||
created_dt = datetime.fromisoformat(created.replace("Z", "+00:00"))
|
||||
age_days = (now - created_dt).days
|
||||
except (ValueError, TypeError):
|
||||
age_days = 0
|
||||
else:
|
||||
age_days = 0
|
||||
|
||||
score = self._score(repo, age_days)
|
||||
tags = self._tags(repo)
|
||||
|
||||
# Source ID: repo full_name
|
||||
source_id = repo.get("full_name", "").replace("/", "__")
|
||||
|
||||
# URL
|
||||
url = repo.get("html_url", "")
|
||||
|
||||
# Title: repo name with description
|
||||
name = repo.get("name", "")
|
||||
desc = repo.get("description", "") or ""
|
||||
if desc:
|
||||
title = f"{name}: {desc[:100]}"
|
||||
else:
|
||||
title = name
|
||||
|
||||
# README as extracted_text
|
||||
# Search API doesn't return readme_url — construct it from owner/repo
|
||||
owner = repo.get("owner", {}).get("login", "")
|
||||
repo_name = repo.get("name", "")
|
||||
readme_text = ""
|
||||
if owner and repo_name and idx < readme_budget:
|
||||
readme_text = self._get_readme(f"https://api.github.com/repos/{owner}/{repo_name}/readme")
|
||||
time.sleep(0.5) # polite spacing between README fetches
|
||||
|
||||
# Structured metadata
|
||||
raw_meta = {
|
||||
"full_name": repo.get("full_name", ""),
|
||||
"owner": repo.get("owner", {}).get("login", ""),
|
||||
"stars": repo.get("stargazers_count", 0),
|
||||
"forks": repo.get("forks_count", 0),
|
||||
"open_issues": repo.get("open_issues_count", 0),
|
||||
"language": repo.get("language", ""),
|
||||
"topics": repo.get("topics", []),
|
||||
"created_at": repo.get("created_at", ""),
|
||||
"updated_at": repo.get("updated_at", ""),
|
||||
"pushed_at": repo.get("pushed_at", ""),
|
||||
"age_days": age_days,
|
||||
"readme_length": len(readme_text),
|
||||
"score_type": "actual", # based on real star counts
|
||||
}
|
||||
|
||||
now_str = now.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
entries.append({
|
||||
"source": "github",
|
||||
"source_id": source_id,
|
||||
"url": url,
|
||||
"title": title,
|
||||
"extracted_text": readme_text,
|
||||
"summary": None, # LLM later
|
||||
"category_tags": json.dumps(tags),
|
||||
"signal_score": round(score, 2),
|
||||
"raw_metadata": json.dumps(raw_meta),
|
||||
"first_seen": now_str,
|
||||
"last_updated": now_str,
|
||||
})
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
parser = argparse.ArgumentParser(description="GitHub adapter for AI Research Oracle")
|
||||
parser.add_argument("--query", default="", help="Search query (empty = trending AI)")
|
||||
parser.add_argument("--limit", type=int, default=20, help="Max entries")
|
||||
parser.add_argument("--db", default=os.path.join(os.path.dirname(__file__), "..", "oracle.db"), help="SQLite DB")
|
||||
parser.add_argument("--schema", default=os.path.join(os.path.dirname(__file__), "..", "schema.sql"), help="Schema file")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Don't store in DB")
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"=== GitHub Adapter ===")
|
||||
print(f" Query: {args.query or '(trending AI)'}")
|
||||
print(f" Limit: {args.limit}")
|
||||
print()
|
||||
|
||||
adapter = GitHubAdapter()
|
||||
entries = adapter.fetch(query=args.query, limit=args.limit)
|
||||
|
||||
print(f" Fetched {len(entries)} entries")
|
||||
|
||||
if not args.dry_run:
|
||||
conn = sqlite3.connect(args.db)
|
||||
if os.path.exists(args.schema):
|
||||
with open(args.schema) as f:
|
||||
conn.executescript(f.read())
|
||||
conn.commit()
|
||||
|
||||
cur = conn.cursor()
|
||||
stored = 0
|
||||
for entry in entries:
|
||||
try:
|
||||
cur.execute("""
|
||||
INSERT OR REPLACE INTO entries
|
||||
(source, source_id, url, title, extracted_text, summary,
|
||||
category_tags, signal_score, raw_metadata, first_seen, last_updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
entry["source"], entry["source_id"], entry["url"], entry["title"],
|
||||
entry["extracted_text"], entry["summary"],
|
||||
entry["category_tags"], entry["signal_score"],
|
||||
entry["raw_metadata"], entry["first_seen"], entry["last_updated"],
|
||||
))
|
||||
stored += 1
|
||||
except Exception as e:
|
||||
print(f" DB error: {e}")
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f" Stored {stored} entries")
|
||||
|
||||
# Print top 5
|
||||
print(f"\n Top entries:")
|
||||
for i, e in enumerate(entries[:5]):
|
||||
meta = json.loads(e["raw_metadata"]) if isinstance(e["raw_metadata"], str) else e["raw_metadata"]
|
||||
print(f" [{i+1}] score={e['signal_score']:.2f} stars={meta.get('stars', '?')}")
|
||||
print(f" {e['title'][:90]}")
|
||||
print(f" {e['url']}")
|
||||
print(f" text={len(e.get('extracted_text', ''))}ch")
|
||||
|
||||
print(f"\n Done.")
|
||||
@@ -0,0 +1,535 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Reddit adapter v2 for AI Research Oracle.
|
||||
Uses Reddit RSS feeds (Atom XML) with proper filtering and scoring.
|
||||
|
||||
Improvements over v1 (proof of concept):
|
||||
- Filters AutoModerator posts and sticky threads
|
||||
- Proper User-Agent header
|
||||
- Attempts old.reddit.com .json for real upvote/comment data
|
||||
- Falls back to RSS with estimated scoring if JSON blocked
|
||||
- Rate limit awareness with retry logic
|
||||
|
||||
Rate limits: Reddit aggressively 429s unauthenticated requests.
|
||||
Strategy: 3s spacing between subreddits, retry with backoff.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
from html import unescape
|
||||
|
||||
from adapters import SourceAdapter
|
||||
|
||||
|
||||
class RedditAdapter(SourceAdapter):
|
||||
"""Reddit RSS + JSON adapter."""
|
||||
|
||||
# Default subreddits for AI content
|
||||
DEFAULT_SUBREDDITS = [
|
||||
"MachineLearning", "artificial", "LocalLLaMA", "Startups",
|
||||
]
|
||||
|
||||
# Posts to filter out (sticky threads, recurring mod posts)
|
||||
FILTER_TITLES = {
|
||||
"self-promotion thread",
|
||||
"monthly who's hiring",
|
||||
"weekly self-promotion",
|
||||
"discussion thread",
|
||||
"show and tell",
|
||||
"weekly show",
|
||||
}
|
||||
|
||||
FILTER_AUTHORS = {
|
||||
"automod",
|
||||
"automoderator",
|
||||
}
|
||||
|
||||
def __init__(self, subreddits=None, rate_limit=3, user_agent=None):
|
||||
"""
|
||||
Args:
|
||||
subreddits: List of subreddit names.
|
||||
rate_limit: Seconds between subreddit requests.
|
||||
user_agent: Custom User-Agent header.
|
||||
"""
|
||||
self.subreddits = subreddits or self.DEFAULT_SUBREDDITS
|
||||
self.rate_limit = rate_limit
|
||||
self.user_agent = user_agent or "python:ai-oracle:v0.1 (by tony_tech)"
|
||||
|
||||
def name(self) -> str:
|
||||
return "reddit"
|
||||
|
||||
def _clean_html(self, html: str) -> str:
|
||||
"""Extract readable text from Reddit's HTML content."""
|
||||
if not html:
|
||||
return ""
|
||||
text = re.sub(r"<!--.*?-->", "", html, flags=re.DOTALL)
|
||||
text = re.sub(r"<div[^>]*>", "\n", text)
|
||||
text = re.sub(r"</div>", "\n", text)
|
||||
text = re.sub(r"<br\s*/?>", "\n", text, flags=re.I)
|
||||
text = re.sub(r"<p[^>]*>", "\n", text)
|
||||
text = re.sub(r"</p>", "\n", text)
|
||||
text = re.sub(r"<a[^>]*href=\"([^\"]+)\"[^>]*>([^<]*)</a>", r"\2", text)
|
||||
text = re.sub(r"<[^>]+>", "", text)
|
||||
text = unescape(text)
|
||||
text = re.sub(r"\n\s*\n+", "\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
def _is_sticky(self, entry: dict) -> bool:
|
||||
"""Check if a post is a sticky/mod post that should be filtered."""
|
||||
title_lower = entry.get("title", "").lower().strip()
|
||||
author_lower = entry.get("author", "").lower().strip()
|
||||
|
||||
# Filter by author
|
||||
if author_lower in self.FILTER_AUTHORS:
|
||||
return True
|
||||
|
||||
# Filter by title patterns
|
||||
for pattern in self.FILTER_TITLES:
|
||||
if pattern in title_lower:
|
||||
return True
|
||||
|
||||
# Filter very short posts (likely link-only with no content)
|
||||
if len(entry.get("content", "")) < 20:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _fetch_rss(self, subreddit: str) -> list[dict]:
|
||||
"""Fetch RSS feed for a subreddit."""
|
||||
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
||||
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
xml_data = resp.read().decode("utf-8")
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 429:
|
||||
wait = 5 * (attempt + 1)
|
||||
time.sleep(wait)
|
||||
continue
|
||||
print(f" RSS HTTP {e.code} for r/{subreddit}")
|
||||
return []
|
||||
except Exception as e:
|
||||
print(f" RSS error r/{subreddit}: {e}")
|
||||
return []
|
||||
else:
|
||||
print(f" r/{subreddit}: still rate limited, skip")
|
||||
return []
|
||||
|
||||
# Parse Atom XML
|
||||
entries = []
|
||||
root = ET.fromstring(xml_data)
|
||||
|
||||
for entry_el in root.iter():
|
||||
tag = entry_el.tag.split("}")[-1] if "}" in entry_el.tag else entry_el.tag
|
||||
if tag != "entry":
|
||||
continue
|
||||
|
||||
data = {"title": "", "url": "", "author": "", "content": "", "published": "", "id": ""}
|
||||
|
||||
for child in entry_el:
|
||||
ctag = child.tag.split("}")[-1]
|
||||
if ctag == "title":
|
||||
data["title"] = unescape((child.text or "").strip())
|
||||
elif ctag == "link":
|
||||
data["url"] = child.get("href", "")
|
||||
elif ctag == "author":
|
||||
for ac in child:
|
||||
if ac.tag.split("}")[-1] == "name":
|
||||
data["author"] = unescape((ac.text or "").strip())
|
||||
elif ctag == "content":
|
||||
data["content"] = child.text or ""
|
||||
elif ctag == "published":
|
||||
data["published"] = child.text or ""
|
||||
elif ctag == "id":
|
||||
data["id"] = child.text or ""
|
||||
|
||||
if data["title"] and data["url"]:
|
||||
entries.append(data)
|
||||
|
||||
return entries
|
||||
|
||||
def _try_json(self, subreddit: str, sort: str = "hot") -> list[dict]:
|
||||
"""Try to fetch JSON data from old.reddit.com for real scores."""
|
||||
url = f"https://old.reddit.com/r/{subreddit}/{sort}.json?limit=25"
|
||||
req = urllib.request.Request(url, headers={
|
||||
"User-Agent": self.user_agent,
|
||||
"Accept": "application/json",
|
||||
})
|
||||
|
||||
for attempt in range(2):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
posts = []
|
||||
if isinstance(data, dict) and "data" in data:
|
||||
for child in data["data"].get("children", []):
|
||||
d = child.get("data", {})
|
||||
if d.get("title"):
|
||||
posts.append({
|
||||
"title": d.get("title", ""),
|
||||
"url": d.get("url", ""),
|
||||
"author": d.get("author", ""),
|
||||
"selftext": d.get("selftext", ""),
|
||||
"score": d.get("score", 0),
|
||||
"num_comments": d.get("num_comments", 0),
|
||||
"ups": d.get("ups", 0),
|
||||
"id": d.get("id", ""),
|
||||
"created_utc": d.get("created_utc", 0),
|
||||
"permalink": d.get("permalink", ""),
|
||||
"link_flair_text": d.get("link_flair_text", ""),
|
||||
"subreddit": subreddit,
|
||||
"source": "json",
|
||||
})
|
||||
return posts
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (429, 403, 404):
|
||||
return [] # JSON endpoint blocked, fall back to RSS
|
||||
if attempt < 1:
|
||||
time.sleep(5)
|
||||
continue
|
||||
return []
|
||||
except Exception:
|
||||
return []
|
||||
return []
|
||||
|
||||
def _score(self, entry: dict, source: str) -> float:
|
||||
"""Score based on available signals.
|
||||
|
||||
If source is 'json', use real upvote/comment counts.
|
||||
If source is 'rss', use heuristics (content length, flair, etc.).
|
||||
"""
|
||||
if source == "json" and entry.get("score", 0) > 0:
|
||||
import math
|
||||
# Real upvote-based scoring
|
||||
score = entry.get("score", 0)
|
||||
comments = entry.get("num_comments", 0)
|
||||
|
||||
# Log scale on upvotes, bonus for engagement
|
||||
star_score = min(math.log1p(score) / 2.0, 8.0)
|
||||
engagement = min(comments / 50.0, 2.0)
|
||||
|
||||
return min(round(star_score + engagement, 2), 10.0)
|
||||
else:
|
||||
# RSS heuristic — no real scores available
|
||||
content = entry.get("content", "")
|
||||
title = entry.get("title", "")
|
||||
|
||||
# Base score
|
||||
base = 3.0
|
||||
|
||||
# Content length bonus (substantive posts get higher scores)
|
||||
content_len = len(self._clean_html(content))
|
||||
if content_len > 5000:
|
||||
base += 2.0
|
||||
elif content_len > 2000:
|
||||
base += 1.5
|
||||
elif content_len > 1000:
|
||||
base += 1.0
|
||||
elif content_len > 500:
|
||||
base += 0.6
|
||||
elif content_len > 200:
|
||||
base += 0.3
|
||||
elif content_len > 50:
|
||||
base += 0.1
|
||||
|
||||
# Post type from title markers
|
||||
if " [R]" in title or " [r]" in title: # Research
|
||||
base += 1.5
|
||||
elif " [P]" in title or " [p]" in title: # Project
|
||||
base += 1.2
|
||||
elif " [N]" in title or " [n]" in title: # News
|
||||
base += 0.8
|
||||
elif " [D]" in title or " [d]" in title: # Discussion
|
||||
base += 0.5
|
||||
|
||||
# Flair bonus
|
||||
flair = entry.get("link_flair_text", "").lower()
|
||||
if "research" in flair:
|
||||
base += 1.0
|
||||
elif "project" in flair:
|
||||
base += 0.8
|
||||
|
||||
return min(round(base, 2), 10.0)
|
||||
|
||||
def _tags(self, entry: dict, source: str) -> list:
|
||||
"""Generate category tags from Reddit metadata."""
|
||||
tags = ["reddit"]
|
||||
|
||||
subreddit = entry.get("subreddit", "").lower()
|
||||
if "machinelearning" in subreddit:
|
||||
tags.append("machine-learning")
|
||||
elif "artificial" in subreddit:
|
||||
tags.append("ai-general")
|
||||
elif "localllama" in subreddit:
|
||||
tags.append("local-llm")
|
||||
elif "startups" in subreddit:
|
||||
tags.append("startups")
|
||||
|
||||
title = entry.get("title", "")
|
||||
if " [P]" in title or " [p]" in title:
|
||||
tags.append("project")
|
||||
elif " [R]" in title or " [r]" in title:
|
||||
tags.append("research")
|
||||
elif " [D]" in title or " [d]" in title:
|
||||
tags.append("discussion")
|
||||
elif " [N]" in title or " [n]" in title:
|
||||
tags.append("news")
|
||||
else:
|
||||
tags.append("general")
|
||||
|
||||
# Check for self-referential virality signal
|
||||
content = entry.get("content", "").lower()
|
||||
title_lower = title.lower()
|
||||
if any(kw in title_lower or kw in content for kw in [
|
||||
"stars", "viral", "trending", "blow up", "trending github",
|
||||
]):
|
||||
tags.append("meta:virality")
|
||||
|
||||
return tags
|
||||
|
||||
def _fetch_rss(self, subreddit: str) -> list[dict]:
|
||||
"""Fetch RSS feed for a subreddit."""
|
||||
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
||||
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
xml_data = resp.read().decode("utf-8")
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 429:
|
||||
wait = 5 * (attempt + 1)
|
||||
time.sleep(wait)
|
||||
continue
|
||||
print(f" RSS HTTP {e.code} for r/{subreddit}")
|
||||
return []
|
||||
except Exception as e:
|
||||
print(f" RSS error r/{subreddit}: {e}")
|
||||
return []
|
||||
else:
|
||||
print(f" r/{subreddit}: still rate limited, skip")
|
||||
return []
|
||||
|
||||
# Parse Atom XML
|
||||
entries = []
|
||||
root = ET.fromstring(xml_data)
|
||||
|
||||
for entry_el in root.iter():
|
||||
tag = entry_el.tag.split("}")[-1] if "}" in entry_el.tag else entry_el.tag
|
||||
if tag != "entry":
|
||||
continue
|
||||
|
||||
data = {"title": "", "url": "", "author": "", "content": "", "published": "", "id": ""}
|
||||
|
||||
for child in entry_el:
|
||||
ctag = child.tag.split("}")[-1]
|
||||
if ctag == "title":
|
||||
data["title"] = unescape((child.text or "").strip())
|
||||
elif ctag == "link":
|
||||
data["url"] = child.get("href", "")
|
||||
elif ctag == "author":
|
||||
for ac in child:
|
||||
if ac.tag.split("}")[-1] == "name":
|
||||
data["author"] = unescape((ac.text or "").strip())
|
||||
elif ctag == "content":
|
||||
data["content"] = child.text or ""
|
||||
elif ctag == "published":
|
||||
data["published"] = child.text or ""
|
||||
elif ctag == "id":
|
||||
data["id"] = child.text or ""
|
||||
|
||||
if data["title"] and data["url"]:
|
||||
entries.append(data)
|
||||
|
||||
return entries
|
||||
|
||||
def fetch(self, query: str = "", limit: int = 20) -> list[dict]:
|
||||
"""
|
||||
Fetch Reddit posts.
|
||||
|
||||
Tries JSON first (real scores), falls back to RSS (heuristic scoring).
|
||||
Filters AutoModerator and sticky posts.
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
all_entries = []
|
||||
seen_ids = set()
|
||||
json_worked = False
|
||||
|
||||
# Try JSON first
|
||||
for sub in self.subreddits[:2]: # Try JSON on first 2 subreddits
|
||||
posts = self._try_json(sub)
|
||||
if posts:
|
||||
json_worked = True
|
||||
for p in posts:
|
||||
if p["id"] not in seen_ids:
|
||||
seen_ids.add(p["id"])
|
||||
all_entries.append(p)
|
||||
time.sleep(self.rate_limit)
|
||||
|
||||
# If JSON didn't work, fall back to RSS for all subreddits
|
||||
if not json_worked:
|
||||
print(" JSON endpoints blocked, using RSS fallback")
|
||||
for sub in self.subreddits:
|
||||
entries = self._fetch_rss(sub)
|
||||
for e in entries:
|
||||
# Convert RSS format to unified format
|
||||
entry = {
|
||||
"title": e["title"],
|
||||
"url": e["url"],
|
||||
"author": e["author"],
|
||||
"content": e["content"],
|
||||
"published": e["published"],
|
||||
"id": e["id"].replace("t3_", ""),
|
||||
"score": 0,
|
||||
"num_comments": 0,
|
||||
"ups": 0,
|
||||
"selftext": "",
|
||||
"created_utc": 0,
|
||||
"permalink": "",
|
||||
"link_flair_text": "",
|
||||
"subreddit": sub,
|
||||
"source": "rss",
|
||||
}
|
||||
if entry["id"] not in seen_ids:
|
||||
seen_ids.add(entry["id"])
|
||||
all_entries.append(entry)
|
||||
time.sleep(self.rate_limit)
|
||||
|
||||
# Filter sticky/mod posts
|
||||
filtered = []
|
||||
for entry in all_entries:
|
||||
if not self._is_sticky(entry):
|
||||
filtered.append(entry)
|
||||
|
||||
print(f" Filtered: {len(all_entries)} → {len(filtered)} (removed {len(all_entries) - len(filtered)} sticky/mod)")
|
||||
|
||||
# Sort by score, take top limit
|
||||
for entry in filtered:
|
||||
entry["_score"] = self._score(entry, entry.get("source", "rss"))
|
||||
filtered.sort(key=lambda e: e.get("_score", 0), reverse=True)
|
||||
filtered = filtered[:limit]
|
||||
|
||||
# Convert to DB format
|
||||
entries = []
|
||||
for entry in filtered:
|
||||
source_id = entry.get("id", "") or entry.get("url", "").split("/")[-1]
|
||||
score = entry.pop("_score", 0)
|
||||
|
||||
# Clean title
|
||||
title = entry.get("title", "")
|
||||
|
||||
# Extracted text: selftext (JSON) or content (RSS)
|
||||
if entry.get("source") == "json":
|
||||
extracted_text = entry.get("selftext", "")
|
||||
else:
|
||||
extracted_text = self._clean_html(entry.get("content", ""))
|
||||
|
||||
tags = self._tags(entry, entry.get("source", "rss"))
|
||||
|
||||
# Structured metadata
|
||||
raw_meta = {
|
||||
"subreddit": entry.get("subreddit", ""),
|
||||
"author": entry.get("author", ""),
|
||||
"published": entry.get("published", ""),
|
||||
"score": entry.get("score", 0),
|
||||
"num_comments": entry.get("num_comments", 0),
|
||||
"ups": entry.get("ups", 0),
|
||||
"link_flair_text": entry.get("link_flair_text", ""),
|
||||
"source_type": entry.get("source", "rss"), # json or rss
|
||||
"content_length": len(extracted_text),
|
||||
"score_type": "actual" if entry.get("source") == "json" and entry.get("score", 0) > 0 else "estimated",
|
||||
}
|
||||
|
||||
now_str = now.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
entries.append({
|
||||
"source": "reddit",
|
||||
"source_id": source_id,
|
||||
"url": entry.get("url", ""),
|
||||
"title": title,
|
||||
"extracted_text": extracted_text,
|
||||
"summary": None,
|
||||
"category_tags": json.dumps(tags),
|
||||
"signal_score": score,
|
||||
"raw_metadata": json.dumps(raw_meta),
|
||||
"first_seen": now_str,
|
||||
"last_updated": now_str,
|
||||
})
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
import sqlite3
|
||||
|
||||
parser = argparse.ArgumentParser(description="Reddit adapter v2 for AI Research Oracle")
|
||||
parser.add_argument("--limit", type=int, default=20, help="Max entries")
|
||||
parser.add_argument("--db", default=os.path.join(os.path.dirname(__file__), "..", "oracle.db"), help="SQLite DB")
|
||||
parser.add_argument("--schema", default=os.path.join(os.path.dirname(__file__), "..", "schema.sql"), help="Schema file")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Don't store in DB")
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"=== Reddit Adapter v2 ===")
|
||||
print(f" Limit: {args.limit}")
|
||||
print()
|
||||
|
||||
adapter = RedditAdapter()
|
||||
entries = adapter.fetch(limit=args.limit)
|
||||
|
||||
print(f"\n Fetched {len(entries)} entries")
|
||||
|
||||
if not args.dry_run:
|
||||
conn = sqlite3.connect(args.db)
|
||||
if os.path.exists(args.schema):
|
||||
with open(args.schema) as f:
|
||||
conn.executescript(f.read())
|
||||
conn.commit()
|
||||
|
||||
cur = conn.cursor()
|
||||
stored = 0
|
||||
for entry in entries:
|
||||
try:
|
||||
cur.execute("""
|
||||
INSERT OR REPLACE INTO entries
|
||||
(source, source_id, url, title, extracted_text, summary,
|
||||
category_tags, signal_score, raw_metadata, first_seen, last_updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
entry["source"], entry["source_id"], entry["url"], entry["title"],
|
||||
entry["extracted_text"], entry["summary"],
|
||||
entry["category_tags"], entry["signal_score"],
|
||||
entry["raw_metadata"], entry["first_seen"], entry["last_updated"],
|
||||
))
|
||||
stored += 1
|
||||
except Exception as e:
|
||||
print(f" DB error: {e}")
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f" Stored {stored} entries")
|
||||
|
||||
# Print top 5
|
||||
print(f"\n Top entries:")
|
||||
for i, e in enumerate(entries[:5]):
|
||||
meta = json.loads(e["raw_metadata"]) if isinstance(e["raw_metadata"], str) else e["raw_metadata"]
|
||||
src = meta.get("source_type", "?")
|
||||
ups = meta.get("score", "?")
|
||||
comments = meta.get("num_comments", "?")
|
||||
score_type = meta.get("score_type", "?")
|
||||
print(f" [{i+1}] score={e['signal_score']:.2f} ({score_type}) ups={ups} comments={comments} src={src}")
|
||||
print(f" {e['title'][:90]}")
|
||||
print(f" r/{meta.get('subreddit', '?')} by {meta.get('author', '?')}")
|
||||
print(f" text={meta.get('content_length', 0)}ch")
|
||||
|
||||
print(f"\n Done.")
|
||||
Reference in New Issue
Block a user