Tony_tech/athena-oracle
3
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Sprint 0+1: Package restructure, source tiers, verdicts, multi-variant editions

Browse Source 
- New oracle/ package (11 modules) with unified CLI (python -m oracle)
- Source tiers: Tier 1 (arxiv/github/hf), Tier 2 (rss/hn), Tier 3 (reddit)
- Composite verdicts: PUBLISH/WATCH/ARCHIVE/DROP based on signal score + age
- Content-hash dedup: SHA-256[:16] normalized, atomic at insert time
- Multi-variant editions: 4 YAML configs (default/research/devops/brief)
- Variant engine: filter → rank → render (HTML + JSON, themed)
- Per-adapter timeout (10s) + threading fallback
- Consolidated 12 root scripts → thin wrappers + oracle/ package
- Archived stale scripts (_engagement, _live_compare, reddit_proof)
- Updated .gitignore, README.md, schema.sql
This commit is contained in:
Epictetus
2026-07-22 13:32:15 +00:00
parent 9f72ff4d6a
commit 07c5f9a5c2
38 changed files with 3195 additions and 3234 deletions
Download Patch File Download Diff File Expand all files Collapse all files
adapters/_store.py
+28 -2
View File
@@ -89,8 +89,9 @@ def true_first_seen(raw_meta, source, now_str):
UPSERT_SQL = """
INSERT INTO entries
(source, source_id, url, title, extracted_text, summary,
category_tags, signal_score, raw_metadata, first_seen, last_updated)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
category_tags, signal_score, raw_metadata, first_seen, last_updated,
content_hash, source_tier, verdict)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(source, source_id) DO UPDATE SET
source = excluded.source,
source_id = excluded.source_id,
@@ -101,6 +102,9 @@ ON CONFLICT(source, source_id) DO UPDATE SET
signal_score = excluded.signal_score,
raw_metadata = excluded.raw_metadata,
last_updated = excluded.last_updated,
content_hash = excluded.content_hash,
source_tier = excluded.source_tier,
verdict = excluded.verdict,
first_seen = COALESCE((SELECT first_seen FROM entries WHERE source = excluded.source AND source_id = excluded.source_id), excluded.first_seen)
"""
@@ -111,16 +115,38 @@ def upsert_entries(conn, entries):
`entries` is the list of dicts as built by each adapter; each dict must
already have first_seen set to the TRUE publish date (via true_first_seen)
and last_updated to the harvest time.
Now also sets content_hash and source_tier at insertion time.
Returns count of rows written.
"""
from oracle.dedup import content_hash, get_source_tier, compute_verdict, age_hours
cur = conn.cursor()
written = 0
for e in entries:
# Compute content hash
title = e.get("title", "")
url = e.get("url", "")
body = e.get("extracted_text", "")[:500]
h = content_hash(title, url, body)
# Get source tier
source = e.get("source", "")
tier_info = get_source_tier(source)
tier = tier_info["tier"]
# Compute verdict
first_seen = e.get("first_seen", "")
score = float(e.get("signal_score") or 0)
age = age_hours(first_seen)
verdict = compute_verdict(score, age)
cur.execute(UPSERT_SQL, (
e["source"], e["source_id"], e["url"], e["title"],
e.get("extracted_text"), e.get("summary"),
e.get("category_tags"), e.get("signal_score"),
e.get("raw_metadata"), e["first_seen"], e["last_updated"],
h, tier, verdict,
))
written += 1
conn.commit()
adapters/arxiv.py
+12
View File
@@ -355,6 +355,18 @@ class ArxivAdapter(SourceAdapter):
if paper.get("_applied_domain"):
tags.append(paper["_applied_domain"])
# Local-serving / efficient-inference signal (suggested source: arXiv cs.LG
# MoE/quantization papers → "local-serving" tag filter)
serving_kw = [
"quantiz", "quantization", "moe", "mixture of experts",
"serving", "efficient inference", "pruning", "distill",
"knowledge distillation", "low-rank", "lora", "parameter-efficient",
"vram", "memory efficient", "edge inference", "on-device",
]
combined = f"{title_lower} {summary_lower}"
if any(kw in combined for kw in serving_kw):
tags.append("local-serving")
return tags
def fetch(self, query: str = "", limit: int = 20) -> list[dict]:
adapters/reddit.py
+43 -23
View File
@@ -15,7 +15,9 @@ Strategy: 3s spacing between subreddits, retry with backoff.
"""
import json
import math
import os
import random
import re
import time
import urllib.request
@@ -51,11 +53,11 @@ class RedditAdapter(SourceAdapter):
"automoderator",
}
def __init__(self, subreddits=None, rate_limit=1, user_agent=None):
def __init__(self, subreddits=None, rate_limit=4, user_agent=None):
"""
Args:
subreddits: List of subreddit names.
rate_limit: Seconds between subreddit requests.
rate_limit: Base seconds between subreddit requests (with jitter).
user_agent: Custom User-Agent header.
"""
self.subreddits = subreddits or self.DEFAULT_SUBREDDITS
@@ -65,6 +67,12 @@ class RedditAdapter(SourceAdapter):
def name(self) -> str:
return "reddit"
def _sleep_with_jitter(self, base=None):
"""Sleep with ±30% jitter to avoid pattern detection."""
base = base or self.rate_limit
jitter = base * 0.3 * (2 * random.random() - 1) # ±30%
time.sleep(base + jitter)
def _clean_html(self, html: str) -> str:
"""Extract readable text from Reddit's HTML content."""
if not html:
@@ -106,7 +114,8 @@ class RedditAdapter(SourceAdapter):
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
for attempt in range(2): # max 2 attempts, fail fast
backoff = [3, 8] # staggered backoff: 3s then 8s
for attempt in range(3): # max 3 attempts
try:
with urllib.request.urlopen(req, timeout=10) as resp:
xml_data = resp.read().decode("utf-8")
@@ -116,8 +125,10 @@ class RedditAdapter(SourceAdapter):
print(f" RSS blocked (HTTP {e.code}) for r/{subreddit}")
return []
if e.code == 429:
if attempt == 0:
time.sleep(2) # single retry with short backoff
if attempt < len(backoff):
delay = backoff[attempt]
print(f" RSS 429 for r/{subreddit}, retrying in {delay}s")
time.sleep(delay)
continue
print(f" RSS rate-limited for r/{subreddit}, skip")
return []
@@ -127,7 +138,7 @@ class RedditAdapter(SourceAdapter):
print(f" RSS error r/{subreddit}: {e}")
return []
else:
print(f" r/{subreddit}: still rate limited, skip")
print(f" r/{subreddit}: still rate limited after 3 attempts, skip")
return []
# Parse Atom XML
@@ -301,6 +312,17 @@ class RedditAdapter(SourceAdapter):
]):
tags.append("meta:virality")
# Local-inference / on-device signal (suggested source: r/MachineLearning
# "I tried X on-device" posts — high builder signal → local-inference feed)
on_device_kw = [
"on-device", "on device", "local inference", "local llm",
"ran locally", "running locally", "in my pocket", "on my phone",
"edge device", "offline", "no gpu", "consumer gpu", "rtx",
"single gpu", "self-host", "self host",
]
if any(kw in title_lower or kw in content for kw in on_device_kw):
tags.append("local-inference")
return tags
def _fetch_rss(self, subreddit: str) -> list[dict]:
@@ -308,7 +330,8 @@ class RedditAdapter(SourceAdapter):
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
for attempt in range(2): # max 2 attempts, fail fast
backoff = [3, 8] # staggered backoff: 3s then 8s
for attempt in range(3): # max 3 attempts
try:
with urllib.request.urlopen(req, timeout=10) as resp:
xml_data = resp.read().decode("utf-8")
@@ -318,8 +341,10 @@ class RedditAdapter(SourceAdapter):
print(f" RSS blocked (HTTP {e.code}) for r/{subreddit}")
return []
if e.code == 429:
if attempt == 0:
time.sleep(2) # single retry with short backoff
if attempt < len(backoff):
delay = backoff[attempt]
print(f" RSS 429 for r/{subreddit}, retrying in {delay}s")
time.sleep(delay)
continue
print(f" RSS rate-limited for r/{subreddit}, skip")
return []
@@ -329,7 +354,7 @@ class RedditAdapter(SourceAdapter):
print(f" RSS error r/{subreddit}: {e}")
return []
else:
print(f" r/{subreddit}: still rate limited, skip")
print(f" r/{subreddit}: still rate limited after 3 attempts, skip")
return []
# Parse Atom XML
@@ -373,31 +398,26 @@ class RedditAdapter(SourceAdapter):
Filters AutoModerator and sticky posts.
"""
now = datetime.now(timezone.utc)
# Try JSON first — if it's blocked on the first subreddit, bail fast
# rather than wasting time on all subreddits
# Try JSON first — if it's blocked on the first subreddit, skip
# the test-RSS call (which would waste a request and risk rate-limiting)
# and go straight to the RSS loop
first_json = self._try_json(self.subreddits[0])
if not first_json:
# JSON is blocked site-wide, try one RSS to confirm
test_rss = self._fetch_rss(self.subreddits[0])
if not test_rss:
print(" Reddit blocked (403/429), returning empty")
return []
# RSS works — fall through to full fetch below
json_worked = bool(first_json)
all_entries = []
seen_ids = set()
json_worked = bool(first_json)
# Add first JSON results
# Add first JSON results if any
for p in first_json:
if p["id"] not in seen_ids:
seen_ids.add(p["id"])
all_entries.append(p)
time.sleep(self.rate_limit)
# If JSON didn't work, fall back to RSS for all subreddits
if not json_worked:
print(" JSON endpoints blocked, using RSS fallback")
# Brief cooldown before RSS barrage
time.sleep(3)
for sub in self.subreddits:
entries = self._fetch_rss(sub)
for e in entries:
@@ -422,7 +442,7 @@ class RedditAdapter(SourceAdapter):
if entry["id"] not in seen_ids:
seen_ids.add(entry["id"])
all_entries.append(entry)
time.sleep(self.rate_limit)
self._sleep_with_jitter()
# Filter sticky/mod posts
filtered = []