Sprint 0+1: Package restructure, source tiers, verdicts, multi-variant editions
- New oracle/ package (11 modules) with unified CLI (python -m oracle) - Source tiers: Tier 1 (arxiv/github/hf), Tier 2 (rss/hn), Tier 3 (reddit) - Composite verdicts: PUBLISH/WATCH/ARCHIVE/DROP based on signal score + age - Content-hash dedup: SHA-256[:16] normalized, atomic at insert time - Multi-variant editions: 4 YAML configs (default/research/devops/brief) - Variant engine: filter → rank → render (HTML + JSON, themed) - Per-adapter timeout (10s) + threading fallback - Consolidated 12 root scripts → thin wrappers + oracle/ package - Archived stale scripts (_engagement, _live_compare, reddit_proof) - Updated .gitignore, README.md, schema.sql
This commit is contained in:
Epictetus
+43
-23
@@ -15,7 +15,9 @@ Strategy: 3s spacing between subreddits, retry with backoff.
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
import urllib.request
|
||||
@@ -51,11 +53,11 @@ class RedditAdapter(SourceAdapter):
|
||||
"automoderator",
|
||||
}
|
||||
|
||||
def __init__(self, subreddits=None, rate_limit=1, user_agent=None):
|
||||
def __init__(self, subreddits=None, rate_limit=4, user_agent=None):
|
||||
"""
|
||||
Args:
|
||||
subreddits: List of subreddit names.
|
||||
rate_limit: Seconds between subreddit requests.
|
||||
rate_limit: Base seconds between subreddit requests (with jitter).
|
||||
user_agent: Custom User-Agent header.
|
||||
"""
|
||||
self.subreddits = subreddits or self.DEFAULT_SUBREDDITS
|
||||
@@ -65,6 +67,12 @@ class RedditAdapter(SourceAdapter):
|
||||
def name(self) -> str:
|
||||
return "reddit"
|
||||
|
||||
def _sleep_with_jitter(self, base=None):
|
||||
"""Sleep with ±30% jitter to avoid pattern detection."""
|
||||
base = base or self.rate_limit
|
||||
jitter = base * 0.3 * (2 * random.random() - 1) # ±30%
|
||||
time.sleep(base + jitter)
|
||||
|
||||
def _clean_html(self, html: str) -> str:
|
||||
"""Extract readable text from Reddit's HTML content."""
|
||||
if not html:
|
||||
@@ -106,7 +114,8 @@ class RedditAdapter(SourceAdapter):
|
||||
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
||||
|
||||
for attempt in range(2): # max 2 attempts, fail fast
|
||||
backoff = [3, 8] # staggered backoff: 3s then 8s
|
||||
for attempt in range(3): # max 3 attempts
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
xml_data = resp.read().decode("utf-8")
|
||||
@@ -116,8 +125,10 @@ class RedditAdapter(SourceAdapter):
|
||||
print(f" RSS blocked (HTTP {e.code}) for r/{subreddit}")
|
||||
return []
|
||||
if e.code == 429:
|
||||
if attempt == 0:
|
||||
time.sleep(2) # single retry with short backoff
|
||||
if attempt < len(backoff):
|
||||
delay = backoff[attempt]
|
||||
print(f" RSS 429 for r/{subreddit}, retrying in {delay}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
print(f" RSS rate-limited for r/{subreddit}, skip")
|
||||
return []
|
||||
@@ -127,7 +138,7 @@ class RedditAdapter(SourceAdapter):
|
||||
print(f" RSS error r/{subreddit}: {e}")
|
||||
return []
|
||||
else:
|
||||
print(f" r/{subreddit}: still rate limited, skip")
|
||||
print(f" r/{subreddit}: still rate limited after 3 attempts, skip")
|
||||
return []
|
||||
|
||||
# Parse Atom XML
|
||||
@@ -301,6 +312,17 @@ class RedditAdapter(SourceAdapter):
|
||||
]):
|
||||
tags.append("meta:virality")
|
||||
|
||||
# Local-inference / on-device signal (suggested source: r/MachineLearning
|
||||
# "I tried X on-device" posts — high builder signal → local-inference feed)
|
||||
on_device_kw = [
|
||||
"on-device", "on device", "local inference", "local llm",
|
||||
"ran locally", "running locally", "in my pocket", "on my phone",
|
||||
"edge device", "offline", "no gpu", "consumer gpu", "rtx",
|
||||
"single gpu", "self-host", "self host",
|
||||
]
|
||||
if any(kw in title_lower or kw in content for kw in on_device_kw):
|
||||
tags.append("local-inference")
|
||||
|
||||
return tags
|
||||
|
||||
def _fetch_rss(self, subreddit: str) -> list[dict]:
|
||||
@@ -308,7 +330,8 @@ class RedditAdapter(SourceAdapter):
|
||||
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
||||
|
||||
for attempt in range(2): # max 2 attempts, fail fast
|
||||
backoff = [3, 8] # staggered backoff: 3s then 8s
|
||||
for attempt in range(3): # max 3 attempts
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
xml_data = resp.read().decode("utf-8")
|
||||
@@ -318,8 +341,10 @@ class RedditAdapter(SourceAdapter):
|
||||
print(f" RSS blocked (HTTP {e.code}) for r/{subreddit}")
|
||||
return []
|
||||
if e.code == 429:
|
||||
if attempt == 0:
|
||||
time.sleep(2) # single retry with short backoff
|
||||
if attempt < len(backoff):
|
||||
delay = backoff[attempt]
|
||||
print(f" RSS 429 for r/{subreddit}, retrying in {delay}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
print(f" RSS rate-limited for r/{subreddit}, skip")
|
||||
return []
|
||||
@@ -329,7 +354,7 @@ class RedditAdapter(SourceAdapter):
|
||||
print(f" RSS error r/{subreddit}: {e}")
|
||||
return []
|
||||
else:
|
||||
print(f" r/{subreddit}: still rate limited, skip")
|
||||
print(f" r/{subreddit}: still rate limited after 3 attempts, skip")
|
||||
return []
|
||||
|
||||
# Parse Atom XML
|
||||
@@ -373,31 +398,26 @@ class RedditAdapter(SourceAdapter):
|
||||
Filters AutoModerator and sticky posts.
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
# Try JSON first — if it's blocked on the first subreddit, bail fast
|
||||
# rather than wasting time on all subreddits
|
||||
# Try JSON first — if it's blocked on the first subreddit, skip
|
||||
# the test-RSS call (which would waste a request and risk rate-limiting)
|
||||
# and go straight to the RSS loop
|
||||
first_json = self._try_json(self.subreddits[0])
|
||||
if not first_json:
|
||||
# JSON is blocked site-wide, try one RSS to confirm
|
||||
test_rss = self._fetch_rss(self.subreddits[0])
|
||||
if not test_rss:
|
||||
print(" Reddit blocked (403/429), returning empty")
|
||||
return []
|
||||
# RSS works — fall through to full fetch below
|
||||
json_worked = bool(first_json)
|
||||
|
||||
all_entries = []
|
||||
seen_ids = set()
|
||||
json_worked = bool(first_json)
|
||||
|
||||
# Add first JSON results
|
||||
# Add first JSON results if any
|
||||
for p in first_json:
|
||||
if p["id"] not in seen_ids:
|
||||
seen_ids.add(p["id"])
|
||||
all_entries.append(p)
|
||||
time.sleep(self.rate_limit)
|
||||
|
||||
# If JSON didn't work, fall back to RSS for all subreddits
|
||||
if not json_worked:
|
||||
print(" JSON endpoints blocked, using RSS fallback")
|
||||
# Brief cooldown before RSS barrage
|
||||
time.sleep(3)
|
||||
for sub in self.subreddits:
|
||||
entries = self._fetch_rss(sub)
|
||||
for e in entries:
|
||||
@@ -422,7 +442,7 @@ class RedditAdapter(SourceAdapter):
|
||||
if entry["id"] not in seen_ids:
|
||||
seen_ids.add(entry["id"])
|
||||
all_entries.append(entry)
|
||||
time.sleep(self.rate_limit)
|
||||
self._sleep_with_jitter()
|
||||
|
||||
# Filter sticky/mod posts
|
||||
filtered = []
|
||||
|
||||
Reference in New Issue
Block a user