Fix Reddit adapter: fast-bail when blocked (403/429)

Bug: Reddit was taking 120s+ when rate-limited (3 attempts × 4 subreddits × backoff).
Fix: 1) Try JSON on first subreddit; if blocked, test one RSS.
     2) If both fail, return empty immediately (<3s).
     3) Reduce default rate_limit 3→1s.
     4) 403 now immediate fail in RSS (was retrying).
     5) Max RSS retries 3→2, single 2s backoff.

Result: 120s timeout → 2.1s when blocked. Zero entries returned but no hang.
This commit is contained in:
Epictetus
2026-07-08 15:36:50 +00:00
parent 2c6701f5a3
commit df5226859e
+40 -23
View File
@@ -50,7 +50,7 @@ class RedditAdapter(SourceAdapter):
"automoderator",
}
def __init__(self, subreddits=None, rate_limit=3, user_agent=None):
def __init__(self, subreddits=None, rate_limit=1, user_agent=None):
"""
Args:
subreddits: List of subreddit names.
@@ -105,16 +105,21 @@ class RedditAdapter(SourceAdapter):
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
for attempt in range(3):
for attempt in range(2): # max 2 attempts, fail fast
try:
with urllib.request.urlopen(req, timeout=15) as resp:
with urllib.request.urlopen(req, timeout=10) as resp:
xml_data = resp.read().decode("utf-8")
break
except urllib.error.HTTPError as e:
if e.code in (403,):
print(f" RSS blocked (HTTP {e.code}) for r/{subreddit}")
return []
if e.code == 429:
wait = 5 * (attempt + 1)
time.sleep(wait)
continue
if attempt == 0:
time.sleep(2) # single retry with short backoff
continue
print(f" RSS rate-limited for r/{subreddit}, skip")
return []
print(f" RSS HTTP {e.code} for r/{subreddit}")
return []
except Exception as e:
@@ -167,7 +172,7 @@ class RedditAdapter(SourceAdapter):
for attempt in range(2):
try:
with urllib.request.urlopen(req, timeout=15) as resp:
with urllib.request.urlopen(req, timeout=10) as resp:
data = json.loads(resp.read().decode("utf-8"))
posts = []
@@ -302,16 +307,21 @@ class RedditAdapter(SourceAdapter):
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
for attempt in range(3):
for attempt in range(2): # max 2 attempts, fail fast
try:
with urllib.request.urlopen(req, timeout=15) as resp:
with urllib.request.urlopen(req, timeout=10) as resp:
xml_data = resp.read().decode("utf-8")
break
except urllib.error.HTTPError as e:
if e.code in (403,):
print(f" RSS blocked (HTTP {e.code}) for r/{subreddit}")
return []
if e.code == 429:
wait = 5 * (attempt + 1)
time.sleep(wait)
continue
if attempt == 0:
time.sleep(2) # single retry with short backoff
continue
print(f" RSS rate-limited for r/{subreddit}, skip")
return []
print(f" RSS HTTP {e.code} for r/{subreddit}")
return []
except Exception as e:
@@ -362,20 +372,27 @@ class RedditAdapter(SourceAdapter):
Filters AutoModerator and sticky posts.
"""
now = datetime.now(timezone.utc)
# Try JSON first — if it's blocked on the first subreddit, bail fast
# rather than wasting time on all subreddits
first_json = self._try_json(self.subreddits[0])
if not first_json:
# JSON is blocked site-wide, try one RSS to confirm
test_rss = self._fetch_rss(self.subreddits[0])
if not test_rss:
print(" Reddit blocked (403/429), returning empty")
return []
# RSS works — fall through to full fetch below
all_entries = []
seen_ids = set()
json_worked = False
json_worked = bool(first_json)
# Try JSON first
for sub in self.subreddits[:2]: # Try JSON on first 2 subreddits
posts = self._try_json(sub)
if posts:
json_worked = True
for p in posts:
if p["id"] not in seen_ids:
seen_ids.add(p["id"])
all_entries.append(p)
time.sleep(self.rate_limit)
# Add first JSON results
for p in first_json:
if p["id"] not in seen_ids:
seen_ids.add(p["id"])
all_entries.append(p)
time.sleep(self.rate_limit)
# If JSON didn't work, fall back to RSS for all subreddits
if not json_worked: