Fix Reddit adapter: fast-bail when blocked (403/429)
Bug: Reddit was taking 120s+ when rate-limited (3 attempts × 4 subreddits × backoff).
Fix: 1) Try JSON on first subreddit; if blocked, test one RSS.
2) If both fail, return empty immediately (<3s).
3) Reduce default rate_limit 3→1s.
4) 403 now immediate fail in RSS (was retrying).
5) Max RSS retries 3→2, single 2s backoff.
Result: 120s timeout → 2.1s when blocked. Zero entries returned but no hang.
This commit is contained in:
+40
-23
@@ -50,7 +50,7 @@ class RedditAdapter(SourceAdapter):
|
||||
"automoderator",
|
||||
}
|
||||
|
||||
def __init__(self, subreddits=None, rate_limit=3, user_agent=None):
|
||||
def __init__(self, subreddits=None, rate_limit=1, user_agent=None):
|
||||
"""
|
||||
Args:
|
||||
subreddits: List of subreddit names.
|
||||
@@ -105,16 +105,21 @@ class RedditAdapter(SourceAdapter):
|
||||
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
||||
|
||||
for attempt in range(3):
|
||||
for attempt in range(2): # max 2 attempts, fail fast
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
xml_data = resp.read().decode("utf-8")
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (403,):
|
||||
print(f" RSS blocked (HTTP {e.code}) for r/{subreddit}")
|
||||
return []
|
||||
if e.code == 429:
|
||||
wait = 5 * (attempt + 1)
|
||||
time.sleep(wait)
|
||||
continue
|
||||
if attempt == 0:
|
||||
time.sleep(2) # single retry with short backoff
|
||||
continue
|
||||
print(f" RSS rate-limited for r/{subreddit}, skip")
|
||||
return []
|
||||
print(f" RSS HTTP {e.code} for r/{subreddit}")
|
||||
return []
|
||||
except Exception as e:
|
||||
@@ -167,7 +172,7 @@ class RedditAdapter(SourceAdapter):
|
||||
|
||||
for attempt in range(2):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
posts = []
|
||||
@@ -302,16 +307,21 @@ class RedditAdapter(SourceAdapter):
|
||||
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
||||
|
||||
for attempt in range(3):
|
||||
for attempt in range(2): # max 2 attempts, fail fast
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
xml_data = resp.read().decode("utf-8")
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (403,):
|
||||
print(f" RSS blocked (HTTP {e.code}) for r/{subreddit}")
|
||||
return []
|
||||
if e.code == 429:
|
||||
wait = 5 * (attempt + 1)
|
||||
time.sleep(wait)
|
||||
continue
|
||||
if attempt == 0:
|
||||
time.sleep(2) # single retry with short backoff
|
||||
continue
|
||||
print(f" RSS rate-limited for r/{subreddit}, skip")
|
||||
return []
|
||||
print(f" RSS HTTP {e.code} for r/{subreddit}")
|
||||
return []
|
||||
except Exception as e:
|
||||
@@ -362,20 +372,27 @@ class RedditAdapter(SourceAdapter):
|
||||
Filters AutoModerator and sticky posts.
|
||||
"""
|
||||
now = datetime.now(timezone.utc)
|
||||
# Try JSON first — if it's blocked on the first subreddit, bail fast
|
||||
# rather than wasting time on all subreddits
|
||||
first_json = self._try_json(self.subreddits[0])
|
||||
if not first_json:
|
||||
# JSON is blocked site-wide, try one RSS to confirm
|
||||
test_rss = self._fetch_rss(self.subreddits[0])
|
||||
if not test_rss:
|
||||
print(" Reddit blocked (403/429), returning empty")
|
||||
return []
|
||||
# RSS works — fall through to full fetch below
|
||||
|
||||
all_entries = []
|
||||
seen_ids = set()
|
||||
json_worked = False
|
||||
json_worked = bool(first_json)
|
||||
|
||||
# Try JSON first
|
||||
for sub in self.subreddits[:2]: # Try JSON on first 2 subreddits
|
||||
posts = self._try_json(sub)
|
||||
if posts:
|
||||
json_worked = True
|
||||
for p in posts:
|
||||
if p["id"] not in seen_ids:
|
||||
seen_ids.add(p["id"])
|
||||
all_entries.append(p)
|
||||
time.sleep(self.rate_limit)
|
||||
# Add first JSON results
|
||||
for p in first_json:
|
||||
if p["id"] not in seen_ids:
|
||||
seen_ids.add(p["id"])
|
||||
all_entries.append(p)
|
||||
time.sleep(self.rate_limit)
|
||||
|
||||
# If JSON didn't work, fall back to RSS for all subreddits
|
||||
if not json_worked:
|
||||
|
||||
Reference in New Issue
Block a user