Fix Reddit adapter: fast-bail when blocked (403/429)
Bug: Reddit was taking 120s+ when rate-limited (3 attempts × 4 subreddits × backoff).
Fix: 1) Try JSON on first subreddit; if blocked, test one RSS.
2) If both fail, return empty immediately (<3s).
3) Reduce default rate_limit 3→1s.
4) 403 now immediate fail in RSS (was retrying).
5) Max RSS retries 3→2, single 2s backoff.
Result: 120s timeout → 2.1s when blocked. Zero entries returned but no hang.
This commit is contained in:
+40
-23
@@ -50,7 +50,7 @@ class RedditAdapter(SourceAdapter):
|
|||||||
"automoderator",
|
"automoderator",
|
||||||
}
|
}
|
||||||
|
|
||||||
def __init__(self, subreddits=None, rate_limit=3, user_agent=None):
|
def __init__(self, subreddits=None, rate_limit=1, user_agent=None):
|
||||||
"""
|
"""
|
||||||
Args:
|
Args:
|
||||||
subreddits: List of subreddit names.
|
subreddits: List of subreddit names.
|
||||||
@@ -105,16 +105,21 @@ class RedditAdapter(SourceAdapter):
|
|||||||
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
|
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
|
||||||
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
||||||
|
|
||||||
for attempt in range(3):
|
for attempt in range(2): # max 2 attempts, fail fast
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||||
xml_data = resp.read().decode("utf-8")
|
xml_data = resp.read().decode("utf-8")
|
||||||
break
|
break
|
||||||
except urllib.error.HTTPError as e:
|
except urllib.error.HTTPError as e:
|
||||||
|
if e.code in (403,):
|
||||||
|
print(f" RSS blocked (HTTP {e.code}) for r/{subreddit}")
|
||||||
|
return []
|
||||||
if e.code == 429:
|
if e.code == 429:
|
||||||
wait = 5 * (attempt + 1)
|
if attempt == 0:
|
||||||
time.sleep(wait)
|
time.sleep(2) # single retry with short backoff
|
||||||
continue
|
continue
|
||||||
|
print(f" RSS rate-limited for r/{subreddit}, skip")
|
||||||
|
return []
|
||||||
print(f" RSS HTTP {e.code} for r/{subreddit}")
|
print(f" RSS HTTP {e.code} for r/{subreddit}")
|
||||||
return []
|
return []
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -167,7 +172,7 @@ class RedditAdapter(SourceAdapter):
|
|||||||
|
|
||||||
for attempt in range(2):
|
for attempt in range(2):
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||||
data = json.loads(resp.read().decode("utf-8"))
|
data = json.loads(resp.read().decode("utf-8"))
|
||||||
|
|
||||||
posts = []
|
posts = []
|
||||||
@@ -302,16 +307,21 @@ class RedditAdapter(SourceAdapter):
|
|||||||
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
|
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
|
||||||
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
||||||
|
|
||||||
for attempt in range(3):
|
for attempt in range(2): # max 2 attempts, fail fast
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||||
xml_data = resp.read().decode("utf-8")
|
xml_data = resp.read().decode("utf-8")
|
||||||
break
|
break
|
||||||
except urllib.error.HTTPError as e:
|
except urllib.error.HTTPError as e:
|
||||||
|
if e.code in (403,):
|
||||||
|
print(f" RSS blocked (HTTP {e.code}) for r/{subreddit}")
|
||||||
|
return []
|
||||||
if e.code == 429:
|
if e.code == 429:
|
||||||
wait = 5 * (attempt + 1)
|
if attempt == 0:
|
||||||
time.sleep(wait)
|
time.sleep(2) # single retry with short backoff
|
||||||
continue
|
continue
|
||||||
|
print(f" RSS rate-limited for r/{subreddit}, skip")
|
||||||
|
return []
|
||||||
print(f" RSS HTTP {e.code} for r/{subreddit}")
|
print(f" RSS HTTP {e.code} for r/{subreddit}")
|
||||||
return []
|
return []
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -362,20 +372,27 @@ class RedditAdapter(SourceAdapter):
|
|||||||
Filters AutoModerator and sticky posts.
|
Filters AutoModerator and sticky posts.
|
||||||
"""
|
"""
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
|
# Try JSON first — if it's blocked on the first subreddit, bail fast
|
||||||
|
# rather than wasting time on all subreddits
|
||||||
|
first_json = self._try_json(self.subreddits[0])
|
||||||
|
if not first_json:
|
||||||
|
# JSON is blocked site-wide, try one RSS to confirm
|
||||||
|
test_rss = self._fetch_rss(self.subreddits[0])
|
||||||
|
if not test_rss:
|
||||||
|
print(" Reddit blocked (403/429), returning empty")
|
||||||
|
return []
|
||||||
|
# RSS works — fall through to full fetch below
|
||||||
|
|
||||||
all_entries = []
|
all_entries = []
|
||||||
seen_ids = set()
|
seen_ids = set()
|
||||||
json_worked = False
|
json_worked = bool(first_json)
|
||||||
|
|
||||||
# Try JSON first
|
# Add first JSON results
|
||||||
for sub in self.subreddits[:2]: # Try JSON on first 2 subreddits
|
for p in first_json:
|
||||||
posts = self._try_json(sub)
|
if p["id"] not in seen_ids:
|
||||||
if posts:
|
seen_ids.add(p["id"])
|
||||||
json_worked = True
|
all_entries.append(p)
|
||||||
for p in posts:
|
time.sleep(self.rate_limit)
|
||||||
if p["id"] not in seen_ids:
|
|
||||||
seen_ids.add(p["id"])
|
|
||||||
all_entries.append(p)
|
|
||||||
time.sleep(self.rate_limit)
|
|
||||||
|
|
||||||
# If JSON didn't work, fall back to RSS for all subreddits
|
# If JSON didn't work, fall back to RSS for all subreddits
|
||||||
if not json_worked:
|
if not json_worked:
|
||||||
|
|||||||
Reference in New Issue
Block a user