Tony_tech/athena-oracle
3
0
Fork 0
Code Issues 6 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Fix arxiv/reddit timeouts: batch queries, retry logic, reduced backoff

Browse Source
This commit is contained in:
Epictetus
2026-07-22 14:51:51 +00:00
parent 641d531d88
commit ce440149f0
3 changed files with 38 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
adapters/reddit.py
+6 -6
View File
@@ -114,10 +114,10 @@ class RedditAdapter(SourceAdapter):
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
backoff = [3, 8] # staggered backoff: 3s then 8s
backoff = [2, 5] # staggered backoff: 2s then 5s
for attempt in range(3): # max 3 attempts
try:
with urllib.request.urlopen(req, timeout=10) as resp:
with urllib.request.urlopen(req, timeout=8) as resp:
xml_data = resp.read().decode("utf-8")
break
except urllib.error.HTTPError as e:
@@ -128,7 +128,7 @@ class RedditAdapter(SourceAdapter):
if attempt < len(backoff):
delay = backoff[attempt]
print(f" RSS 429 for r/{subreddit}, retrying in {delay}s")
time.sleep(delay)
jitter_sleep(delay)
continue
print(f" RSS rate-limited for r/{subreddit}, skip")
return []
@@ -330,10 +330,10 @@ class RedditAdapter(SourceAdapter):
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
backoff = [3, 8] # staggered backoff: 3s then 8s
backoff = [2, 5] # staggered backoff: 2s then 5s
for attempt in range(3): # max 3 attempts
try:
with urllib.request.urlopen(req, timeout=10) as resp:
with urllib.request.urlopen(req, timeout=8) as resp:
xml_data = resp.read().decode("utf-8")
break
except urllib.error.HTTPError as e:
@@ -344,7 +344,7 @@ class RedditAdapter(SourceAdapter):
if attempt < len(backoff):
delay = backoff[attempt]
print(f" RSS 429 for r/{subreddit}, retrying in {delay}s")
time.sleep(delay)
jitter_sleep(delay)
continue
print(f" RSS rate-limited for r/{subreddit}, skip")
return []