fix(adapters): shared retry helper + run_log failure_class + enable RSS (issues #1 #2 #9)

- adapters/__init__.py: add http_get() unified retry (429/5xx only, max 2
  attempts, capped exp backoff) + AdapterHTTPError carrying failure_class;
  SourceAdapter.last_failure_class set on failure for pipeline capture.
- arxiv/github/huggingface/hackernews/reddit: route HTTP through http_get.
  Preserves GitHub 403 rate-limit retry and Reddit 403/429 fast-bail.
- schema.sql + pipeline.py: add run_log.failure_class column; rollup most-
  severe class across sources (5xx>4xx>429>error>zero_fetch>ok).
- pipeline.py: ENABLE RSS in ENABLED_SOURCES (was registered, disabled).
- RSS smoke test surfaced 3 broken feeds (anthropic 404, googleai 404,
  metaai 301) — left as-is, captured in feed_failures; URL fix is separate
  discovery task, not guessed.

Verified: full dry-run fetches all 6 sources; github live fetch OK;
Reddit 429 fast-bail preserved; no import/syntax errors.
This commit is contained in:
Epictetus
2026-07-10 16:34:05 +00:00
parent 7ee1af3d7b
commit 23cce4d609
8 changed files with 209 additions and 137 deletions
+39 -51
View File
@@ -24,7 +24,7 @@ import xml.etree.ElementTree as ET
from datetime import datetime, timezone
from html import unescape
from adapters import SourceAdapter
from adapters import SourceAdapter, http_get, AdapterHTTPError
class RedditAdapter(SourceAdapter):
@@ -101,33 +101,27 @@ class RedditAdapter(SourceAdapter):
return False
def _fetch_rss(self, subreddit: str) -> list[dict]:
"""Fetch RSS feed for a subreddit."""
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
"""Fetch RSS feed for a subreddit (shared retry helper).
for attempt in range(2): # max 2 attempts, fail fast
try:
with urllib.request.urlopen(req, timeout=10) as resp:
xml_data = resp.read().decode("utf-8")
break
except urllib.error.HTTPError as e:
if e.code in (403,):
print(f" RSS blocked (HTTP {e.code}) for r/{subreddit}")
return []
if e.code == 429:
if attempt == 0:
time.sleep(2) # single retry with short backoff
continue
print(f" RSS rate-limited for r/{subreddit}, skip")
return []
print(f" RSS HTTP {e.code} for r/{subreddit}")
return []
except Exception as e:
print(f" RSS error r/{subreddit}: {e}")
return []
else:
print(f" r/{subreddit}: still rate limited, skip")
Preserves prior fast-bail: 403 -> immediate []; 429 -> single 2s
retry then []; 5xx -> helper retry then []. Not slower than before.
"""
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
try:
raw = http_get(url, headers={"User-Agent": self.user_agent},
timeout=10, max_retries=1, backoff_base=2, owner=self)
except AdapterHTTPError as e:
if e.status == 403:
print(f" RSS blocked (HTTP 403) for r/{subreddit}")
elif e.status == 429:
print(f" RSS rate-limited for r/{subreddit}, skip")
else:
print(f" RSS {e.failure_class} for r/{subreddit}")
return []
except Exception as e:
print(f" RSS error r/{subreddit}: {e}")
return []
xml_data = raw.decode("utf-8")
# Parse Atom XML
entries = []
@@ -303,33 +297,27 @@ class RedditAdapter(SourceAdapter):
return tags
def _fetch_rss(self, subreddit: str) -> list[dict]:
"""Fetch RSS feed for a subreddit."""
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
"""Fetch RSS feed for a subreddit (shared retry helper).
for attempt in range(2): # max 2 attempts, fail fast
try:
with urllib.request.urlopen(req, timeout=10) as resp:
xml_data = resp.read().decode("utf-8")
break
except urllib.error.HTTPError as e:
if e.code in (403,):
print(f" RSS blocked (HTTP {e.code}) for r/{subreddit}")
return []
if e.code == 429:
if attempt == 0:
time.sleep(2) # single retry with short backoff
continue
print(f" RSS rate-limited for r/{subreddit}, skip")
return []
print(f" RSS HTTP {e.code} for r/{subreddit}")
return []
except Exception as e:
print(f" RSS error r/{subreddit}: {e}")
return []
else:
print(f" r/{subreddit}: still rate limited, skip")
Preserves prior fast-bail: 403 -> immediate []; 429 -> single 2s
retry then []; 5xx -> helper retry then []. Not slower than before.
"""
url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
try:
raw = http_get(url, headers={"User-Agent": self.user_agent},
timeout=10, max_retries=1, backoff_base=2, owner=self)
except AdapterHTTPError as e:
if e.status == 403:
print(f" RSS blocked (HTTP 403) for r/{subreddit}")
elif e.status == 429:
print(f" RSS rate-limited for r/{subreddit}, skip")
else:
print(f" RSS {e.failure_class} for r/{subreddit}")
return []
except Exception as e:
print(f" RSS error r/{subreddit}: {e}")
return []
xml_data = raw.decode("utf-8")
# Parse Atom XML
entries = []