fix(adapters): shared retry helper + run_log failure_class + enable RSS (issues #1 #2 #9)

- adapters/__init__.py: add http_get() unified retry (429/5xx only, max 2
  attempts, capped exp backoff) + AdapterHTTPError carrying failure_class;
  SourceAdapter.last_failure_class set on failure for pipeline capture.
- arxiv/github/huggingface/hackernews/reddit: route HTTP through http_get.
  Preserves GitHub 403 rate-limit retry and Reddit 403/429 fast-bail.
- schema.sql + pipeline.py: add run_log.failure_class column; rollup most-
  severe class across sources (5xx>4xx>429>error>zero_fetch>ok).
- pipeline.py: ENABLE RSS in ENABLED_SOURCES (was registered, disabled).
- RSS smoke test surfaced 3 broken feeds (anthropic 404, googleai 404,
  metaai 301) — left as-is, captured in feed_failures; URL fix is separate
  discovery task, not guessed.

Verified: full dry-run fetches all 6 sources; github live fetch OK;
Reddit 429 fast-bail preserved; no import/syntax errors.
This commit is contained in:
Epictetus
2026-07-10 16:34:05 +00:00
parent 7ee1af3d7b
commit 23cce4d609
8 changed files with 209 additions and 137 deletions
+22 -35
View File
@@ -17,7 +17,7 @@ import urllib.error
import urllib.parse
from datetime import datetime, timedelta, timezone
from adapters import SourceAdapter
from adapters import SourceAdapter, http_get, AdapterHTTPError
class GitHubAdapter(SourceAdapter):
@@ -43,40 +43,27 @@ class GitHubAdapter(SourceAdapter):
return headers
def _request(self, url: str, max_retries: int = 2) -> dict | list | None:
"""Make a GET request with retry on 403 (rate limit)."""
req = urllib.request.Request(url, headers=self._headers())
for attempt in range(max_retries + 1):
try:
with urllib.request.urlopen(req, timeout=15) as resp:
data = json.loads(resp.read().decode("utf-8"))
# Check rate limit headers
remaining = int(resp.headers.get("X-RateLimit-Remaining", 0))
if remaining <= 5:
print(f" ⚠ Rate limit low ({remaining} remaining), stopping")
break
return data
except urllib.error.HTTPError as e:
if e.code == 403:
# Rate limited — reset time is in headers
reset = int(e.headers.get("X-RateLimit-Reset", 0))
if reset:
wait = max(reset - int(time.time()), 0) + 1
print(f" ⚠ Rate limited, wait {wait}s")
else:
wait = 30 * (attempt + 1)
print(f" 403 on attempt {attempt + 1}, retry in {wait}s")
time.sleep(min(wait, 300)) # cap at 5 min
continue
print(f" HTTP {e.code} for {url}")
return None
except Exception as e:
print(f" Request error: {e}")
return None
return None
"""GET via shared retry helper; 403 rate-limit handled as transient."""
try:
raw, headers = http_get(
url, headers=self._headers(), timeout=15,
max_retries=max_retries, retry_403_ratelimit=True,
return_headers=True, owner=self)
except AdapterHTTPError as e:
print(f" {e.failure_class}: GitHub {url}")
return None
# Informational: flag if we're close to the unauth rate ceiling
try:
remaining = int(headers.get("X-RateLimit-Remaining", 0))
if remaining <= 5:
print(f" ⚠ Rate limit low ({remaining} remaining)")
except Exception:
pass
try:
return json.loads(raw.decode("utf-8"))
except Exception as e:
print(f" GitHub decode error: {e}")
return None
def _search_repos(self, query: str, sort: str = "stars", order: str = "desc", per_page: int = 30) -> list:
"""Search repositories via GitHub API."""