fix(adapters): shared retry helper + run_log failure_class + enable RSS (issues #1 #2 #9)

- adapters/__init__.py: add http_get() unified retry (429/5xx only, max 2
  attempts, capped exp backoff) + AdapterHTTPError carrying failure_class;
  SourceAdapter.last_failure_class set on failure for pipeline capture.
- arxiv/github/huggingface/hackernews/reddit: route HTTP through http_get.
  Preserves GitHub 403 rate-limit retry and Reddit 403/429 fast-bail.
- schema.sql + pipeline.py: add run_log.failure_class column; rollup most-
  severe class across sources (5xx>4xx>429>error>zero_fetch>ok).
- pipeline.py: ENABLE RSS in ENABLED_SOURCES (was registered, disabled).
- RSS smoke test surfaced 3 broken feeds (anthropic 404, googleai 404,
  metaai 301) — left as-is, captured in feed_failures; URL fix is separate
  discovery task, not guessed.

Verified: full dry-run fetches all 6 sources; github live fetch OK;
Reddit 429 fast-bail preserved; no import/syntax errors.
This commit is contained in:
Epictetus
2026-07-10 16:34:05 +00:00
parent 7ee1af3d7b
commit 23cce4d609
8 changed files with 209 additions and 137 deletions
+85
View File
@@ -1,11 +1,18 @@
"""Source adapters for AI Research Oracle."""
import urllib.request
import urllib.error
import time
from abc import ABC, abstractmethod
class SourceAdapter(ABC):
"""Base class for all ingestion adapters."""
def __init__(self):
# Set by http_get when a request fails (issue #2 classification)
self.last_failure_class = None
@abstractmethod
def name(self) -> str:
"""Source name: 'github', 'arxiv', 'reddit'."""
@@ -15,3 +22,81 @@ class SourceAdapter(ABC):
def fetch(self, query: str = "", limit: int = 20) -> list[dict]:
"""Return entries matching DB schema fields."""
pass
class AdapterHTTPError(Exception):
"""Raised by http_get on non-retryable or exhausted HTTP failures.
failure_class is one of: '429', '5xx', '4xx', 'error'.
Consumed by pipeline.py to populate run_log.failure_class (issue #2).
"""
def __init__(self, status, failure_class, message=""):
super().__init__(message)
self.status = status
self.failure_class = failure_class
def classify_http_status(code):
"""Map an HTTP status code to a run_log failure_class."""
if code == 429:
return "429"
if 500 <= code < 600:
return "5xx"
return "4xx" # 401/403/404 etc = client/config problem, not retried
def http_get(url, headers=None, timeout=15, max_retries=2,
backoff_base=2, retry_403_ratelimit=False, return_headers=False,
owner=None):
"""GET with a unified retry policy shared by all adapters (issue #1).
Retries ONLY on 429 and 5xx (transient). 4xx other than 429 are NOT
retried (config/client problems). Optional GitHub-style 403 rate-limit
handling via retry_403_ratelimit (waits for X-RateLimit-Reset header).
Returns response body bytes (or (body, headers) tuple if return_headers).
Raises AdapterHTTPError on non-retryable or exhausted failures, carrying
.failure_class for run_log classification. If owner is provided, sets
owner.last_failure_class so the pipeline can record it (issue #2).
"""
def _fail(code, fclass, msg):
if owner is not None:
owner.last_failure_class = fclass
raise AdapterHTTPError(code, fclass, msg)
req = urllib.request.Request(url, headers=headers or {})
last_err = None
for attempt in range(max_retries + 1):
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
body = resp.read()
if return_headers:
return body, resp.headers
return body
except urllib.error.HTTPError as e:
last_err = e
code = e.code
if code == 429 or (code == 403 and retry_403_ratelimit):
if attempt < max_retries:
if code == 403:
reset = int(e.headers.get("X-RateLimit-Reset", 0))
wait = (max(reset - int(time.time()), 0) + 1
if reset else backoff_base * (attempt + 1) * 15)
time.sleep(min(wait, 300))
else:
time.sleep(min(backoff_base * (attempt + 1), 30))
continue
fclass = "429" if code == 429 else "4xx"
_fail(code, fclass, f"HTTP {code} for {url} (exhausted)")
if 500 <= code < 600:
if attempt < max_retries:
time.sleep(min(backoff_base * (attempt + 1), 30))
continue
_fail(code, "5xx", f"HTTP {code} for {url} (exhausted)")
# other 4xx (401/403 w/o flag/404) — not retried
_fail(code, classify_http_status(code), f"HTTP {code} for {url}")
except Exception as e: # non-HTTP (timeout, DNS, conn reset)
_fail(None, "error", f"Request error: {e}")
# Defensive: loop should always raise or return
_fail(getattr(last_err, "code", None), "error", str(last_err))