feat: apply fix/adapter-health-1-2-9 changes (unified retry, failure_class, RSS enabled)
- adapters/__init__.py: add http_get() unified retry helper + AdapterHTTPError + failure_class classification (429/5xx/4xx/error/zero_fetch/ok) + last_failure_class on SourceAdapter for pipeline capture - pipeline.py: ENABLED_SOURCES now includes RSS + failure_class rollup in run_log (most severe across all adapters) + per-source failure_class in source_stats - schema.sql: add failure_class column to run_log table Backport of fix/adapter-health-1-2-9 branch (issues #1, #2, #9).
This commit is contained in:
@@ -1,11 +1,18 @@
|
||||
"""Source adapters for AI Research Oracle."""
|
||||
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class SourceAdapter(ABC):
|
||||
"""Base class for all ingestion adapters."""
|
||||
|
||||
def __init__(self):
|
||||
# Set by http_get when a request fails (issue #2 classification)
|
||||
self.last_failure_class = None
|
||||
|
||||
@abstractmethod
|
||||
def name(self) -> str:
|
||||
"""Source name: 'github', 'arxiv', 'reddit'."""
|
||||
@@ -15,3 +22,81 @@ class SourceAdapter(ABC):
|
||||
def fetch(self, query: str = "", limit: int = 20) -> list[dict]:
|
||||
"""Return entries matching DB schema fields."""
|
||||
pass
|
||||
|
||||
|
||||
class AdapterHTTPError(Exception):
|
||||
"""Raised by http_get on non-retryable or exhausted HTTP failures.
|
||||
|
||||
failure_class is one of: '429', '5xx', '4xx', 'error'.
|
||||
Consumed by pipeline.py to populate run_log.failure_class (issue #2).
|
||||
"""
|
||||
|
||||
def __init__(self, status, failure_class, message=""):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.failure_class = failure_class
|
||||
|
||||
|
||||
def classify_http_status(code):
|
||||
"""Map an HTTP status code to a run_log failure_class."""
|
||||
if code == 429:
|
||||
return "429"
|
||||
if 500 <= code < 600:
|
||||
return "5xx"
|
||||
return "4xx" # 401/403/404 etc = client/config problem, not retried
|
||||
|
||||
|
||||
def http_get(url, headers=None, timeout=15, max_retries=2,
|
||||
backoff_base=2, retry_403_ratelimit=False, return_headers=False,
|
||||
owner=None):
|
||||
"""GET with a unified retry policy shared by all adapters (issue #1).
|
||||
|
||||
Retries ONLY on 429 and 5xx (transient). 4xx other than 429 are NOT
|
||||
retried (config/client problems). Optional GitHub-style 403 rate-limit
|
||||
handling via retry_403_ratelimit (waits for X-RateLimit-Reset header).
|
||||
|
||||
Returns response body bytes (or (body, headers) tuple if return_headers).
|
||||
Raises AdapterHTTPError on non-retryable or exhausted failures, carrying
|
||||
.failure_class for run_log classification. If owner is provided, sets
|
||||
owner.last_failure_class so the pipeline can record it (issue #2).
|
||||
"""
|
||||
def _fail(code, fclass, msg):
|
||||
if owner is not None:
|
||||
owner.last_failure_class = fclass
|
||||
raise AdapterHTTPError(code, fclass, msg)
|
||||
|
||||
req = urllib.request.Request(url, headers=headers or {})
|
||||
last_err = None
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
body = resp.read()
|
||||
if return_headers:
|
||||
return body, resp.headers
|
||||
return body
|
||||
except urllib.error.HTTPError as e:
|
||||
last_err = e
|
||||
code = e.code
|
||||
if code == 429 or (code == 403 and retry_403_ratelimit):
|
||||
if attempt < max_retries:
|
||||
if code == 403:
|
||||
reset = int(e.headers.get("X-RateLimit-Reset", 0))
|
||||
wait = (max(reset - int(time.time()), 0) + 1
|
||||
if reset else backoff_base * (attempt + 1) * 15)
|
||||
time.sleep(min(wait, 300))
|
||||
else:
|
||||
time.sleep(min(backoff_base * (attempt + 1), 30))
|
||||
continue
|
||||
fclass = "429" if code == 429 else "4xx"
|
||||
_fail(code, fclass, f"HTTP {code} for {url} (exhausted)")
|
||||
if 500 <= code < 600:
|
||||
if attempt < max_retries:
|
||||
time.sleep(min(backoff_base * (attempt + 1), 30))
|
||||
continue
|
||||
_fail(code, "5xx", f"HTTP {code} for {url} (exhausted)")
|
||||
# other 4xx (401/403 w/o flag/404) — not retried
|
||||
_fail(code, classify_http_status(code), f"HTTP {code} for {url}")
|
||||
except Exception as e: # non-HTTP (timeout, DNS, conn reset)
|
||||
_fail(None, "error", f"Request error: {e}")
|
||||
# Defensive: loop should always raise or return
|
||||
_fail(getattr(last_err, "code", None), "error", str(last_err))
|
||||
|
||||
Reference in New Issue
Block a user