"""Source adapters for AI Research Oracle.""" import random import urllib.request import urllib.error import time from abc import ABC, abstractmethod # --- Browser-grade headers (anti-bot) --- # Picked once per session so fingerprint stays consistent. _SESSION_UA: str | None = None _BROWSER_UAS = [ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:139.0) Gecko/20100101 Firefox/139.0", ] def browser_user_agent() -> str: """Return a realistic browser User-Agent (same for the session).""" global _SESSION_UA if _SESSION_UA is None: _SESSION_UA = random.choice(_BROWSER_UAS) return _SESSION_UA def browser_headers(api_mode: bool = False) -> dict: """Build realistic browser headers for adapter requests. Args: api_mode: If True, use Accept: application/json (for JSON APIs). """ headers: dict = { "User-Agent": browser_user_agent(), "Accept-Language": "en-US,en;q=0.9", "Accept-Encoding": "gzip, deflate, br", "DNT": "1", } if api_mode: headers["Accept"] = "application/json, text/json, */*;q=0.8" else: headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8" return headers def jitter_sleep(base: float = 1.0, range_frac: float = 0.3) -> None: """Sleep for base ± range_frac fraction to break mechanical patterns.""" actual = base + base * range_frac * (2 * random.random() - 1) time.sleep(actual) class SourceAdapter(ABC): """Base class for all ingestion adapters.""" def __init__(self): # Set by http_get when a request fails (issue #2 classification) self.last_failure_class = None @abstractmethod def name(self) -> str: """Source name: 'github', 'arxiv', 'reddit'.""" pass @abstractmethod def fetch(self, query: str = "", limit: int = 20) -> list[dict]: """Return entries matching DB schema fields.""" pass class AdapterHTTPError(Exception): """Raised by http_get on non-retryable or exhausted HTTP failures. failure_class is one of: '429', '5xx', '4xx', 'error'. Consumed by pipeline.py to populate run_log.failure_class (issue #2). """ def __init__(self, status, failure_class, message=""): super().__init__(message) self.status = status self.failure_class = failure_class def classify_http_status(code): """Map an HTTP status code to a run_log failure_class.""" if code == 429: return "429" if 500 <= code < 600: return "5xx" return "4xx" # 401/403/404 etc = client/config problem, not retried def http_get(url, headers=None, timeout=15, max_retries=2, backoff_base=2, retry_403_ratelimit=False, return_headers=False, owner=None): """GET with a unified retry policy shared by all adapters (issue #1). Retries ONLY on 429 and 5xx (transient). 4xx other than 429 are NOT retried (config/client problems). Optional GitHub-style 403 rate-limit handling via retry_403_ratelimit (waits for X-RateLimit-Reset header). Returns response body bytes (or (body, headers) tuple if return_headers). Raises AdapterHTTPError on non-retryable or exhausted failures, carrying .failure_class for run_log classification. If owner is provided, sets owner.last_failure_class so the pipeline can record it (issue #2). """ def _fail(code, fclass, msg): if owner is not None: owner.last_failure_class = fclass raise AdapterHTTPError(code, fclass, msg) req = urllib.request.Request(url, headers=headers or {}) last_err = None for attempt in range(max_retries + 1): try: with urllib.request.urlopen(req, timeout=timeout) as resp: body = resp.read() if return_headers: return body, resp.headers return body except urllib.error.HTTPError as e: last_err = e code = e.code if code == 429 or (code == 403 and retry_403_ratelimit): if attempt < max_retries: if code == 403: reset = int(e.headers.get("X-RateLimit-Reset", 0)) wait = (max(reset - int(time.time()), 0) + 1 if reset else backoff_base * (attempt + 1) * 15) time.sleep(min(wait, 300)) else: time.sleep(min(backoff_base * (attempt + 1), 30)) continue fclass = "429" if code == 429 else "4xx" _fail(code, fclass, f"HTTP {code} for {url} (exhausted)") if 500 <= code < 600: if attempt < max_retries: time.sleep(min(backoff_base * (attempt + 1), 30)) continue _fail(code, "5xx", f"HTTP {code} for {url} (exhausted)") # other 4xx (401/403 w/o flag/404) — not retried _fail(code, classify_http_status(code), f"HTTP {code} for {url}") except Exception as e: # non-HTTP (timeout, DNS, conn reset) _fail(None, "error", f"Request error: {e}") # Defensive: loop should always raise or return _fail(getattr(last_err, "code", None), "error", str(last_err))