athena-oracle/adapters/__init__.py

"""Source adapters for AI Research Oracle."""

import random
import urllib.request
import urllib.error
import time
from abc import ABC, abstractmethod

# --- Browser-grade headers (anti-bot) ---
# Picked once per session so fingerprint stays consistent.
_SESSION_UA: str | None = None

_BROWSER_UAS = [
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:139.0) Gecko/20100101 Firefox/139.0",
]


def browser_user_agent() -> str:
    """Return a realistic browser User-Agent (same for the session)."""
    global _SESSION_UA
    if _SESSION_UA is None:
        _SESSION_UA = random.choice(_BROWSER_UAS)
    return _SESSION_UA


def browser_headers(api_mode: bool = False) -> dict:
    """Build realistic browser headers for adapter requests.

    Args:
        api_mode: If True, use Accept: application/json (for JSON APIs).
    """
    headers: dict = {
        "User-Agent": browser_user_agent(),
        "Accept-Language": "en-US,en;q=0.9",
        "Accept-Encoding": "gzip, deflate, br",
        "DNT": "1",
    }
    if api_mode:
        headers["Accept"] = "application/json, text/json, */*;q=0.8"
    else:
        headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"
    return headers


def jitter_sleep(base: float = 1.0, range_frac: float = 0.3) -> None:
    """Sleep for base ± range_frac fraction to break mechanical patterns."""
    actual = base + base * range_frac * (2 * random.random() - 1)
    time.sleep(actual)


class SourceAdapter(ABC):
    """Base class for all ingestion adapters."""

    def __init__(self):
        # Set by http_get when a request fails (issue #2 classification)
        self.last_failure_class = None

    @abstractmethod
    def name(self) -> str:
        """Source name: 'github', 'arxiv', 'reddit'."""
        pass

    @abstractmethod
    def fetch(self, query: str = "", limit: int = 20) -> list[dict]:
        """Return entries matching DB schema fields."""
        pass


class AdapterHTTPError(Exception):
    """Raised by http_get on non-retryable or exhausted HTTP failures.

    failure_class is one of: '429', '5xx', '4xx', 'error'.
    Consumed by pipeline.py to populate run_log.failure_class (issue #2).
    """

    def __init__(self, status, failure_class, message=""):
        super().__init__(message)
        self.status = status
        self.failure_class = failure_class


def classify_http_status(code):
    """Map an HTTP status code to a run_log failure_class."""
    if code == 429:
        return "429"
    if 500 <= code < 600:
        return "5xx"
    return "4xx"  # 401/403/404 etc = client/config problem, not retried


def http_get(url, headers=None, timeout=15, max_retries=2,
             backoff_base=2, retry_403_ratelimit=False, return_headers=False,
             owner=None):
    """GET with a unified retry policy shared by all adapters (issue #1).

    Retries ONLY on 429 and 5xx (transient). 4xx other than 429 are NOT
    retried (config/client problems). Optional GitHub-style 403 rate-limit
    handling via retry_403_ratelimit (waits for X-RateLimit-Reset header).

    Returns response body bytes (or (body, headers) tuple if return_headers).
    Raises AdapterHTTPError on non-retryable or exhausted failures, carrying
    .failure_class for run_log classification. If owner is provided, sets
    owner.last_failure_class so the pipeline can record it (issue #2).
    """
    def _fail(code, fclass, msg):
        if owner is not None:
            owner.last_failure_class = fclass
        raise AdapterHTTPError(code, fclass, msg)

    req = urllib.request.Request(url, headers=headers or {})
    last_err = None
    for attempt in range(max_retries + 1):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                body = resp.read()
                if return_headers:
                    return body, resp.headers
                return body
        except urllib.error.HTTPError as e:
            last_err = e
            code = e.code
            if code == 429 or (code == 403 and retry_403_ratelimit):
                if attempt < max_retries:
                    if code == 403:
                        reset = int(e.headers.get("X-RateLimit-Reset", 0))
                        wait = (max(reset - int(time.time()), 0) + 1
                                if reset else backoff_base * (attempt + 1) * 15)
                        time.sleep(min(wait, 300))
                    else:
                        time.sleep(min(backoff_base * (attempt + 1), 30))
                    continue
                fclass = "429" if code == 429 else "4xx"
                _fail(code, fclass, f"HTTP {code} for {url} (exhausted)")
            if 500 <= code < 600:
                if attempt < max_retries:
                    time.sleep(min(backoff_base * (attempt + 1), 30))
                    continue
                _fail(code, "5xx", f"HTTP {code} for {url} (exhausted)")
            # other 4xx (401/403 w/o flag/404) — not retried
            _fail(code, classify_http_status(code), f"HTTP {code} for {url}")
        except Exception as e:  # non-HTTP (timeout, DNS, conn reset)
            _fail(None, "error", f"Request error: {e}")
    # Defensive: loop should always raise or return
    _fail(getattr(last_err, "code", None), "error", str(last_err))