athena-oracle/adapters/_http.py

#!/usr/bin/env python3
"""
Shared HTTP utilities for Athena adapters.

Purpose: Make adapter traffic look like real browser requests
instead of bot signatures. Centralized so all adapters benefit.

Features:
- Realistic User-Agent rotation (Chrome/Firefox/Safari on Linux/Windows/macOS)
- Standard browser headers (Accept, Accept-Language, DNT)
- Jittered sleep to break mechanical timing patterns
- Exponential backoff on 429/503 responses
"""

import random
import time
import urllib.request
import urllib.error


# Realistic User-Agent strings — rotated per session
USER_AGENTS = [
    # Chrome on Linux
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
    # Chrome on Windows
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
    # Firefox on Linux
    "Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0",
    # Firefox on Windows
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:139.0) Gecko/20100101 Firefox/139.0",
    # Safari on macOS
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15",
]

# Standard browser headers that every real request includes
BROWSER_HEADERS = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9",
    "Accept-Encoding": "gzip, deflate, br",
    "DNT": "1",
    "Sec-Fetch-Dest": "document",
    "Sec-Fetch-Mode": "navigate",
    "Sec-Fetch-Site": "none",
    "Sec-Fetch-User": "?1",
    "Upgrade-Insecure-Requests": "1",
}


class HTTPClient:
    """Browser-like HTTP client for adapters.

    Usage:
        client = HTTPClient(name="arxiv")
        data = client.get("https://example.com/api")
        client.jitter_sleep(3)  # polite spacing with randomness
    """

    def __init__(self, name: str = "athena", api_mode: bool = False):
        """
        Args:
            name: Adapter name for User-Agent identification fallback.
            api_mode: If True, use Accept: application/json headers
                     (for JSON APIs). If False, use browser-like HTML headers.
        """
        self.name = name
        self.api_mode = api_mode
        # Pick a User-Agent once per session — avoids fingerprint rotation
        # which is MORE suspicious than sticking to one identity.
        self.user_agent = random.choice(USER_AGENTS)

    def _headers(self) -> dict:
        """Build headers dict for a request."""
        headers = {"User-Agent": self.user_agent}

        if self.api_mode:
            headers.update({
                "Accept": "application/json, text/json, */*;q=0.8",
                "Accept-Language": "en-US,en;q=0.9",
                "Accept-Encoding": "gzip, deflate, br",
                "DNT": "1",
            })
        else:
            headers.update(BROWSER_HEADERS)

        return headers

    def get(self, url: str, extra_headers: dict | None = None,
            timeout: int = 30, max_retries: int = 2,
            backoff_base: float = 2.0) -> bytes | None:
        """Make a GET request with browser-like headers and retry/backoff.

        Args:
            url: Request URL.
            extra_headers: Additional headers to merge in.
            timeout: Request timeout in seconds.
            max_retries: Max retry attempts on 429/503.
            backoff_base: Base seconds for exponential backoff (2^n * base).

        Returns:
            Response body bytes, or None on persistent failure.
        """
        headers = self._headers()
        if extra_headers:
            headers.update(extra_headers)

        req = urllib.request.Request(url, headers=headers)

        for attempt in range(max_retries + 1):
            try:
                with urllib.request.urlopen(req, timeout=timeout) as resp:
                    return resp.read()
            except urllib.error.HTTPError as e:
                if e.code in (429, 503):
                    wait = backoff_base ** (attempt + 1) + random.uniform(0, 1)
                    if attempt < max_retries:
                        print(f"    HTTP {e.code}, retrying in {wait:.1f}s")
                        time.sleep(wait)
                        continue
                    print(f"    HTTP {e.code} after {max_retries} retries, giving up")
                    return None
                # Other HTTP errors — don't retry
                print(f"    HTTP {e.code} for {url[:80]}")
                return None
            except urllib.error.URLError as e:
                if attempt < max_retries:
                    wait = backoff_base ** (attempt + 1)
                    print(f"    URLError: {e.reason}, retrying in {wait:.1f}s")
                    time.sleep(wait)
                    continue
                print(f"    URLError after retries: {e.reason}")
                return None
            except Exception as e:
                print(f"    Request error: {e}")
                return None

        return None

    def get_json(self, url: str, extra_headers: dict | None = None,
                 timeout: int = 30, max_retries: int = 2) -> dict | list | None:
        """Make a GET request and parse JSON response."""
        import json
        data = self.get(url, extra_headers=extra_headers, timeout=timeout,
                       max_retries=max_retries)
        if data is None:
            return None
        try:
            return json.loads(data.decode("utf-8"))
        except (json.JSONDecodeError, UnicodeDecodeError) as e:
            print(f"    JSON decode error: {e}")
            return None

    @staticmethod
    def jitter_sleep(base_seconds: float, jitter_range: float = 0.3) -> None:
        """Sleep for base_seconds ± jitter_range fraction.

        Breaks mechanical timing patterns. Default ±30% jitter.

        Args:
            base_seconds: Base sleep duration.
            jitter_range: Fraction of base to randomize (0.3 = ±30%).
        """
        jitter = base_seconds * jitter_range * (2 * random.random() - 1)
        actual = base_seconds + jitter
        time.sleep(actual)