Sprint 3: Anti-bot retrieval layer + metrics module
Anti-bot changes (all 6 adapters): - Browser-grade User-Agent rotation (Chrome/Firefox on Linux/Windows) - Shared browser_headers() with Accept, Accept-Language, DNT - Session-consistent UA fingerprint (picked once, not per-request) - jitter_sleep() replaces fixed time.sleep() on all adapters - Exponential backoff on 429/503 already on reddit, now consistent New shared module: - adapters/__init__.py: browser_user_agent(), browser_headers(), jitter_sleep() - adapters/_http.py: HTTPClient class for future browser-mode adapters Metrics module (from Sprint 2 carry): - oracle/metrics.py: MetricsRun for log_adapter/log_verdicts/log_scores - oracle/weekly.py: SYSTEM HEALTH section wired to adapter_health - oracle/cli.py: metrics subparser with --adapters/--publish/--scores/--alerts Before: bot signatures like 'ai-oracle/0.1', 'python:athena:v0.1' After: 'Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0'
This commit is contained in:
@@ -1,10 +1,55 @@
|
||||
"""Source adapters for AI Research Oracle."""
|
||||
|
||||
import random
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
# --- Browser-grade headers (anti-bot) ---
|
||||
# Picked once per session so fingerprint stays consistent.
|
||||
_SESSION_UA: str | None = None
|
||||
|
||||
_BROWSER_UAS = [
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:139.0) Gecko/20100101 Firefox/139.0",
|
||||
]
|
||||
|
||||
|
||||
def browser_user_agent() -> str:
|
||||
"""Return a realistic browser User-Agent (same for the session)."""
|
||||
global _SESSION_UA
|
||||
if _SESSION_UA is None:
|
||||
_SESSION_UA = random.choice(_BROWSER_UAS)
|
||||
return _SESSION_UA
|
||||
|
||||
|
||||
def browser_headers(api_mode: bool = False) -> dict:
|
||||
"""Build realistic browser headers for adapter requests.
|
||||
|
||||
Args:
|
||||
api_mode: If True, use Accept: application/json (for JSON APIs).
|
||||
"""
|
||||
headers: dict = {
|
||||
"User-Agent": browser_user_agent(),
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"DNT": "1",
|
||||
}
|
||||
if api_mode:
|
||||
headers["Accept"] = "application/json, text/json, */*;q=0.8"
|
||||
else:
|
||||
headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"
|
||||
return headers
|
||||
|
||||
|
||||
def jitter_sleep(base: float = 1.0, range_frac: float = 0.3) -> None:
|
||||
"""Sleep for base ± range_frac fraction to break mechanical patterns."""
|
||||
actual = base + base * range_frac * (2 * random.random() - 1)
|
||||
time.sleep(actual)
|
||||
|
||||
|
||||
class SourceAdapter(ABC):
|
||||
"""Base class for all ingestion adapters."""
|
||||
|
||||
Reference in New Issue
Block a user