Sprint 3: Anti-bot retrieval layer + metrics module
Anti-bot changes (all 6 adapters): - Browser-grade User-Agent rotation (Chrome/Firefox on Linux/Windows) - Shared browser_headers() with Accept, Accept-Language, DNT - Session-consistent UA fingerprint (picked once, not per-request) - jitter_sleep() replaces fixed time.sleep() on all adapters - Exponential backoff on 429/503 already on reddit, now consistent New shared module: - adapters/__init__.py: browser_user_agent(), browser_headers(), jitter_sleep() - adapters/_http.py: HTTPClient class for future browser-mode adapters Metrics module (from Sprint 2 carry): - oracle/metrics.py: MetricsRun for log_adapter/log_verdicts/log_scores - oracle/weekly.py: SYSTEM HEALTH section wired to adapter_health - oracle/cli.py: metrics subparser with --adapters/--publish/--scores/--alerts Before: bot signatures like 'ai-oracle/0.1', 'python:athena:v0.1' After: 'Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0'
This commit is contained in:
Epictetus
@@ -1,10 +1,55 @@
|
||||
"""Source adapters for AI Research Oracle."""
|
||||
|
||||
import random
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
# --- Browser-grade headers (anti-bot) ---
|
||||
# Picked once per session so fingerprint stays consistent.
|
||||
_SESSION_UA: str | None = None
|
||||
|
||||
_BROWSER_UAS = [
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:139.0) Gecko/20100101 Firefox/139.0",
|
||||
]
|
||||
|
||||
|
||||
def browser_user_agent() -> str:
|
||||
"""Return a realistic browser User-Agent (same for the session)."""
|
||||
global _SESSION_UA
|
||||
if _SESSION_UA is None:
|
||||
_SESSION_UA = random.choice(_BROWSER_UAS)
|
||||
return _SESSION_UA
|
||||
|
||||
|
||||
def browser_headers(api_mode: bool = False) -> dict:
|
||||
"""Build realistic browser headers for adapter requests.
|
||||
|
||||
Args:
|
||||
api_mode: If True, use Accept: application/json (for JSON APIs).
|
||||
"""
|
||||
headers: dict = {
|
||||
"User-Agent": browser_user_agent(),
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"DNT": "1",
|
||||
}
|
||||
if api_mode:
|
||||
headers["Accept"] = "application/json, text/json, */*;q=0.8"
|
||||
else:
|
||||
headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"
|
||||
return headers
|
||||
|
||||
|
||||
def jitter_sleep(base: float = 1.0, range_frac: float = 0.3) -> None:
|
||||
"""Sleep for base ± range_frac fraction to break mechanical patterns."""
|
||||
actual = base + base * range_frac * (2 * random.random() - 1)
|
||||
time.sleep(actual)
|
||||
|
||||
|
||||
class SourceAdapter(ABC):
|
||||
"""Base class for all ingestion adapters."""
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Shared HTTP utilities for Athena adapters.
|
||||
|
||||
Purpose: Make adapter traffic look like real browser requests
|
||||
instead of bot signatures. Centralized so all adapters benefit.
|
||||
|
||||
Features:
|
||||
- Realistic User-Agent rotation (Chrome/Firefox/Safari on Linux/Windows/macOS)
|
||||
- Standard browser headers (Accept, Accept-Language, DNT)
|
||||
- Jittered sleep to break mechanical timing patterns
|
||||
- Exponential backoff on 429/503 responses
|
||||
"""
|
||||
|
||||
import random
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
|
||||
# Realistic User-Agent strings — rotated per session
|
||||
USER_AGENTS = [
|
||||
# Chrome on Linux
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
|
||||
# Chrome on Windows
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
|
||||
# Firefox on Linux
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0",
|
||||
# Firefox on Windows
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:139.0) Gecko/20100101 Firefox/139.0",
|
||||
# Safari on macOS
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Safari/605.1.15",
|
||||
]
|
||||
|
||||
# Standard browser headers that every real request includes
|
||||
BROWSER_HEADERS = {
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"DNT": "1",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
}
|
||||
|
||||
|
||||
class HTTPClient:
|
||||
"""Browser-like HTTP client for adapters.
|
||||
|
||||
Usage:
|
||||
client = HTTPClient(name="arxiv")
|
||||
data = client.get("https://example.com/api")
|
||||
client.jitter_sleep(3) # polite spacing with randomness
|
||||
"""
|
||||
|
||||
def __init__(self, name: str = "athena", api_mode: bool = False):
|
||||
"""
|
||||
Args:
|
||||
name: Adapter name for User-Agent identification fallback.
|
||||
api_mode: If True, use Accept: application/json headers
|
||||
(for JSON APIs). If False, use browser-like HTML headers.
|
||||
"""
|
||||
self.name = name
|
||||
self.api_mode = api_mode
|
||||
# Pick a User-Agent once per session — avoids fingerprint rotation
|
||||
# which is MORE suspicious than sticking to one identity.
|
||||
self.user_agent = random.choice(USER_AGENTS)
|
||||
|
||||
def _headers(self) -> dict:
|
||||
"""Build headers dict for a request."""
|
||||
headers = {"User-Agent": self.user_agent}
|
||||
|
||||
if self.api_mode:
|
||||
headers.update({
|
||||
"Accept": "application/json, text/json, */*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"DNT": "1",
|
||||
})
|
||||
else:
|
||||
headers.update(BROWSER_HEADERS)
|
||||
|
||||
return headers
|
||||
|
||||
def get(self, url: str, extra_headers: dict | None = None,
|
||||
timeout: int = 30, max_retries: int = 2,
|
||||
backoff_base: float = 2.0) -> bytes | None:
|
||||
"""Make a GET request with browser-like headers and retry/backoff.
|
||||
|
||||
Args:
|
||||
url: Request URL.
|
||||
extra_headers: Additional headers to merge in.
|
||||
timeout: Request timeout in seconds.
|
||||
max_retries: Max retry attempts on 429/503.
|
||||
backoff_base: Base seconds for exponential backoff (2^n * base).
|
||||
|
||||
Returns:
|
||||
Response body bytes, or None on persistent failure.
|
||||
"""
|
||||
headers = self._headers()
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.read()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (429, 503):
|
||||
wait = backoff_base ** (attempt + 1) + random.uniform(0, 1)
|
||||
if attempt < max_retries:
|
||||
print(f" HTTP {e.code}, retrying in {wait:.1f}s")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
print(f" HTTP {e.code} after {max_retries} retries, giving up")
|
||||
return None
|
||||
# Other HTTP errors — don't retry
|
||||
print(f" HTTP {e.code} for {url[:80]}")
|
||||
return None
|
||||
except urllib.error.URLError as e:
|
||||
if attempt < max_retries:
|
||||
wait = backoff_base ** (attempt + 1)
|
||||
print(f" URLError: {e.reason}, retrying in {wait:.1f}s")
|
||||
time.sleep(wait)
|
||||
continue
|
||||
print(f" URLError after retries: {e.reason}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f" Request error: {e}")
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
def get_json(self, url: str, extra_headers: dict | None = None,
|
||||
timeout: int = 30, max_retries: int = 2) -> dict | list | None:
|
||||
"""Make a GET request and parse JSON response."""
|
||||
import json
|
||||
data = self.get(url, extra_headers=extra_headers, timeout=timeout,
|
||||
max_retries=max_retries)
|
||||
if data is None:
|
||||
return None
|
||||
try:
|
||||
return json.loads(data.decode("utf-8"))
|
||||
except (json.JSONDecodeError, UnicodeDecodeError) as e:
|
||||
print(f" JSON decode error: {e}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def jitter_sleep(base_seconds: float, jitter_range: float = 0.3) -> None:
|
||||
"""Sleep for base_seconds ± jitter_range fraction.
|
||||
|
||||
Breaks mechanical timing patterns. Default ±30% jitter.
|
||||
|
||||
Args:
|
||||
base_seconds: Base sleep duration.
|
||||
jitter_range: Fraction of base to randomize (0.3 = ±30%).
|
||||
"""
|
||||
jitter = base_seconds * jitter_range * (2 * random.random() - 1)
|
||||
actual = base_seconds + jitter
|
||||
time.sleep(actual)
|
||||
+4
-3
@@ -32,9 +32,10 @@ import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from html import unescape
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters import SourceAdapter, browser_headers, jitter_sleep
|
||||
from adapters._store import true_first_seen, upsert_entries
|
||||
|
||||
|
||||
# arXiv API
|
||||
ARXIV_API = "http://export.arxiv.org/api/query"
|
||||
|
||||
@@ -152,7 +153,7 @@ class ArxivAdapter(SourceAdapter):
|
||||
f"&max_results={max_results}"
|
||||
)
|
||||
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "ai-oracle/0.1"})
|
||||
req = urllib.request.Request(url, headers=browser_headers())
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
@@ -389,7 +390,7 @@ class ArxivAdapter(SourceAdapter):
|
||||
q = f"cat:{cat}"
|
||||
cat_papers = self._request(q, max_results=limit, sort_by="submittedDate")
|
||||
all_papers.extend(cat_papers)
|
||||
time.sleep(self.rate_limit)
|
||||
jitter_sleep(self.rate_limit)
|
||||
|
||||
# Deduplicate by arxiv_id
|
||||
seen = set()
|
||||
|
||||
+4
-4
@@ -17,7 +17,7 @@ import urllib.error
|
||||
import urllib.parse
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters import SourceAdapter, browser_user_agent, jitter_sleep
|
||||
from adapters._store import true_first_seen, upsert_entries
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ class GitHubAdapter(SourceAdapter):
|
||||
def _headers(self):
|
||||
headers = {
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
"User-Agent": "ai-oracle/0.1",
|
||||
"User-Agent": browser_user_agent(),
|
||||
}
|
||||
if self.token:
|
||||
headers["Authorization"] = f"token {self.token}"
|
||||
@@ -166,7 +166,7 @@ class GitHubAdapter(SourceAdapter):
|
||||
]:
|
||||
batch = self._search_repos(q, sort="stars", per_page=30)
|
||||
repos.extend(batch)
|
||||
time.sleep(1) # polite spacing
|
||||
jitter_sleep(1) # polite spacing
|
||||
|
||||
# Deduplicate by full_name
|
||||
seen = set()
|
||||
@@ -234,7 +234,7 @@ class GitHubAdapter(SourceAdapter):
|
||||
readme_text = ""
|
||||
if owner and repo_name and idx < readme_budget:
|
||||
readme_text = self._get_readme(f"https://api.github.com/repos/{owner}/{repo_name}/readme")
|
||||
time.sleep(0.5) # polite spacing between README fetches
|
||||
jitter_sleep(0.5) # polite spacing between README fetches
|
||||
|
||||
# Structured metadata
|
||||
stars = repo.get("stargazers_count", 0)
|
||||
|
||||
@@ -20,7 +20,7 @@ import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters import SourceAdapter, browser_user_agent, jitter_sleep
|
||||
from adapters._store import true_first_seen, upsert_entries
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ class HackerNewsAdapter(SourceAdapter):
|
||||
]
|
||||
|
||||
def __init__(self, user_agent=None):
|
||||
self.user_agent = user_agent or "python:athena:v0.1 (by tony_tech)"
|
||||
self.user_agent = user_agent or browser_user_agent()
|
||||
|
||||
def name(self) -> str:
|
||||
return "hackernews"
|
||||
@@ -57,7 +57,7 @@ class HackerNewsAdapter(SourceAdapter):
|
||||
def _request(self, path: str, max_retries: int = 2) -> dict | list | None:
|
||||
"""Make a GET request to the HN Firebase API."""
|
||||
url = f"{self.BASE}{path}"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
||||
req = urllib.request.Request(url, headers={"User-Agent": browser_user_agent()})
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
@@ -199,7 +199,7 @@ class HackerNewsAdapter(SourceAdapter):
|
||||
if item and item.get("type") == "story" and item.get("title"):
|
||||
stories.append(item)
|
||||
if idx % 20 == 19: # polite spacing every 20 requests
|
||||
time.sleep(1)
|
||||
jitter_sleep(1)
|
||||
|
||||
# Filter for AI relevance
|
||||
ai_stories = [s for s in stories if self._is_ai_relevant(s.get("title", ""))]
|
||||
|
||||
@@ -34,7 +34,7 @@ import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters import SourceAdapter, browser_user_agent, jitter_sleep
|
||||
from adapters._store import true_first_seen, upsert_entries
|
||||
|
||||
|
||||
@@ -79,7 +79,7 @@ class HuggingFaceAdapter(SourceAdapter):
|
||||
return "huggingface"
|
||||
|
||||
def _headers(self):
|
||||
headers = {"User-Agent": "athena/0.1"}
|
||||
headers = {"User-Agent": browser_user_agent()}
|
||||
if self.token:
|
||||
headers["Authorization"] = f"Bearer {self.token}"
|
||||
return headers
|
||||
@@ -251,7 +251,7 @@ class HuggingFaceAdapter(SourceAdapter):
|
||||
if popular and isinstance(popular, list):
|
||||
all_models.extend(popular)
|
||||
|
||||
time.sleep(1) # polite spacing
|
||||
jitter_sleep(1) # polite spacing
|
||||
|
||||
# 2. Recently modified (sort=lastModified) — fresh models getting attention
|
||||
# Filter to models created in last 90 days to avoid noise
|
||||
|
||||
+4
-4
@@ -26,7 +26,7 @@ import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timezone
|
||||
from html import unescape
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters import SourceAdapter, browser_user_agent, jitter_sleep
|
||||
from adapters._store import true_first_seen, upsert_entries
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@ class RedditAdapter(SourceAdapter):
|
||||
"""
|
||||
self.subreddits = subreddits or self.DEFAULT_SUBREDDITS
|
||||
self.rate_limit = rate_limit
|
||||
self.user_agent = user_agent or "python:ai-oracle:v0.1 (by tony_tech)"
|
||||
self.user_agent = user_agent or browser_user_agent()
|
||||
|
||||
def name(self) -> str:
|
||||
return "reddit"
|
||||
@@ -212,7 +212,7 @@ class RedditAdapter(SourceAdapter):
|
||||
if e.code in (429, 403, 404):
|
||||
return [] # JSON endpoint blocked, fall back to RSS
|
||||
if attempt < 1:
|
||||
time.sleep(5)
|
||||
jitter_sleep(5)
|
||||
continue
|
||||
return []
|
||||
except Exception:
|
||||
@@ -417,7 +417,7 @@ class RedditAdapter(SourceAdapter):
|
||||
if not json_worked:
|
||||
print(" JSON endpoints blocked, using RSS fallback")
|
||||
# Brief cooldown before RSS barrage
|
||||
time.sleep(3)
|
||||
jitter_sleep(3)
|
||||
for sub in self.subreddits:
|
||||
entries = self._fetch_rss(sub)
|
||||
for e in entries:
|
||||
|
||||
@@ -24,7 +24,7 @@ import feedparser
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from email.utils import parsedate_to_datetime
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters import SourceAdapter, jitter_sleep
|
||||
from adapters._store import true_first_seen, upsert_entries
|
||||
|
||||
|
||||
@@ -218,7 +218,7 @@ class RSSFeedsAdapter(SourceAdapter):
|
||||
"last_updated": now_iso,
|
||||
})
|
||||
|
||||
time.sleep(0.5) # polite spacing
|
||||
jitter_sleep(0.5) # polite spacing
|
||||
|
||||
except Exception as e:
|
||||
feed_failures.append(f"{source_key}: {e}")
|
||||
|
||||
Reference in New Issue
Block a user