Sprint 3: Anti-bot retrieval layer + metrics module
Anti-bot changes (all 6 adapters): - Browser-grade User-Agent rotation (Chrome/Firefox on Linux/Windows) - Shared browser_headers() with Accept, Accept-Language, DNT - Session-consistent UA fingerprint (picked once, not per-request) - jitter_sleep() replaces fixed time.sleep() on all adapters - Exponential backoff on 429/503 already on reddit, now consistent New shared module: - adapters/__init__.py: browser_user_agent(), browser_headers(), jitter_sleep() - adapters/_http.py: HTTPClient class for future browser-mode adapters Metrics module (from Sprint 2 carry): - oracle/metrics.py: MetricsRun for log_adapter/log_verdicts/log_scores - oracle/weekly.py: SYSTEM HEALTH section wired to adapter_health - oracle/cli.py: metrics subparser with --adapters/--publish/--scores/--alerts Before: bot signatures like 'ai-oracle/0.1', 'python:athena:v0.1' After: 'Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0'
This commit is contained in:
@@ -20,7 +20,7 @@ import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters import SourceAdapter, browser_user_agent, jitter_sleep
|
||||
from adapters._store import true_first_seen, upsert_entries
|
||||
|
||||
|
||||
@@ -49,7 +49,7 @@ class HackerNewsAdapter(SourceAdapter):
|
||||
]
|
||||
|
||||
def __init__(self, user_agent=None):
|
||||
self.user_agent = user_agent or "python:athena:v0.1 (by tony_tech)"
|
||||
self.user_agent = user_agent or browser_user_agent()
|
||||
|
||||
def name(self) -> str:
|
||||
return "hackernews"
|
||||
@@ -57,7 +57,7 @@ class HackerNewsAdapter(SourceAdapter):
|
||||
def _request(self, path: str, max_retries: int = 2) -> dict | list | None:
|
||||
"""Make a GET request to the HN Firebase API."""
|
||||
url = f"{self.BASE}{path}"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
|
||||
req = urllib.request.Request(url, headers={"User-Agent": browser_user_agent()})
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
@@ -199,7 +199,7 @@ class HackerNewsAdapter(SourceAdapter):
|
||||
if item and item.get("type") == "story" and item.get("title"):
|
||||
stories.append(item)
|
||||
if idx % 20 == 19: # polite spacing every 20 requests
|
||||
time.sleep(1)
|
||||
jitter_sleep(1)
|
||||
|
||||
# Filter for AI relevance
|
||||
ai_stories = [s for s in stories if self._is_ai_relevant(s.get("title", ""))]
|
||||
|
||||
Reference in New Issue
Block a user