Sprint 3: Anti-bot retrieval layer + metrics module
Anti-bot changes (all 6 adapters): - Browser-grade User-Agent rotation (Chrome/Firefox on Linux/Windows) - Shared browser_headers() with Accept, Accept-Language, DNT - Session-consistent UA fingerprint (picked once, not per-request) - jitter_sleep() replaces fixed time.sleep() on all adapters - Exponential backoff on 429/503 already on reddit, now consistent New shared module: - adapters/__init__.py: browser_user_agent(), browser_headers(), jitter_sleep() - adapters/_http.py: HTTPClient class for future browser-mode adapters Metrics module (from Sprint 2 carry): - oracle/metrics.py: MetricsRun for log_adapter/log_verdicts/log_scores - oracle/weekly.py: SYSTEM HEALTH section wired to adapter_health - oracle/cli.py: metrics subparser with --adapters/--publish/--scores/--alerts Before: bot signatures like 'ai-oracle/0.1', 'python:athena:v0.1' After: 'Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0'
This commit is contained in:
+4
-4
@@ -17,7 +17,7 @@ import urllib.error
|
||||
import urllib.parse
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters import SourceAdapter, browser_user_agent, jitter_sleep
|
||||
from adapters._store import true_first_seen, upsert_entries
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ class GitHubAdapter(SourceAdapter):
|
||||
def _headers(self):
|
||||
headers = {
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
"User-Agent": "ai-oracle/0.1",
|
||||
"User-Agent": browser_user_agent(),
|
||||
}
|
||||
if self.token:
|
||||
headers["Authorization"] = f"token {self.token}"
|
||||
@@ -166,7 +166,7 @@ class GitHubAdapter(SourceAdapter):
|
||||
]:
|
||||
batch = self._search_repos(q, sort="stars", per_page=30)
|
||||
repos.extend(batch)
|
||||
time.sleep(1) # polite spacing
|
||||
jitter_sleep(1) # polite spacing
|
||||
|
||||
# Deduplicate by full_name
|
||||
seen = set()
|
||||
@@ -234,7 +234,7 @@ class GitHubAdapter(SourceAdapter):
|
||||
readme_text = ""
|
||||
if owner and repo_name and idx < readme_budget:
|
||||
readme_text = self._get_readme(f"https://api.github.com/repos/{owner}/{repo_name}/readme")
|
||||
time.sleep(0.5) # polite spacing between README fetches
|
||||
jitter_sleep(0.5) # polite spacing between README fetches
|
||||
|
||||
# Structured metadata
|
||||
stars = repo.get("stargazers_count", 0)
|
||||
|
||||
Reference in New Issue
Block a user