Sprint 3: Anti-bot retrieval layer + metrics module

Anti-bot changes (all 6 adapters):
- Browser-grade User-Agent rotation (Chrome/Firefox on Linux/Windows)
- Shared browser_headers() with Accept, Accept-Language, DNT
- Session-consistent UA fingerprint (picked once, not per-request)
- jitter_sleep() replaces fixed time.sleep() on all adapters
- Exponential backoff on 429/503 already on reddit, now consistent

New shared module:
- adapters/__init__.py: browser_user_agent(), browser_headers(), jitter_sleep()
- adapters/_http.py: HTTPClient class for future browser-mode adapters

Metrics module (from Sprint 2 carry):
- oracle/metrics.py: MetricsRun for log_adapter/log_verdicts/log_scores
- oracle/weekly.py: SYSTEM HEALTH section wired to adapter_health
- oracle/cli.py: metrics subparser with --adapters/--publish/--scores/--alerts

Before: bot signatures like 'ai-oracle/0.1', 'python:athena:v0.1'
After: 'Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0'
This commit is contained in:
Epictetus
2026-07-22 14:26:42 +00:00
parent 3ec955e143
commit 641d531d88
11 changed files with 694 additions and 21 deletions
+34
View File
@@ -9,6 +9,7 @@ import sqlite3
from datetime import datetime, timedelta
from pathlib import Path
from collections import Counter
from oracle.metrics import get_adapter_health, get_metric_alerts
WEEKLY_TEMPLATE = """# Weekly Review — {week_start} to {week_end}
@@ -41,6 +42,10 @@ WEEKLY_TEMPLATE = """# Weekly Review — {week_start} to {week_end}
{wow_md}
## SYSTEM HEALTH
{health_md}
## RECOMMENDATIONS
{recs_md}
@@ -216,10 +221,36 @@ def generate_weekly(
output_path: str | None = None,
) -> dict:
"""Generate a complete weekly review."""
db_path = conn.execute("SELECT file FROM pragma_database_list LIMIT 1").fetchone()[0] or "oracle.db"
stats = fetch_weekly_stats(conn, days)
trending = _trending_themes(conn, days)
recommendations = _generate_recommendations(stats, trending)
# System health from metrics layer
health_data = get_adapter_health(db_path, days=days)
alerts = get_metric_alerts(db_path)
health_lines = []
if health_data:
for h in health_data:
status = ""
if h.get("max_consecutive_failures", 0) and h["max_consecutive_failures"] >= 3:
status = "🔴"
elif h.get("total_errors", 0) and h["runs"] and h["total_errors"] / h["runs"] > 0.5:
status = "🟡"
health_lines.append(
f"- {status} **{h['adapter_name']}**: {h['total_fetched']} fetched, "
f"{h['total_errors']} errors over {h['runs']} runs "
f"(avg {h['avg_runtime_ms']:.0f}ms)"
)
else:
health_lines.append("_No adapter health data yet (metrics tracking started recently)._")
if alerts:
health_lines.extend(f"- {a}" for a in alerts)
health_md = "\n".join(health_lines)
# Format sections
top_sources_md = "\n".join(
f"- **{s}**: {c} entries" for s, c in stats["top_sources"]
@@ -254,6 +285,8 @@ def generate_weekly(
**stats,
"trending": trending,
"recommendations": recommendations,
"health": health_data,
"alerts": alerts,
}
review["markdown"] = WEEKLY_TEMPLATE.format(
week_start=stats["week_start"],
@@ -268,6 +301,7 @@ def generate_weekly(
tier_md=tier_md,
trending_md=trending_md,
wow_md=wow,
health_md=health_md,
recs_md=recs_md,
)