Sprint 3: Anti-bot retrieval layer + metrics module
Anti-bot changes (all 6 adapters): - Browser-grade User-Agent rotation (Chrome/Firefox on Linux/Windows) - Shared browser_headers() with Accept, Accept-Language, DNT - Session-consistent UA fingerprint (picked once, not per-request) - jitter_sleep() replaces fixed time.sleep() on all adapters - Exponential backoff on 429/503 already on reddit, now consistent New shared module: - adapters/__init__.py: browser_user_agent(), browser_headers(), jitter_sleep() - adapters/_http.py: HTTPClient class for future browser-mode adapters Metrics module (from Sprint 2 carry): - oracle/metrics.py: MetricsRun for log_adapter/log_verdicts/log_scores - oracle/weekly.py: SYSTEM HEALTH section wired to adapter_health - oracle/cli.py: metrics subparser with --adapters/--publish/--scores/--alerts Before: bot signatures like 'ai-oracle/0.1', 'python:athena:v0.1' After: 'Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0'
This commit is contained in:
@@ -9,6 +9,7 @@ import sqlite3
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from collections import Counter
|
||||
from oracle.metrics import get_adapter_health, get_metric_alerts
|
||||
|
||||
|
||||
WEEKLY_TEMPLATE = """# Weekly Review — {week_start} to {week_end}
|
||||
@@ -41,6 +42,10 @@ WEEKLY_TEMPLATE = """# Weekly Review — {week_start} to {week_end}
|
||||
|
||||
{wow_md}
|
||||
|
||||
## SYSTEM HEALTH
|
||||
|
||||
{health_md}
|
||||
|
||||
## RECOMMENDATIONS
|
||||
|
||||
{recs_md}
|
||||
@@ -216,10 +221,36 @@ def generate_weekly(
|
||||
output_path: str | None = None,
|
||||
) -> dict:
|
||||
"""Generate a complete weekly review."""
|
||||
db_path = conn.execute("SELECT file FROM pragma_database_list LIMIT 1").fetchone()[0] or "oracle.db"
|
||||
stats = fetch_weekly_stats(conn, days)
|
||||
trending = _trending_themes(conn, days)
|
||||
recommendations = _generate_recommendations(stats, trending)
|
||||
|
||||
# System health from metrics layer
|
||||
health_data = get_adapter_health(db_path, days=days)
|
||||
alerts = get_metric_alerts(db_path)
|
||||
|
||||
health_lines = []
|
||||
if health_data:
|
||||
for h in health_data:
|
||||
status = "✅"
|
||||
if h.get("max_consecutive_failures", 0) and h["max_consecutive_failures"] >= 3:
|
||||
status = "🔴"
|
||||
elif h.get("total_errors", 0) and h["runs"] and h["total_errors"] / h["runs"] > 0.5:
|
||||
status = "🟡"
|
||||
health_lines.append(
|
||||
f"- {status} **{h['adapter_name']}**: {h['total_fetched']} fetched, "
|
||||
f"{h['total_errors']} errors over {h['runs']} runs "
|
||||
f"(avg {h['avg_runtime_ms']:.0f}ms)"
|
||||
)
|
||||
else:
|
||||
health_lines.append("_No adapter health data yet (metrics tracking started recently)._")
|
||||
|
||||
if alerts:
|
||||
health_lines.extend(f"- {a}" for a in alerts)
|
||||
|
||||
health_md = "\n".join(health_lines)
|
||||
|
||||
# Format sections
|
||||
top_sources_md = "\n".join(
|
||||
f"- **{s}**: {c} entries" for s, c in stats["top_sources"]
|
||||
@@ -254,6 +285,8 @@ def generate_weekly(
|
||||
**stats,
|
||||
"trending": trending,
|
||||
"recommendations": recommendations,
|
||||
"health": health_data,
|
||||
"alerts": alerts,
|
||||
}
|
||||
review["markdown"] = WEEKLY_TEMPLATE.format(
|
||||
week_start=stats["week_start"],
|
||||
@@ -268,6 +301,7 @@ def generate_weekly(
|
||||
tier_md=tier_md,
|
||||
trending_md=trending_md,
|
||||
wow_md=wow,
|
||||
health_md=health_md,
|
||||
recs_md=recs_md,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user