athena-oracle/oracle/weekly.py

"""Weekly review — synthesize the week's activity into patterns and trends.

CyrilXBT masterclass pattern: Sunday 7PM synthesis that identifies
recurring themes, tracks source performance, and generates insights.

Operates as a pure-rules aggregation layer — no AI synthesis needed.
"""
import sqlite3
from datetime import datetime, timedelta
from pathlib import Path
from collections import Counter
from oracle.metrics import get_adapter_health, get_metric_alerts


WEEKLY_TEMPLATE = """# Weekly Review — {week_start} to {week_end}

## SIGNAL SUMMARY

- **Total entries**: {total_entries}
- **New this week**: {new_this_week}
- **PUBLISH verdicts**: {publish_count}
- **WATCH verdicts**: {watch_count}
- **Average signal score**: {avg_score:.2f}

## TOP SOURCES (by entry count)

{sources_md}

## TOP CATEGORIES (by frequency)

{categories_md}

## SOURCE TIER BREAKDOWN

{tier_md}

## TRENDING THEMES

{trending_md}

## WEEK OVER WEEK COMPARISON

{wow_md}

## SYSTEM HEALTH

{health_md}

## RECOMMENDATIONS

{recs_md}
"""


def get_connection(db_path: str) -> sqlite3.Connection:
    conn = sqlite3.connect(db_path)
    conn.row_factory = sqlite3.Row
    return conn


def _week_bounds(days_ago: int = 7) -> tuple:
    end = datetime.utcnow()
    start = end - timedelta(days=days_ago)
    return start, end


def fetch_weekly_stats(conn: sqlite3.Connection, days: int = 7) -> dict:
    """Aggregate weekly statistics from Athena entries."""
    start, end = _week_bounds(days)

    # Total entries
    cur = conn.execute("SELECT COUNT(*) as c FROM entries")
    total = cur.fetchone()["c"]

    # New this week
    cur = conn.execute(
        "SELECT COUNT(*) as c FROM entries WHERE first_seen >= ?",
        (start.isoformat(),),
    )
    new_this_week = cur.fetchone()["c"]

    # Verdict breakdown
    cur = conn.execute(
        "SELECT verdict, COUNT(*) as c FROM entries "
        "WHERE first_seen >= ? GROUP BY verdict",
        (start.isoformat(),),
    )
    verdicts = {row["verdict"]: row["c"] for row in cur}

    # Average signal score
    cur = conn.execute(
        "SELECT AVG(signal_score) as avg_score FROM entries WHERE first_seen >= ?",
        (start.isoformat(),),
    )
    avg_score = cur.fetchone()["avg_score"] or 0

    # Top sources
    cur = conn.execute(
        "SELECT source, COUNT(*) as c FROM entries "
        "WHERE first_seen >= ? GROUP BY source ORDER BY c DESC LIMIT 10",
        (start.isoformat(),),
    )
    top_sources = [(row["source"], row["c"]) for row in cur]

    # Top categories
    cur = conn.execute(
        "SELECT category_tags, COUNT(*) as c FROM entries "
        "WHERE first_seen >= ? AND category_tags IS NOT NULL AND category_tags != '' "
        "GROUP BY category_tags ORDER BY c DESC LIMIT 10",
        (start.isoformat(),),
    )
    top_categories = [(row["category_tags"], row["c"]) for row in cur]

    # Tier breakdown
    cur = conn.execute(
        "SELECT source_tier, COUNT(*) as c FROM entries "
        "WHERE first_seen >= ? GROUP BY source_tier ORDER BY source_tier",
        (start.isoformat(),),
    )
    tier_breakdown = {
        {1: "Tier 1 (High)", 2: "Tier 2 (Mid)", 3: "Tier 3 (Low)"}
        .get(row["source_tier"], f"Tier {row['source_tier']}"): row["c"]
        for row in cur
    }

    return {
        "week_start": start.strftime("%Y-%m-%d"),
        "week_end": end.strftime("%Y-%m-%d"),
        "total_entries": total,
        "new_this_week": new_this_week,
        "verdicts": verdicts,
        "publish_count": verdicts.get("PUBLISH", 0),
        "watch_count": verdicts.get("WATCH", 0),
        "avg_score": avg_score,
        "top_sources": top_sources,
        "top_categories": top_categories,
        "tier_breakdown": tier_breakdown,
    }


def _trending_themes(conn: sqlite3.Connection, days: int = 7) -> list:
    """Identify themes that appear more frequently this week vs last week."""
    start, end = _week_bounds(days)
    prev_start = start - timedelta(days=days)

    # This week's categories
    cur = conn.execute(
        "SELECT category_tags, COUNT(*) as c FROM entries "
        "WHERE first_seen >= ? AND first_seen < ? AND category_tags IS NOT NULL "
        "GROUP BY category_tags",
        (start.isoformat(), end.isoformat()),
    )
    this_week = {row["category_tags"]: row["c"] for row in cur}

    # Last week's categories
    cur = conn.execute(
        "SELECT category_tags, COUNT(*) as c FROM entries "
        "WHERE first_seen >= ? AND first_seen < ? AND category_tags IS NOT NULL "
        "GROUP BY category_tags",
        (prev_start.isoformat(), start.isoformat()),
    )
    last_week = {row["category_tags"]: row["c"] for row in cur}

    # Find trending (increased by 50%+ or new)
    trending = []
    for cat, count in this_week.items():
        prev_count = last_week.get(cat, 0)
        if prev_count == 0 and count >= 2:
            trending.append((cat, count, "NEW"))
        elif prev_count > 0 and count >= prev_count * 1.5:
            trending.append((cat, count, f"+{int((count - prev_count) / prev_count * 100)}%"))

    trending.sort(key=lambda x: x[1], reverse=True)
    return trending


def _generate_recommendations(stats: dict, trending: list) -> list:
    """Generate recommendations based on weekly patterns."""
    recs = []

    # Source diversity check
    if stats["top_sources"]:
        top_source_pct = stats["top_sources"][0][1] / max(stats["new_this_week"], 1) * 100
        if top_source_pct > 40:
            recs.append(
                f"⚠ High concentration from {stats['top_sources'][0][0]} "
                f"({top_source_pct:.0f}% of entries). Consider diversifying sources."
            )

    # PUBLISH rate
    if stats["publish_count"] == 0 and stats["new_this_week"] > 10:
        recs.append(
            "🔍 Zero PUBLISH verdicts despite "
            f"{stats['new_this_week']} new entries. "
            "Signal scoring may need recalibration."
        )

    # Trending themes
    if trending:
        recs.append(
            f"📈 Trending themes: {', '.join(t[0] for t in trending[:3])}. "
            "Consider deeper coverage."
        )

    # Score distribution
    if stats["avg_score"] < 3.0 and stats["new_this_week"] > 0:
        recs.append(
            f"📉 Average signal score ({stats['avg_score']:.2f}) is low. "
            "Check adapter signal_score assignments."
        )

    if not recs:
        recs.append("✅ System operating within normal parameters.")

    return recs


def generate_weekly(
    conn: sqlite3.Connection,
    days: int = 7,
    output_path: str | None = None,
) -> dict:
    """Generate a complete weekly review."""
    db_path = conn.execute("SELECT file FROM pragma_database_list LIMIT 1").fetchone()[0] or "oracle.db"
    stats = fetch_weekly_stats(conn, days)
    trending = _trending_themes(conn, days)
    recommendations = _generate_recommendations(stats, trending)

    # System health from metrics layer
    health_data = get_adapter_health(db_path, days=days)
    alerts = get_metric_alerts(db_path)

    health_lines = []
    if health_data:
        for h in health_data:
            status = "✅"
            if h.get("max_consecutive_failures", 0) and h["max_consecutive_failures"] >= 3:
                status = "🔴"
            elif h.get("total_errors", 0) and h["runs"] and h["total_errors"] / h["runs"] > 0.5:
                status = "🟡"
            health_lines.append(
                f"- {status} **{h['adapter_name']}**: {h['total_fetched']} fetched, "
                f"{h['total_errors']} errors over {h['runs']} runs "
                f"(avg {h['avg_runtime_ms']:.0f}ms)"
            )
    else:
        health_lines.append("_No adapter health data yet (metrics tracking started recently)._")

    if alerts:
        health_lines.extend(f"- {a}" for a in alerts)

    health_md = "\n".join(health_lines)

    # Format sections
    top_sources_md = "\n".join(
        f"- **{s}**: {c} entries" for s, c in stats["top_sources"]
    ) or "_No sources._"

    top_cats_md = "\n".join(
        f"- **{c}**: {n} entries" for c, n in stats["top_categories"]
    ) or "_No categories._"

    tier_md = "\n".join(
        f"- **{label}**: {count}" for label, count in stats["tier_breakdown"].items()
    ) or "_No data._"

    trending_md = "\n".join(
        f"- **{t[0]}**: {t[1]} entries ({t[2]})" for t in trending
    ) or "_No trending themes detected._"

    # Week-over-week
    prev_count = stats["total_entries"] - stats["new_this_week"]
    if prev_count > 0:
        change = (stats["new_this_week"] - prev_count) / prev_count * 100
        wow = (
            f"New this week: {stats['new_this_week']} vs previous: {prev_count} "
            f"({change:+.0f}%)"
        )
    else:
        wow = f"First full week of data: {stats['new_this_week']} entries"

    recs_md = "\n".join(f"- {r}" for r in recommendations)

    review = {
        **stats,
        "trending": trending,
        "recommendations": recommendations,
        "health": health_data,
        "alerts": alerts,
    }
    review["markdown"] = WEEKLY_TEMPLATE.format(
        week_start=stats["week_start"],
        week_end=stats["week_end"],
        total_entries=stats["total_entries"],
        new_this_week=stats["new_this_week"],
        publish_count=stats["publish_count"],
        watch_count=stats["watch_count"],
        avg_score=stats["avg_score"],
        sources_md=top_sources_md,
        categories_md=top_cats_md,
        tier_md=tier_md,
        trending_md=trending_md,
        wow_md=wow,
        health_md=health_md,
        recs_md=recs_md,
    )

    if output_path:
        out = Path(output_path)
        out.parent.mkdir(parents=True, exist_ok=True)
        out.write_text(review["markdown"], encoding="utf-8")
        review["saved"] = str(out)

    return review


# ── CLI ──

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser(description="Generate Athena weekly review")
    parser.add_argument("--db", default="oracle.db")
    parser.add_argument("--days", type=int, default=7)
    parser.add_argument("--output", "-o", help="Output path for markdown review")
    parser.add_argument("--json", action="store_true", help="Output as JSON")

    args = parser.parse_args()
    conn = get_connection(args.db)

    try:
        review = generate_weekly(conn, days=args.days)
        if args.json:
            import json
            print(json.dumps({k: v for k, v in review.items() if k != "markdown"}, indent=2, default=str))
        elif args.output:
            generate_weekly(conn, days=args.days, output_path=args.output)
            print(f"Saved: {args.output}")
        else:
            print(review.get("markdown", "No review generated"))
    finally:
        conn.close()