Sprint 0+1: Package restructure, source tiers, verdicts, multi-variant editions
- New oracle/ package (11 modules) with unified CLI (python -m oracle) - Source tiers: Tier 1 (arxiv/github/hf), Tier 2 (rss/hn), Tier 3 (reddit) - Composite verdicts: PUBLISH/WATCH/ARCHIVE/DROP based on signal score + age - Content-hash dedup: SHA-256[:16] normalized, atomic at insert time - Multi-variant editions: 4 YAML configs (default/research/devops/brief) - Variant engine: filter → rank → render (HTML + JSON, themed) - Per-adapter timeout (10s) + threading fallback - Consolidated 12 root scripts → thin wrappers + oracle/ package - Archived stale scripts (_engagement, _live_compare, reddit_proof) - Updated .gitignore, README.md, schema.sql
This commit is contained in:
Epictetus
@@ -0,0 +1,21 @@
|
||||
"""Athena — AI Research Oracle.
|
||||
|
||||
Unified intelligence pipeline for AI news aggregation, scoring, and rendering.
|
||||
|
||||
Architecture:
|
||||
oracle/ - Core package (this directory)
|
||||
adapters/ - Source adapters (GitHub, arXiv, Reddit, HN, HF, RSS)
|
||||
athena/ - Scoring and classification logic
|
||||
cli.py - CLI entry point (subcommands)
|
||||
db.py - Database operations and schema
|
||||
render.py - Static site rendering
|
||||
scoring.py - Athena scoring engine
|
||||
clickability.py - Clickability index and decay
|
||||
summarize.py - Rule-based summarization
|
||||
recency.py - Recency guard and freshness filtering
|
||||
themes.py - Theme-based trend tracking
|
||||
archive.py - Soft-cap archival
|
||||
config.py - Configuration and constants
|
||||
"""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
@@ -0,0 +1,4 @@
|
||||
"""Allow running as: python -m oracle"""
|
||||
from oracle.cli import main
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Oracle soft-cap archival.
|
||||
|
||||
Bounds live `entries` growth by moving old / excess rows into
|
||||
`entries_archive` (preserving data — soft cap, not hard delete).
|
||||
"""
|
||||
import os
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
from oracle.config import DB_PATH
|
||||
|
||||
ARCHIVE_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS entries_archive (
|
||||
id INTEGER PRIMARY KEY,
|
||||
source TEXT, source_id TEXT, url TEXT, title TEXT,
|
||||
extracted_text TEXT, summary TEXT, category_tags TEXT,
|
||||
signal_score REAL, raw_metadata TEXT,
|
||||
first_seen TEXT, last_updated TEXT,
|
||||
archived_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now'))
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def archive(days=30, cap=5000, dry_run=False, db_path=None):
|
||||
"""Archive entries older than N days or beyond the cap.
|
||||
|
||||
Returns (total_live, archived_count).
|
||||
"""
|
||||
path = db_path or str(DB_PATH)
|
||||
if not os.path.exists(path):
|
||||
print("No oracle.db — nothing to archive")
|
||||
return 0, 0
|
||||
|
||||
conn = sqlite3.connect(path)
|
||||
conn.execute(ARCHIVE_SCHEMA)
|
||||
|
||||
cutoff = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - days * 86400))
|
||||
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT id, last_updated FROM entries")
|
||||
rows = cur.fetchall()
|
||||
n_total = len(rows)
|
||||
|
||||
old_ids = [r[0] for r in rows if (r[1] or "") < cutoff]
|
||||
beyond = max(0, n_total - cap)
|
||||
if beyond > 0:
|
||||
ordered = sorted(rows, key=lambda r: r[1] or "")[:beyond]
|
||||
cap_ids = [r[0] for r in ordered]
|
||||
else:
|
||||
cap_ids = []
|
||||
|
||||
move_ids = sorted(set(old_ids) | set(cap_ids))
|
||||
|
||||
if not move_ids:
|
||||
print(f"Archive check: {n_total} live entries, none older than {days}d "
|
||||
f"or beyond cap {cap}. Nothing to archive.")
|
||||
conn.close()
|
||||
return n_total, 0
|
||||
|
||||
print(f"Archive check: {n_total} live entries -> would archive {len(move_ids)} "
|
||||
f"(old={len(old_ids)}, cap={len(cap_ids)}).")
|
||||
|
||||
if dry_run:
|
||||
print("DRY RUN — no changes made.")
|
||||
conn.close()
|
||||
return n_total, 0
|
||||
|
||||
q = ",".join("?" * len(move_ids))
|
||||
conn.execute(
|
||||
f"""INSERT OR REPLACE INTO entries_archive
|
||||
(id, source, source_id, url, title, extracted_text, summary,
|
||||
category_tags, signal_score, raw_metadata, first_seen, last_updated)
|
||||
SELECT id, source, source_id, url, title, extracted_text, summary,
|
||||
category_tags, signal_score, raw_metadata, first_seen, last_updated
|
||||
FROM entries WHERE id IN ({q})""",
|
||||
move_ids,
|
||||
)
|
||||
conn.execute(f"DELETE FROM entries WHERE id IN ({q})", move_ids)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"Archived {len(move_ids)} entries (live now {n_total - len(move_ids)}). "
|
||||
f"Preserved in entries_archive.")
|
||||
return n_total, len(move_ids)
|
||||
+497
@@ -0,0 +1,497 @@
|
||||
"""CLI entry point for the AI Research Oracle.
|
||||
|
||||
Single command with subcommands for all operations.
|
||||
Replaces scattered root scripts with one unified interface.
|
||||
|
||||
Usage:
|
||||
python -m oracle <command> [args]
|
||||
|
||||
Commands:
|
||||
ingest Run the ingestion pipeline (fetch + store + score)
|
||||
summarize Generate summaries for unscored entries
|
||||
query Query the database (top, search, recent, stats, snapshot)
|
||||
render Render static site from clickability index
|
||||
archive Soft-cap archival of old entries
|
||||
themes Theme-based trend tracking
|
||||
recency Recency guard analysis
|
||||
health System health check
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
# Ensure project root is on path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from oracle.config import DB_PATH, SCHEMA_PATH, ENABLED_SOURCES, DEFAULT_LIMIT, SOURCE_TIERS, VERDICT_THRESHOLDS
|
||||
from oracle.db import get_connection, init_db, get_stats, query_top, query_recent, query_search, query_by_tag, migrate_world_monitor
|
||||
from oracle.scoring import attach_scoring, migrate as scoring_migrate
|
||||
from oracle.summarize import run_summarization
|
||||
from oracle.render import render as render_site
|
||||
from oracle.archive import archive as archive_entries
|
||||
from oracle.themes import scan as theme_scan
|
||||
from oracle.clickability import fetch_items, compute_index, decay_index
|
||||
from oracle.recency import filter_fresh
|
||||
from oracle.dedup import content_hash, compute_verdict, tier_adjusted_score, backfill_hashes, apply_verdicts, get_source_tier
|
||||
|
||||
|
||||
def cmd_ingest(args):
|
||||
"""Run the ingestion pipeline."""
|
||||
import time
|
||||
import sqlite3
|
||||
import signal
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters._store import upsert_entries
|
||||
|
||||
sources = args.sources.split(",") if args.sources else ENABLED_SOURCES
|
||||
now = datetime.now(timezone.utc)
|
||||
print(f"=== AI Research Oracle Pipeline ===")
|
||||
print(f" Sources: {', '.join(sources)}")
|
||||
print(f" Limit: {args.limit}/source")
|
||||
print(f" Dry run: {args.dry_run}")
|
||||
print()
|
||||
|
||||
# Import adapters dynamically
|
||||
adapter_modules = {
|
||||
"github": "adapters.github",
|
||||
"arxiv": "adapters.arxiv",
|
||||
"reddit": "adapters.reddit",
|
||||
"hackernews": "adapters.hackernews",
|
||||
"huggingface": "adapters.huggingface",
|
||||
"rss": "adapters.rss_feeds",
|
||||
}
|
||||
adapter_classes = {
|
||||
"github": "GitHubAdapter",
|
||||
"arxiv": "ArxivAdapter",
|
||||
"reddit": "RedditAdapter",
|
||||
"hackernews": "HackerNewsAdapter",
|
||||
"huggingface": "HuggingFaceAdapter",
|
||||
"rss": "RSSFeedsAdapter",
|
||||
}
|
||||
|
||||
db_path = str(DB_PATH)
|
||||
schema_path = str(SCHEMA_PATH)
|
||||
|
||||
all_entries = []
|
||||
source_stats = {}
|
||||
|
||||
for source_name in sources:
|
||||
if source_name not in adapter_modules:
|
||||
print(f" ⚠ Unknown source: {source_name}")
|
||||
continue
|
||||
|
||||
print(f" [{source_name}]")
|
||||
mod = __import__(adapter_modules[source_name], fromlist=[adapter_classes[source_name]])
|
||||
adapter = getattr(mod, adapter_classes[source_name])()
|
||||
|
||||
# Fetch with per-adapter timeout (prevents blocking on slow endpoints)
|
||||
entries = []
|
||||
error = None
|
||||
try:
|
||||
entries = adapter.fetch(limit=args.limit, timeout=10)
|
||||
except TypeError:
|
||||
# Old adapter signature without timeout param — use thread-based fallback
|
||||
import threading
|
||||
result = {"entries": [], "error": None}
|
||||
def _fetch():
|
||||
try:
|
||||
result["entries"] = adapter.fetch(limit=args.limit)
|
||||
except Exception as e:
|
||||
result["error"] = str(e)
|
||||
t = threading.Thread(target=_fetch, daemon=True)
|
||||
t.start()
|
||||
t.join(timeout=10)
|
||||
if t.is_alive():
|
||||
error = f"timeout after 10s"
|
||||
else:
|
||||
entries = result["entries"]
|
||||
error = result["error"]
|
||||
except Exception as e:
|
||||
error = str(e)
|
||||
|
||||
if error:
|
||||
print(f" ✗ {source_name} failed: {error}")
|
||||
source_stats[source_name] = {"fetched": 0, "stored": 0, "error": error}
|
||||
continue
|
||||
|
||||
for entry in entries:
|
||||
meta = json.loads(entry["raw_metadata"]) if isinstance(entry["raw_metadata"], str) else entry["raw_metadata"]
|
||||
meta["adapter_version"] = "1.0"
|
||||
entry["raw_metadata"] = json.dumps(meta)
|
||||
|
||||
all_entries.extend(entries)
|
||||
source_stats[source_name] = {"fetched": len(entries), "stored": 0}
|
||||
print(f" Fetched: {len(entries)} entries")
|
||||
time.sleep(0.5)
|
||||
|
||||
if not args.dry_run and all_entries:
|
||||
conn = init_db(db_path, schema_path)
|
||||
stored = upsert_entries(conn, all_entries)
|
||||
|
||||
for entry in all_entries:
|
||||
src = entry["source"]
|
||||
if src in source_stats:
|
||||
source_stats[src]["stored"] += 1
|
||||
|
||||
# Score new entries
|
||||
try:
|
||||
attach_scoring(db_path)
|
||||
except Exception as e:
|
||||
print(f" ⚠ scoring attach failed: {e}")
|
||||
|
||||
conn.close()
|
||||
print(f" Total stored: {stored} entries")
|
||||
else:
|
||||
print(f" Total fetched: {len(all_entries)} entries (dry run)")
|
||||
|
||||
print(f"\n Source summary:")
|
||||
for src, stats in source_stats.items():
|
||||
error = stats.get("error", "")
|
||||
error_str = f" ✗ {error}" if error else ""
|
||||
print(f" {src}: {stats['fetched']} fetched, {stats.get('stored', '—')} stored{error_str}")
|
||||
|
||||
if all_entries:
|
||||
print(f"\n Top entries by signal score:")
|
||||
sorted_entries = sorted(all_entries, key=lambda e: e["signal_score"], reverse=True)
|
||||
for i, entry in enumerate(sorted_entries[:5]):
|
||||
print(f" [{i+1}] {entry['source'].upper():6} score={entry['signal_score']:.2f} {entry['title'][:70]}")
|
||||
|
||||
print(f"\n Done.")
|
||||
|
||||
|
||||
def cmd_summarize(args):
|
||||
"""Run the summarization engine."""
|
||||
run_summarization(source=args.source, limit=args.limit)
|
||||
|
||||
|
||||
def cmd_query(args):
|
||||
"""Query the database."""
|
||||
conn = get_connection()
|
||||
qc = getattr(args, "query_command", None)
|
||||
|
||||
if qc == "top":
|
||||
entries = query_top(conn, n=args.n, source=args.source, min_score=args.min_score)
|
||||
print(f"Top {len(entries)} entries:")
|
||||
_print_entries(entries)
|
||||
|
||||
elif qc == "search":
|
||||
entries = query_search(conn, args.query_text, limit=args.limit)
|
||||
print(f"Search results for '{args.query_text}':")
|
||||
_print_entries(entries)
|
||||
|
||||
elif qc == "recent":
|
||||
entries = query_recent(conn, hours=args.hours)
|
||||
print(f"Entries from last {args.hours}h:")
|
||||
_print_entries(entries)
|
||||
|
||||
elif qc == "by-tag":
|
||||
entries = query_by_tag(conn, args.tag, limit=args.limit)
|
||||
print(f"Entries tagged '{args.tag}':")
|
||||
_print_entries(entries)
|
||||
|
||||
elif qc == "stats":
|
||||
stats = get_stats(conn)
|
||||
print(f"Database: {DB_PATH}")
|
||||
print(f"Total entries: {stats['total_entries']}")
|
||||
print(f"Summarized: {stats['summarized']}")
|
||||
print(f"Pending summary: {stats['pending_summary']}")
|
||||
if stats.get("buckets"):
|
||||
print(f"\nBucket distribution:")
|
||||
for bucket, count in sorted(stats["buckets"].items(), key=lambda x: -x[1]):
|
||||
print(f" {bucket}: {count}")
|
||||
|
||||
elif qc == "snapshot":
|
||||
stats = get_stats(conn)
|
||||
top = query_top(conn, n=10)
|
||||
print("=" * 70)
|
||||
print("AI RESEARCH ORACLE — SNAPSHOT")
|
||||
print("=" * 70)
|
||||
print(f"Time: {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}")
|
||||
print(f"Total entries: {stats['total_entries']}")
|
||||
print()
|
||||
print("Source breakdown:")
|
||||
for src, s in stats["sources"].items():
|
||||
print(f" {src}: {s['cnt']} entries, avg score {s['avg_score']}")
|
||||
print()
|
||||
print("Top 10 by signal score:")
|
||||
_print_entries(top)
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
def _print_entries(entries):
|
||||
"""Pretty-print a list of entries."""
|
||||
if not entries:
|
||||
print(" (no results)")
|
||||
return
|
||||
for i, e in enumerate(entries, 1):
|
||||
summary = json.loads(e["summary"]) if e.get("summary") else {}
|
||||
meta = json.loads(e["raw_metadata"]) if e.get("raw_metadata") else {}
|
||||
print(f" [{i}] {e['source'].upper():10} | score={e['signal_score']:.2f}")
|
||||
print(f" {e['title'][:80]}")
|
||||
if summary.get("one_liner"):
|
||||
print(f" → {summary['one_liner'][:100]}")
|
||||
print()
|
||||
|
||||
|
||||
def cmd_render(args):
|
||||
"""Render static site — single variant or all variants."""
|
||||
from oracle.variants import load_variant, list_variants, render_variant
|
||||
|
||||
if args.list:
|
||||
print("Available variants:")
|
||||
for v in list_variants():
|
||||
print(f" {v}")
|
||||
return
|
||||
|
||||
if args.all_variants:
|
||||
variants = list_variants()
|
||||
if not variants:
|
||||
print("No variants found in variants/")
|
||||
return
|
||||
print(f"=== Rendering all {len(variants)} variants ===\n")
|
||||
for vname in variants:
|
||||
try:
|
||||
config = load_variant(vname)
|
||||
render_variant(config, dry_run=args.dry_run, webroot=args.webroot)
|
||||
except Exception as e:
|
||||
print(f"[variant:{vname}] ERROR: {e}")
|
||||
print()
|
||||
return
|
||||
|
||||
# Single variant (default = 'default' if not specified)
|
||||
vname = args.variant or "default"
|
||||
try:
|
||||
config = load_variant(vname)
|
||||
render_variant(config, dry_run=args.dry_run, webroot=args.webroot)
|
||||
except FileNotFoundError as e:
|
||||
print(f"Error: {e}")
|
||||
|
||||
|
||||
def cmd_archive(args):
|
||||
"""Archive old entries."""
|
||||
archive_entries(days=args.days, cap=args.cap, dry_run=args.dry_run)
|
||||
|
||||
|
||||
def cmd_themes(args):
|
||||
"""Run theme scan."""
|
||||
results = theme_scan(history=args.history)
|
||||
print(f"=== Theme trend scan ===")
|
||||
print(f" Fresh entries this cycle: {results['fresh_count']}")
|
||||
if results["new_arrivals"]:
|
||||
print(" NEW theme arrivals this cycle:")
|
||||
for theme, count in results["new_arrivals"].items():
|
||||
print(f" {theme}: +{count}")
|
||||
else:
|
||||
print(" NEW theme arrivals this cycle: 0")
|
||||
print(f" Cumulative totals: {results['cumulative']}")
|
||||
|
||||
if args.history and results.get("history"):
|
||||
print("\n Per-cycle history:")
|
||||
for day, theme, count in results["history"]:
|
||||
print(f" {day} {theme}: {count}")
|
||||
|
||||
|
||||
def cmd_dedup(args):
|
||||
"""World Monitor migration: tiers, hashes, verdicts."""
|
||||
print("=== World Monitor Migration ===\n")
|
||||
|
||||
conn = get_connection()
|
||||
|
||||
# Show source tier config
|
||||
if args.show_tiers:
|
||||
print("Source tier configuration:")
|
||||
for src, info in SOURCE_TIERS.items():
|
||||
print(f" {src:12} Tier {info['tier']} ({info['label']}) - {info['description']}")
|
||||
print()
|
||||
|
||||
# Show verdict thresholds
|
||||
if args.show_verdicts:
|
||||
print("Verdict thresholds:")
|
||||
for verdict, thresholds in VERDICT_THRESHOLDS.items():
|
||||
print(f" {verdict:8} score >= {thresholds['min_score']}, age <= {thresholds['max_age_h']}h")
|
||||
print()
|
||||
|
||||
# Run migration
|
||||
if args.migrate:
|
||||
print("Running schema migration + backfill...\n")
|
||||
result = migrate_world_monitor(conn)
|
||||
print(f" Columns added: {result['columns_added']}")
|
||||
print(f" Hashes backfilled: {result['hashes_backfilled']}")
|
||||
print(f" Verdicts set: {result['verdicts_set']}")
|
||||
print()
|
||||
|
||||
# Show verdict distribution
|
||||
cur = conn.cursor()
|
||||
try:
|
||||
cur.execute("PRAGMA table_info(entries)")
|
||||
columns = {r[1] for r in cur.fetchall()}
|
||||
except Exception:
|
||||
conn.close()
|
||||
return
|
||||
|
||||
if "verdict" in columns:
|
||||
cur.execute("SELECT verdict, COUNT(*) as cnt FROM entries WHERE verdict != '' GROUP BY verdict ORDER BY cnt DESC")
|
||||
rows = cur.fetchall()
|
||||
if rows:
|
||||
print("Verdict distribution:")
|
||||
for r in rows:
|
||||
print(f" {r['verdict']:8} {r['cnt']}")
|
||||
print()
|
||||
|
||||
if "content_hash" in columns:
|
||||
cur.execute("SELECT COUNT(*) FROM entries WHERE content_hash != '' AND content_hash IS NOT NULL")
|
||||
hashed = cur.fetchone()[0]
|
||||
cur.execute("SELECT COUNT(*) FROM entries")
|
||||
total = cur.fetchone()[0]
|
||||
print(f"Content hashes: {hashed}/{total} entries hashed")
|
||||
|
||||
if "source_tier" in columns:
|
||||
cur.execute("SELECT source_tier, COUNT(*) as cnt FROM entries GROUP BY source_tier ORDER BY source_tier")
|
||||
rows = cur.fetchall()
|
||||
if rows:
|
||||
print(f"\nSource tier distribution:")
|
||||
for r in rows:
|
||||
print(f" Tier {r['source_tier']}: {r['cnt']}")
|
||||
|
||||
conn.close()
|
||||
print()
|
||||
|
||||
|
||||
def cmd_health(args):
|
||||
"""System health check."""
|
||||
print("=== System Health Check ===\n")
|
||||
|
||||
# Database
|
||||
try:
|
||||
conn = get_connection()
|
||||
stats = get_stats(conn)
|
||||
conn.close()
|
||||
print(f"✓ Database: {DB_PATH}")
|
||||
print(f" Total entries: {stats['total_entries']}")
|
||||
print(f" Summarized: {stats['summarized']}")
|
||||
print(f" Pending: {stats['pending_summary']}")
|
||||
except Exception as e:
|
||||
print(f"✗ Database error: {e}")
|
||||
|
||||
# Schema columns
|
||||
try:
|
||||
conn = get_connection()
|
||||
cur = conn.cursor()
|
||||
cur.execute("PRAGMA table_info(entries)")
|
||||
columns = [r[1] for r in cur.fetchall()]
|
||||
conn.close()
|
||||
print(f" Schema columns: {len(columns)}")
|
||||
except Exception as e:
|
||||
print(f" Schema check: {e}")
|
||||
|
||||
# Run log
|
||||
try:
|
||||
conn = get_connection()
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT COUNT(*) FROM run_log")
|
||||
runs = cur.fetchone()[0]
|
||||
cur.execute("SELECT run_time, failure_class FROM run_log ORDER BY id DESC LIMIT 3")
|
||||
recent = cur.fetchall()
|
||||
conn.close()
|
||||
print(f" Pipeline runs logged: {runs}")
|
||||
for r in recent:
|
||||
print(f" {r[0]} class={r[1]}")
|
||||
except Exception as e:
|
||||
print(f" Run log: {e}")
|
||||
|
||||
print()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="AI Research Oracle — Unified CLI",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
sub = parser.add_subparsers(dest="command", help="Available commands")
|
||||
|
||||
# ingest
|
||||
p_ingest = sub.add_parser("ingest", help="Run the ingestion pipeline")
|
||||
p_ingest.add_argument("--sources", default=None, help="Comma-separated sources")
|
||||
p_ingest.add_argument("--limit", type=int, default=DEFAULT_LIMIT, help="Entries per source")
|
||||
p_ingest.add_argument("--dry-run", action="store_true", help="Fetch but don't store")
|
||||
|
||||
# summarize
|
||||
p_summarize = sub.add_parser("summarize", help="Generate summaries")
|
||||
p_summarize.add_argument("--source", default=None, help="Filter by source")
|
||||
p_summarize.add_argument("--limit", type=int, default=0, help="Max entries (0=all)")
|
||||
|
||||
# query (nested subcommands)
|
||||
p_query = sub.add_parser("query", help="Query the database")
|
||||
query_sub = p_query.add_subparsers(dest="query_command")
|
||||
|
||||
p_top = query_sub.add_parser("top", help="Top N entries")
|
||||
p_top.add_argument("n", type=int, nargs="?", default=10)
|
||||
p_top.add_argument("--source", default=None)
|
||||
p_top.add_argument("--min-score", type=float, default=0)
|
||||
|
||||
p_search = query_sub.add_parser("search", help="Keyword search")
|
||||
p_search.add_argument("query_text")
|
||||
p_search.add_argument("--limit", type=int, default=20)
|
||||
|
||||
p_recent = query_sub.add_parser("recent", help="Recent entries")
|
||||
p_recent.add_argument("--hours", type=int, default=24)
|
||||
|
||||
p_tag = query_sub.add_parser("by-tag", help="Entries by tag")
|
||||
p_tag.add_argument("tag")
|
||||
p_tag.add_argument("--limit", type=int, default=20)
|
||||
|
||||
query_sub.add_parser("stats", help="Database statistics")
|
||||
query_sub.add_parser("snapshot", help="Full snapshot")
|
||||
|
||||
# render
|
||||
p_render = sub.add_parser("render", help="Render static site")
|
||||
p_render.add_argument("--variant", default=None, help="Variant name (default, research, devops, brief)")
|
||||
p_render.add_argument("--all", dest="all_variants", action="store_true", help="Render all variants")
|
||||
p_render.add_argument("--list", action="store_true", help="List available variants")
|
||||
p_render.add_argument("--dry-run", action="store_true")
|
||||
p_render.add_argument("--webroot", default=None, help="Output directory")
|
||||
|
||||
# archive
|
||||
p_archive = sub.add_parser("archive", help="Archive old entries")
|
||||
p_archive.add_argument("--days", type=int, default=30)
|
||||
p_archive.add_argument("--cap", type=int, default=5000)
|
||||
p_archive.add_argument("--dry-run", action="store_true")
|
||||
|
||||
# themes
|
||||
p_themes = sub.add_parser("themes", help="Theme trend tracking")
|
||||
p_themes.add_argument("--history", action="store_true")
|
||||
|
||||
# dedup (World Monitor migration)
|
||||
p_dedup = sub.add_parser("dedup", help="World Monitor: tiers, hashes, verdicts")
|
||||
p_dedup.add_argument("--migrate", action="store_true", help="Run schema migration + backfill")
|
||||
p_dedup.add_argument("--show-tiers", action="store_true", help="Show source tier config")
|
||||
p_dedup.add_argument("--show-verdicts", action="store_true", help="Show verdict thresholds")
|
||||
|
||||
# health
|
||||
sub.add_parser("health", help="System health check")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
commands = {
|
||||
"ingest": cmd_ingest,
|
||||
"summarize": cmd_summarize,
|
||||
"query": cmd_query,
|
||||
"render": cmd_render,
|
||||
"archive": cmd_archive,
|
||||
"themes": cmd_themes,
|
||||
"dedup": cmd_dedup,
|
||||
"health": cmd_health,
|
||||
}
|
||||
|
||||
if args.command and args.command in commands:
|
||||
commands[args.command](args)
|
||||
else:
|
||||
parser.print_help()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Clickability Index for Athena entries.
|
||||
|
||||
Read-only against the DB (SELECT only). Computes virality ranking with
|
||||
exponential time-decay so items sink as they age.
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from oracle.config import DB_PATH
|
||||
|
||||
# Virality weights (clickability = how viral/spreadable an item is right now)
|
||||
VEL_W = 0.50
|
||||
ENG_W = 0.50
|
||||
SIG_W = 0.0
|
||||
|
||||
NOW = None # set in fetch_items for age math
|
||||
|
||||
# Category-specific half-lives (hours)
|
||||
CATEGORY_HALF_LIVES = {
|
||||
"breaking": 36.0,
|
||||
"update": 24.0,
|
||||
"OTHER": 18.0,
|
||||
}
|
||||
|
||||
|
||||
def get_connection():
|
||||
return __import__("sqlite3").connect(str(DB_PATH))
|
||||
|
||||
|
||||
def _classify(src: str, title: str, summary: str) -> str:
|
||||
t = (title + " " + (summary or "")).lower()
|
||||
if re.search(r"\bshow\s+hn\b", t) or (src == "hackernews" and re.search(r"\b(show|built|made|launched|shipped)\b", t)):
|
||||
return "SHOW_HN"
|
||||
if re.search(r"\b(gpt-|gpt5|gpt-5|deepseek|glm-|llama|qwen|claude|gemini|mistral|flux|stable-diffusion|sora|kimi|grok)\b", t) \
|
||||
and re.search(r"\b(releases?|released|v\d|launch|unveil|model|new\s+model|update|version)\b", t):
|
||||
return "MODEL_RELEASE"
|
||||
if re.search(r"\b(releases?|released|launches?|unveils?|announces?|debut|new\s+model|gpt-5|deepseek-v|glm-5)\b", t) \
|
||||
and re.search(r"\b(openai|anthropic|google|meta|microsoft|nvidia|ai)\b", t):
|
||||
return "MODEL_RELEASE"
|
||||
if src == "huggingface":
|
||||
return "MODEL_CARD"
|
||||
if src == "arxiv" or re.search(r"\b(paper|study|benchmark|arxiv|proposes|learns?|novel|framework\s+for|towards)\b", t):
|
||||
return "RESEARCH"
|
||||
if re.search(r"\b(sues|lawsuit|funding|raises|acqui|ipo|valued|stealing|trade secret|layoff|hire[ds]?|exec|ceo)\b", t) \
|
||||
and not re.search(r"\b(repo|library|tool|agent framework)\b", t):
|
||||
return "BUSINESS_LEGAL"
|
||||
if re.search(r"\b(burnout|opinion|think|feel|why|essay|culture|linkedin|social media|future of|we made|i think|hot take|i believe|my view|in defense)\b", t):
|
||||
return "CULTURE_OPINION"
|
||||
if re.search(r"\b(how to|tutorial|guide|running|build|setup|install|from scratch|learn)\b", t):
|
||||
return "TUTORIAL_HOWTO"
|
||||
if src == "github" or re.search(r"\b(repo|library|framework|tool|agent|sdk|cli|extension|plugin|app|engine)\b", t):
|
||||
return "DEV_TOOL_DRAMA"
|
||||
return "OTHER"
|
||||
|
||||
|
||||
def _extract(src: str, md: dict) -> tuple:
|
||||
"""Return (velocity_raw, engagement_raw, age_hours)."""
|
||||
if src == "hackernews":
|
||||
pts = md.get("score", 0) or 0
|
||||
cmts = md.get("descendants", 0) or 0
|
||||
age_h = None
|
||||
if md.get("time"):
|
||||
try:
|
||||
age_h = max((NOW - md["time"]) / 3600.0, 0.1)
|
||||
except Exception:
|
||||
age_h = None
|
||||
vel = (pts / age_h) if age_h else pts
|
||||
return vel, (pts + 2 * cmts), age_h
|
||||
if src == "reddit":
|
||||
ups = md.get("ups", 0) or 0
|
||||
cmts = md.get("num_comments", 0) or 0
|
||||
return ups, (ups + 2 * cmts), None
|
||||
if src == "huggingface":
|
||||
likes = md.get("likes", 0) or 0
|
||||
return likes, likes, None
|
||||
if src == "github":
|
||||
spd = md.get("stars_per_day", 0) or 0
|
||||
stars = md.get("stars", 0) or 0
|
||||
return spd, stars, None
|
||||
if src == "arxiv":
|
||||
return 0.0, 0.0, None
|
||||
return 0.0, 0.0, None
|
||||
|
||||
|
||||
def fetch_items(conn) -> list[dict]:
|
||||
"""Fetch all entries and compute raw engagement signals."""
|
||||
global NOW
|
||||
NOW = time.time()
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT id, title, url, source, summary, signal_score, raw_metadata, first_seen,
|
||||
curated_by, manual_section, manual_tier
|
||||
FROM entries
|
||||
""")
|
||||
cols = [d[0] for d in cur.description]
|
||||
out = []
|
||||
for row in cur.fetchall():
|
||||
d = dict(zip(cols, row))
|
||||
try:
|
||||
md = json.loads(d.get("raw_metadata") or "{}")
|
||||
except Exception:
|
||||
md = {}
|
||||
vel, eng, age = _extract(d["source"], md)
|
||||
ct = _classify(d["source"], d.get("title") or "", d.get("summary") or "")
|
||||
created_at = md.get("createdAt") if d["source"] == "huggingface" else None
|
||||
out.append({
|
||||
"id": d["id"],
|
||||
"title": d.get("title") or "",
|
||||
"url": d.get("url") or "",
|
||||
"source": d["source"],
|
||||
"summary": d.get("summary") or "",
|
||||
"signal_score": d.get("signal_score") or 0,
|
||||
"velocity_raw": vel,
|
||||
"engagement_raw": eng,
|
||||
"content_type": ct,
|
||||
"first_seen": d.get("first_seen") or "",
|
||||
"created_at": created_at or "",
|
||||
"age_hours": 0.0,
|
||||
"curated_by": d.get("curated_by") or "",
|
||||
"manual_section": d.get("manual_section") or "",
|
||||
"manual_tier": d.get("manual_tier") or "",
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def log1p_norm(values: list[float]) -> list[float]:
|
||||
"""Log1p + min-max normalization."""
|
||||
log_vals = [math.log1p(max(v, 0)) for v in values]
|
||||
if not log_vals:
|
||||
return []
|
||||
min_v, max_v = min(log_vals), max(log_vals)
|
||||
if max_v == min_v:
|
||||
return [0.0] * len(values)
|
||||
return [(v - min_v) / (max_v - min_v) for v in log_vals]
|
||||
|
||||
|
||||
def compute_index(items: list[dict]) -> list[dict]:
|
||||
"""Compute clickability index for all items."""
|
||||
velocities = [it.get("velocity_raw", 0) or 0 for it in items]
|
||||
engagements = [it.get("engagement_raw", 0) or 0 for it in items]
|
||||
signals = [it.get("signal_score", 0) or 0 for it in items]
|
||||
|
||||
vel_norm = log1p_norm(velocities)
|
||||
eng_norm = log1p_norm(engagements)
|
||||
sig_norm = log1p_norm(signals)
|
||||
|
||||
for i, item in enumerate(items):
|
||||
raw = vel_norm[i] * VEL_W + eng_norm[i] * ENG_W + sig_norm[i] * SIG_W
|
||||
if raw == 0 and sig_norm[i] > 0:
|
||||
raw = 0.05 * sig_norm[i]
|
||||
item["clickability"] = round(raw, 4)
|
||||
item["section"] = ""
|
||||
return items
|
||||
|
||||
|
||||
def _age_hours(item: dict) -> float:
|
||||
"""Effective news-age in hours."""
|
||||
if item.get("source") == "huggingface" and item.get("created_at"):
|
||||
s = item["created_at"]
|
||||
else:
|
||||
s = item.get("first_seen") or ""
|
||||
if not s:
|
||||
return 0.0
|
||||
try:
|
||||
ts = datetime.strptime(s[:19], "%Y-%m-%dT%H:%M:%S").replace(
|
||||
tzinfo=timezone.utc
|
||||
).timestamp()
|
||||
return max((time.time() - ts) / 3600.0, 0.0)
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _get_half_life(item: dict) -> Optional[float]:
|
||||
"""Return section/tier-specific half-life in hours, or None for default."""
|
||||
ms = (item.get("manual_section") or "").upper()
|
||||
if ms in ("HARDWARE", "TIPS"):
|
||||
return 336.0
|
||||
tier = item.get("tier", "normal")
|
||||
if tier == "breaking":
|
||||
return CATEGORY_HALF_LIVES["breaking"]
|
||||
if tier == "update":
|
||||
return CATEGORY_HALF_LIVES["update"]
|
||||
return None
|
||||
|
||||
|
||||
def decay_index(items: list[dict], half_life_h: float = 18.0) -> list[dict]:
|
||||
"""Apply exponential time-decay to clickability."""
|
||||
cutoff = datetime.now(timezone.utc).timestamp() - 24 * 3600
|
||||
for it in items:
|
||||
age = _age_hours(it)
|
||||
it["age_hours"] = round(age, 1)
|
||||
base = it.get("clickability", 0) or 0
|
||||
hl = _get_half_life(it)
|
||||
if hl is None:
|
||||
hl = half_life_h
|
||||
k = math.log(2) / hl
|
||||
it["clickability_decayed"] = round(base * math.exp(-k * age), 4)
|
||||
it["effective_half_life"] = hl
|
||||
fs = it.get("first_seen") or ""
|
||||
try:
|
||||
ts = datetime.fromisoformat(fs.replace("Z", "+00:00")).timestamp()
|
||||
except ValueError:
|
||||
ts = 0
|
||||
it["fresh"] = ts >= cutoff
|
||||
return items
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Centralized configuration for the AI Research Oracle.
|
||||
|
||||
Single source of truth for all paths, defaults, and constants.
|
||||
"""
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# ── Paths ──────────────────────────────────────────────────────────────────
|
||||
ROOT = Path(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
DB_PATH = ROOT / "oracle.db"
|
||||
SCHEMA_PATH = ROOT / "schema.sql"
|
||||
|
||||
# ── Web output ─────────────────────────────────────────────────────────────
|
||||
WEBROOT = "/var/www/preprod2"
|
||||
FALLBACK_WEBROOT = ROOT / "site"
|
||||
SEEN_JSON = ROOT.parent / "ai-oracle-site" / "seen_urls.json"
|
||||
|
||||
# ── Pipeline defaults ──────────────────────────────────────────────────────
|
||||
ENABLED_SOURCES = ["github", "arxiv", "reddit", "hackernews", "huggingface", "rss"]
|
||||
DEFAULT_LIMIT = 20
|
||||
|
||||
# ── Source tiers (World Monitor pattern) ───────────────────────────────────
|
||||
# Tier 1: Primary trusted sources (official releases, peer-reviewed)
|
||||
# Tier 2: Secondary credible sources (curated communities, major outlets)
|
||||
# Tier 3: Tertiary noise sources (user-generated, unverified)
|
||||
SOURCE_TIERS = {
|
||||
"arxiv": {"tier": 1, "label": "PRIMARY", "description": "Peer-reviewed research"},
|
||||
"github": {"tier": 1, "label": "PRIMARY", "description": "Official code releases"},
|
||||
"huggingface": {"tier": 1, "label": "PRIMARY", "description": "Model registry"},
|
||||
"rss": {"tier": 2, "label": "SECONDARY", "description": "Curated tech media"},
|
||||
"hackernews": {"tier": 2, "label": "SECONDARY", "description": "Curated community"},
|
||||
"reddit": {"tier": 3, "label": "TERTIARY", "description": "User-generated discussion"},
|
||||
}
|
||||
|
||||
# Tier-based signal score bonus/penalty (applied to final_score)
|
||||
TIER_BONUS = {1: 0.05, 2: 0.0, 3: -0.05}
|
||||
|
||||
# Freshness SLA per tier (hours after which a source is flagged stale)
|
||||
FRESHNESS_SLA_H = {1: 48, 2: 24, 3: 12}
|
||||
|
||||
# ── Composite verdict thresholds ───────────────────────────────────────────
|
||||
# PUBLISH: High score + fresh, goes to top
|
||||
# WATCH: Medium score, monitor for follow-ups
|
||||
# ARCHIVE: Low score or aged out, move to archive
|
||||
# DROP: Junk score, ignore
|
||||
VERDICT_THRESHOLDS = {
|
||||
"PUBLISH": {"min_score": 6.0, "max_age_h": 48},
|
||||
"WATCH": {"min_score": 4.0, "max_age_h": 168}, # 7 days
|
||||
"ARCHIVE": {"min_score": 2.0, "max_age_h": 720}, # 30 days
|
||||
"DROP": {"min_score": 0.0, "max_age_h": 999999}, # catch-all
|
||||
}
|
||||
|
||||
# ── Content hash dedup ─────────────────────────────────────────────────────
|
||||
CONTENT_HASH_PREFIX = "sha256"
|
||||
HASH_LENGTH = 16 # characters
|
||||
|
||||
# ── Clickability weights ───────────────────────────────────────────────────
|
||||
VEL_W = 0.50
|
||||
ENG_W = 0.50
|
||||
SIG_W = 0.0
|
||||
HALF_LIFE_H = 18.0
|
||||
|
||||
# ── Render ─────────────────────────────────────────────────────────────────
|
||||
TOP_N = 8
|
||||
|
||||
# ── Archive ────────────────────────────────────────────────────────────────
|
||||
ARCHIVE_DAYS = 30
|
||||
ARCHIVE_CAP = 5000
|
||||
+203
@@ -0,0 +1,203 @@
|
||||
"""Database operations for the AI Research Oracle.
|
||||
|
||||
Handles connections, schema initialization, and common queries.
|
||||
"""
|
||||
import os
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
|
||||
from oracle.config import DB_PATH, SCHEMA_PATH, SOURCE_TIERS
|
||||
|
||||
|
||||
def get_connection(db_path: Optional[str] = None) -> sqlite3.Connection:
|
||||
"""Open a database connection."""
|
||||
path = db_path or str(DB_PATH)
|
||||
conn = sqlite3.connect(path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
|
||||
def get_ro_connection(db_path: Optional[str] = None) -> sqlite3.Connection:
|
||||
"""Open a read-only database connection."""
|
||||
path = db_path or str(DB_PATH)
|
||||
return sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
||||
|
||||
|
||||
def init_db(db_path: Optional[str] = None, schema_path: Optional[str] = None) -> sqlite3.Connection:
|
||||
"""Initialize or open the database, applying schema if it exists.
|
||||
|
||||
Schema uses CREATE IF NOT EXISTS so repeated calls are idempotent.
|
||||
Also runs World Monitor migration columns (content_hash, verdict, freshness).
|
||||
"""
|
||||
conn = sqlite3.connect(db_path or str(DB_PATH))
|
||||
sp = schema_path or str(SCHEMA_PATH)
|
||||
if os.path.exists(sp):
|
||||
with open(sp) as f:
|
||||
conn.executescript(f.read())
|
||||
conn.commit()
|
||||
# World Monitor migration columns (idempotent)
|
||||
for col in [
|
||||
"content_hash TEXT DEFAULT ''",
|
||||
"verdict TEXT DEFAULT ''",
|
||||
"source_tier INTEGER DEFAULT 2",
|
||||
]:
|
||||
try:
|
||||
conn.execute(f"ALTER TABLE entries ADD COLUMN {col}")
|
||||
except sqlite3.OperationalError:
|
||||
pass # already exists
|
||||
conn.commit()
|
||||
return conn
|
||||
|
||||
|
||||
def migrate_world_monitor(conn: Optional[sqlite3.Connection] = None) -> dict:
|
||||
"""Apply World Monitor schema migrations + backfill.
|
||||
|
||||
Returns: {columns_added: int, hashes_backfilled: int, verdicts_set: int}
|
||||
"""
|
||||
from oracle.dedup import backfill_hashes, apply_verdicts, get_source_tier
|
||||
|
||||
c = conn or get_connection()
|
||||
cur = c.cursor()
|
||||
|
||||
# Check which columns already exist
|
||||
cur.execute("PRAGMA table_info(entries)")
|
||||
existing = {row[1] for row in cur.fetchall()}
|
||||
|
||||
columns_to_add = []
|
||||
if "content_hash" not in existing:
|
||||
columns_to_add.append("content_hash TEXT DEFAULT ''")
|
||||
if "verdict" not in existing:
|
||||
columns_to_add.append("verdict TEXT DEFAULT ''")
|
||||
if "source_tier" not in existing:
|
||||
columns_to_add.append("source_tier INTEGER DEFAULT 2")
|
||||
|
||||
added = 0
|
||||
for col_def in columns_to_add:
|
||||
try:
|
||||
cur.execute(f"ALTER TABLE entries ADD COLUMN {col_def}")
|
||||
added += 1
|
||||
except sqlite3.OperationalError:
|
||||
pass # race condition or already exists
|
||||
|
||||
c.commit()
|
||||
|
||||
# Backfill content hashes
|
||||
hashes = backfill_hashes(c)
|
||||
|
||||
# Backfill source tiers — reset first so the WHERE clause catches everything
|
||||
cur.execute("UPDATE entries SET source_tier = 0")
|
||||
c.commit()
|
||||
for source_name, tier_info in SOURCE_TIERS.items():
|
||||
cur.execute(
|
||||
"UPDATE entries SET source_tier = ? WHERE source = ?",
|
||||
(tier_info["tier"], source_name),
|
||||
)
|
||||
c.commit()
|
||||
|
||||
# Set verdicts
|
||||
verdicts = apply_verdicts(c)
|
||||
|
||||
return {"columns_added": added, "hashes_backfilled": hashes, "verdicts_set": verdicts}
|
||||
|
||||
|
||||
def get_stats(conn: sqlite3.Connection) -> dict:
|
||||
"""Return database statistics."""
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("SELECT COUNT(*) FROM entries")
|
||||
total = cur.fetchone()[0]
|
||||
|
||||
cur.execute("""
|
||||
SELECT source, COUNT(*) as cnt, ROUND(AVG(signal_score), 2) as avg_score,
|
||||
MIN(signal_score) as min_score, MAX(signal_score) as max_score
|
||||
FROM entries GROUP BY source
|
||||
""")
|
||||
sources = {r["source"]: dict(r) for r in cur.fetchall()}
|
||||
|
||||
cur.execute("SELECT COUNT(*) FROM entries WHERE summary IS NOT NULL")
|
||||
summarized = cur.fetchone()[0]
|
||||
|
||||
cur.execute("SELECT COUNT(*) FROM entries WHERE summary IS NULL")
|
||||
pending = cur.fetchone()[0]
|
||||
|
||||
# Bucket distribution
|
||||
try:
|
||||
cur.execute("""
|
||||
SELECT bucket, COUNT(*) as cnt
|
||||
FROM entries WHERE bucket IS NOT NULL
|
||||
GROUP BY bucket ORDER BY cnt DESC
|
||||
""")
|
||||
buckets = {r["bucket"]: r["cnt"] for r in cur.fetchall()}
|
||||
except Exception:
|
||||
buckets = {}
|
||||
|
||||
return {
|
||||
"total_entries": total,
|
||||
"sources": sources,
|
||||
"summarized": summarized,
|
||||
"pending_summary": pending,
|
||||
"buckets": buckets,
|
||||
}
|
||||
|
||||
|
||||
def query_top(conn: sqlite3.Connection, n: int = 10,
|
||||
source: Optional[str] = None,
|
||||
min_score: float = 0) -> list[dict]:
|
||||
"""Get top N entries by signal score."""
|
||||
cur = conn.cursor()
|
||||
where_parts = []
|
||||
params = []
|
||||
|
||||
if min_score > 0:
|
||||
where_parts.append("signal_score >= ?")
|
||||
params.append(min_score)
|
||||
if source:
|
||||
where_parts.append("source = ?")
|
||||
params.append(source)
|
||||
|
||||
where = (" AND " + " AND ".join(where_parts)) if where_parts else ""
|
||||
cur.execute(
|
||||
f"SELECT * FROM entries {where} ORDER BY signal_score DESC LIMIT ?",
|
||||
params + [n],
|
||||
)
|
||||
return [dict(r) for r in cur.fetchall()]
|
||||
|
||||
|
||||
def query_recent(conn: sqlite3.Connection, hours: int = 24) -> list[dict]:
|
||||
"""Get entries from the last N hours."""
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
cutoff = (datetime.now(timezone.utc) - timedelta(hours=hours)).strftime(
|
||||
"%Y-%m-%dT%H:%M:%SZ"
|
||||
)
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"SELECT * FROM entries WHERE first_seen >= ? ORDER BY first_seen DESC",
|
||||
(cutoff,),
|
||||
)
|
||||
return [dict(r) for r in cur.fetchall()]
|
||||
|
||||
|
||||
def query_search(conn: sqlite3.Connection, q: str, limit: int = 20) -> list[dict]:
|
||||
"""Search entries by title, summary, and key technical point."""
|
||||
cur = conn.cursor()
|
||||
pattern = f"%{q}%"
|
||||
cur.execute("""
|
||||
SELECT * FROM entries
|
||||
WHERE title LIKE ?
|
||||
OR json_extract(summary,'$.one_liner') LIKE ?
|
||||
OR json_extract(summary,'$.key_technical_point') LIKE ?
|
||||
ORDER BY signal_score DESC LIMIT ?
|
||||
""", (pattern, pattern, pattern, limit))
|
||||
return [dict(r) for r in cur.fetchall()]
|
||||
|
||||
|
||||
def query_by_tag(conn: sqlite3.Connection, tag: str, limit: int = 20) -> list[dict]:
|
||||
"""Get entries matching a category tag."""
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT * FROM entries
|
||||
WHERE json_extract(category_tags,'$') LIKE ?
|
||||
ORDER BY signal_score DESC LIMIT ?
|
||||
""", (f'%"{tag}"%', limit))
|
||||
return [dict(r) for r in cur.fetchall()]
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
"""Content-hash dedup and composite verdict engine.
|
||||
|
||||
World Monitor pattern: SHA-256 content hash for dedup, tier-weighted
|
||||
composite verdict (PUBLISH/WATCH/ARCHIVE/DROP) on top of final_score.
|
||||
"""
|
||||
import hashlib
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from oracle.config import (
|
||||
SOURCE_TIERS, TIER_BONUS, VERDICT_THRESHOLDS,
|
||||
HASH_LENGTH, CONTENT_HASH_PREFIX,
|
||||
)
|
||||
|
||||
|
||||
def content_hash(title: str, url: str = "", body: str = "") -> str:
|
||||
"""Deterministic content hash for dedup.
|
||||
|
||||
Normalizes whitespace, lowercases, strips HTML tags, then hashes.
|
||||
Returns hex[:HASH_LENGTH] for compact storage.
|
||||
"""
|
||||
text = f"{title}|{body[:500]}|{url}"
|
||||
text = re.sub(r'\s+', ' ', text).strip().lower()
|
||||
text = re.sub(r'<[^>]+>', '', text)
|
||||
raw = hashlib.sha256(text.encode()).hexdigest()
|
||||
return f"{CONTENT_HASH_PREFIX}:{raw[:HASH_LENGTH]}"
|
||||
|
||||
|
||||
def check_duplicate(conn, title: str, url: str = "", body: str = "", cutoff_days: int = 7) -> bool:
|
||||
"""Check if an entry with similar content_hash already exists within cutoff."""
|
||||
h = content_hash(title, url, body)
|
||||
now = datetime.now(timezone.utc)
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"SELECT COUNT(*) FROM entries WHERE content_hash = ? AND first_seen > ?",
|
||||
(h, (now.timestamp() - cutoff_days * 86400)),
|
||||
)
|
||||
count = cur.fetchone()[0]
|
||||
return count > 0
|
||||
|
||||
|
||||
def get_source_tier(source: str) -> dict:
|
||||
"""Return tier info for a source. Defaults to tier 2."""
|
||||
return SOURCE_TIERS.get(source, {"tier": 2, "label": "SECONDARY", "description": "Unknown source"})
|
||||
|
||||
|
||||
def tier_adjusted_score(base_score: float, source: str) -> float:
|
||||
"""Apply tier bonus/penalty to a base signal score."""
|
||||
tier_info = get_source_tier(source)
|
||||
tier_num = tier_info["tier"]
|
||||
bonus = TIER_BONUS.get(tier_num, 0.0)
|
||||
return round(base_score + bonus, 3)
|
||||
|
||||
|
||||
def compute_verdict(score: float, age_hours: float) -> str:
|
||||
"""Compute composite verdict from score + age.
|
||||
|
||||
Uses signal_score (0-10 scale). PUBLISH > WATCH > ARCHIVE > DROP.
|
||||
"""
|
||||
for verdict, thresholds in VERDICT_THRESHOLDS.items():
|
||||
if score >= thresholds["min_score"] and age_hours <= thresholds["max_age_h"]:
|
||||
return verdict
|
||||
return "DROP"
|
||||
|
||||
|
||||
def age_hours(first_seen_iso: str) -> float:
|
||||
"""Return age in hours from ISO timestamp."""
|
||||
try:
|
||||
ts = first_seen_iso.replace("Z", "+00:00")
|
||||
first = datetime.fromisoformat(ts)
|
||||
now = datetime.now(timezone.utc)
|
||||
return max(0, (now - first).total_seconds() / 3600)
|
||||
except (ValueError, AttributeError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def apply_verdicts(conn):
|
||||
"""Update verdict column for all entries that lack one.
|
||||
|
||||
Uses signal_score (0-10 scale) + first_seen age to compute verdict.
|
||||
"""
|
||||
cur = conn.cursor()
|
||||
# Check if verdict column exists
|
||||
cur.execute("PRAGMA table_info(entries)")
|
||||
columns = {row[1] for row in cur.fetchall()}
|
||||
if "verdict" not in columns:
|
||||
print(" [verdict] column not found, skipping apply")
|
||||
return 0
|
||||
|
||||
# Reset all verdicts so they get recalculated
|
||||
cur.execute("UPDATE entries SET verdict = ''")
|
||||
conn.commit()
|
||||
|
||||
# Fetch all entries with signal scores
|
||||
cur.execute("SELECT id, COALESCE(signal_score, 0), first_seen FROM entries")
|
||||
updated = 0
|
||||
for row in cur.fetchall():
|
||||
entry_id, signal_score, first_seen = row
|
||||
age = age_hours(first_seen)
|
||||
verdict = compute_verdict(signal_score, age)
|
||||
cur.execute("UPDATE entries SET verdict = ? WHERE id = ?", (verdict, entry_id))
|
||||
updated += 1
|
||||
|
||||
conn.commit()
|
||||
return updated
|
||||
|
||||
|
||||
def backfill_hashes(conn, batch_size: int = 500) -> int:
|
||||
"""Backfill content_hash for entries that lack one."""
|
||||
cur = conn.cursor()
|
||||
cur.execute("PRAGMA table_info(entries)")
|
||||
columns = {row[1] for row in cur.fetchall()}
|
||||
if "content_hash" not in columns:
|
||||
print(" [dedup] content_hash column not found, skipping backfill")
|
||||
return 0
|
||||
|
||||
updated = 0
|
||||
while True:
|
||||
cur.execute(
|
||||
"SELECT id, title, url, summary FROM entries "
|
||||
"WHERE content_hash IS NULL OR content_hash = '' "
|
||||
"LIMIT ?",
|
||||
(batch_size,),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
if not rows:
|
||||
break
|
||||
for entry_id, title, url, summary in rows:
|
||||
body = ""
|
||||
if summary:
|
||||
import json
|
||||
try:
|
||||
s = json.loads(summary)
|
||||
body = s.get("one_liner", "") + " " + s.get("key_points", "")
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
body = summary[:200]
|
||||
h = content_hash(title, url, body)
|
||||
cur.execute("UPDATE entries SET content_hash = ? WHERE id = ?", (h, entry_id))
|
||||
updated += 1
|
||||
conn.commit()
|
||||
return updated
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Recency guard — Athena "how old is this news?" gate.
|
||||
|
||||
Age is the dominant gate. TODAY's items are always eligible.
|
||||
Older items are eligible ONLY if never posted before.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from oracle.config import SEEN_JSON
|
||||
|
||||
ORACLE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
|
||||
def _parse(ts):
|
||||
if not ts:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _norm_url(u):
|
||||
if not u:
|
||||
return ""
|
||||
return u.split("?")[0].split("#")[0].rstrip("/").lower()
|
||||
|
||||
|
||||
def _norm_title(t):
|
||||
if not t:
|
||||
return ""
|
||||
t = t.lower()
|
||||
t = re.sub(r"[^a-z0-9 ]", " ", t)
|
||||
return re.sub(r"\s+", " ", t).strip()[:60]
|
||||
|
||||
|
||||
def load_posted(md_dir: str = ORACLE_DIR, seen_json: str = str(SEEN_JSON)):
|
||||
"""Return (md_urls:set, md_titles:set, seen_urls:set)."""
|
||||
md_urls, md_titles, seen_urls = set(), set(), set()
|
||||
|
||||
for fn in sorted(os.listdir(md_dir)):
|
||||
if re.match(r"athena_top.*\.md$", fn):
|
||||
try:
|
||||
txt = open(os.path.join(md_dir, fn), encoding="utf-8", errors="replace").read()
|
||||
except OSError:
|
||||
continue
|
||||
for m in re.findall(r"\]\((https?://[^)\s]+)\)", txt):
|
||||
nu = _norm_url(m)
|
||||
if nu:
|
||||
md_urls.add(nu)
|
||||
for t in re.findall(r"^\|\s*\d+\s*\|\s*(.+?)\s*\|", txt, re.M):
|
||||
nt = _norm_title(t)
|
||||
if nt:
|
||||
md_titles.add(nt)
|
||||
|
||||
if os.path.exists(seen_json):
|
||||
try:
|
||||
with open(seen_json, encoding="utf-8") as f:
|
||||
for u in json.load(f):
|
||||
nu = _norm_url(u)
|
||||
if nu:
|
||||
seen_urls.add(nu)
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
|
||||
return md_urls, md_titles, seen_urls
|
||||
|
||||
|
||||
def is_today(first_seen, now=None):
|
||||
now = now or datetime.now(timezone.utc)
|
||||
d = _parse(first_seen)
|
||||
return bool(d) and d.strftime("%Y-%m-%d") == now.strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def age_days(first_seen, now=None):
|
||||
now = now or datetime.now(timezone.utc)
|
||||
d = _parse(first_seen)
|
||||
if not d:
|
||||
return 9999.0
|
||||
return max((now - d).total_seconds() / 86400.0, 0.0)
|
||||
|
||||
|
||||
def day_bucket(first_seen, now=None):
|
||||
days = age_days(first_seen, now)
|
||||
if days < 1:
|
||||
return "today"
|
||||
if days < 2:
|
||||
return "yesterday"
|
||||
if days <= 6:
|
||||
return "this-week"
|
||||
return "older"
|
||||
|
||||
|
||||
def already_posted(url, title, first_seen, now=None,
|
||||
md_urls=None, md_titles=None, seen_urls=None):
|
||||
"""Age-aware dedup. Today's items are NEVER flagged."""
|
||||
if md_urls is None or md_titles is None or seen_urls is None:
|
||||
md_urls, md_titles, seen_urls = load_posted()
|
||||
if _norm_url(url) in md_urls:
|
||||
return True
|
||||
nt = _norm_title(title)
|
||||
if nt and nt in md_titles:
|
||||
return True
|
||||
if is_today(first_seen, now):
|
||||
return False
|
||||
if _norm_url(url) in seen_urls:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def recency_weight(first_seen, now=None, half_life_days=2.0):
|
||||
"""1.0 for today, decays ~halving every 2 days."""
|
||||
return 0.5 ** (age_days(first_seen, now) / half_life_days)
|
||||
|
||||
|
||||
def blend_score(item, now=None):
|
||||
"""clickability_decayed * recency_weight."""
|
||||
base = item.get("clickability_decayed", 0) or 0
|
||||
return base * recency_weight(item.get("first_seen"), now)
|
||||
|
||||
|
||||
def filter_fresh(items, now=None, recent_window_days=4.0):
|
||||
"""Split into (today_items, older_new_items, dropped_items)."""
|
||||
now = now or datetime.now(timezone.utc)
|
||||
md_urls, md_titles, seen_urls = load_posted()
|
||||
today_items, older_new, dropped = [], [], []
|
||||
for it in items:
|
||||
fs = it.get("first_seen")
|
||||
if is_today(fs, now):
|
||||
today_items.append(it)
|
||||
continue
|
||||
posted = already_posted(it.get("url"), it.get("title"), fs, now,
|
||||
md_urls, md_titles, seen_urls)
|
||||
if posted and age_days(fs, now) > recent_window_days:
|
||||
dropped.append(it)
|
||||
else:
|
||||
older_new.append(it)
|
||||
return today_items, older_new, dropped
|
||||
@@ -0,0 +1,359 @@
|
||||
"""Render Athena entries into a static news site.
|
||||
|
||||
Two-layer layout: Top News (fresh today) + aging Stack (everything else).
|
||||
Read-only against oracle.db. Designed for a 20-min cron run.
|
||||
|
||||
Variant-aware: render_variant_html() builds themed pages from pre-filtered items.
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
from collections import OrderedDict
|
||||
from typing import Optional
|
||||
|
||||
from oracle.clickability import fetch_items, compute_index, decay_index
|
||||
from oracle.config import DB_PATH, WEBROOT, FALLBACK_WEBROOT, HALF_LIFE_H, TOP_N
|
||||
|
||||
|
||||
def _clean_summary(raw):
|
||||
"""Extract the most readable field from summary JSON."""
|
||||
if not raw:
|
||||
return ""
|
||||
try:
|
||||
d = json.loads(raw)
|
||||
if isinstance(d, dict):
|
||||
for k in ("one_liner", "key_technical_point", "potential_use_case"):
|
||||
v = d.get(k)
|
||||
if isinstance(v, str) and v.strip():
|
||||
return re.sub(r"\\+|_|`", "", v).strip()
|
||||
except Exception:
|
||||
pass
|
||||
return re.sub(r"\\+|_|`", "", raw).strip()
|
||||
|
||||
|
||||
# ── Theme palette ──────────────────────────────────────────────────────────
|
||||
|
||||
THEMES = {
|
||||
"dark": {
|
||||
"--bg": "#0b0e14",
|
||||
"--card": "#141925",
|
||||
"--fg": "#e6e9ef",
|
||||
"--mut": "#8b93a7",
|
||||
"--border": "#1f2533",
|
||||
},
|
||||
"light": {
|
||||
"--bg": "#f8f9fa",
|
||||
"--card": "#ffffff",
|
||||
"--fg": "#1a1a2e",
|
||||
"--mut": "#6b7280",
|
||||
"--border": "#e5e7eb",
|
||||
},
|
||||
"midnight": {
|
||||
"--bg": "#0a0a1a",
|
||||
"--card": "#111128",
|
||||
"--fg": "#c8d0e0",
|
||||
"--mut": "#5a6480",
|
||||
"--border": "#1a1a3a",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _theme_css(display: dict) -> str:
|
||||
"""Generate CSS variables for a variant display config."""
|
||||
theme_name = display.get("theme", "dark")
|
||||
palette = THEMES.get(theme_name, THEMES["dark"])
|
||||
accent = display.get("accent", "#5b8cff")
|
||||
vars_list = ", ".join(f"{k}:{v}" for k, v in palette.items())
|
||||
return f":root {{ {vars_list}; --acc:{accent}; }}"
|
||||
|
||||
|
||||
def _fmt_time(first_seen):
|
||||
if not first_seen:
|
||||
return ""
|
||||
try:
|
||||
dt = datetime.datetime.strptime(first_seen, "%Y-%m-%dT%H:%M:%SZ")
|
||||
return dt.strftime("%H:%M")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _card(it, big=False):
|
||||
title = html.escape(it["title"] or "(untitled)")
|
||||
url = html.escape(it["url"] or "#")
|
||||
src = html.escape(it["source"])
|
||||
t = _fmt_time(it.get("first_seen"))
|
||||
summary_raw = _clean_summary(it.get("summary") or "")
|
||||
summary = html.escape(summary_raw[:200])
|
||||
cls = "card big" if big else "card"
|
||||
summary_html = ('<p class="summary">{0}</p>'.format(summary)) if (summary and big) else ""
|
||||
return f"""
|
||||
<article class="{cls}" data-src="{src}">
|
||||
<div class="meta"><span class="src">{src}</span>
|
||||
<span class="time">{t}</span>
|
||||
<span class="sig">sig {it.get('signal_score') or 0:.1f}</span>
|
||||
<span class="score">\U0001f525 {it['clickability_decayed']:.2f}</span></div>
|
||||
<h3><a href="{url}" target="_blank" rel="noopener">{title}</a></h3>
|
||||
{summary_html}
|
||||
</article>"""
|
||||
|
||||
|
||||
def build_html(items):
|
||||
"""Build the full HTML page from ranked items."""
|
||||
now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
||||
ranked = sorted(items, key=lambda x: x["clickability_decayed"], reverse=True)
|
||||
|
||||
fresh = [it for it in ranked if it.get("fresh")]
|
||||
top = fresh[:TOP_N]
|
||||
stack = [it for it in ranked if it not in top]
|
||||
|
||||
by_day = OrderedDict()
|
||||
for it in stack:
|
||||
day = (it.get("first_seen") or "")[:10] or "unknown"
|
||||
by_day.setdefault(day, []).append(it)
|
||||
|
||||
top_html = "".join(_card(it, big=True) for it in top)
|
||||
|
||||
stack_html = ""
|
||||
for day, rows in by_day.items():
|
||||
rows.sort(key=lambda x: x["clickability_decayed"], reverse=True)
|
||||
cards = "".join(_card(it) for it in rows)
|
||||
stack_html += f"""
|
||||
<h3 class="day">\U0001f4c5 {html.escape(day)}</h3>
|
||||
<div class="stack">{cards}</div>"""
|
||||
|
||||
return f"""<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Athena AI News — Ranked by Clickability</title>
|
||||
<style>
|
||||
:root {{ --bg:#0b0e14; --card:#141925; --fg:#e6e9ef; --mut:#8b93a7; --acc:#5b8cff; }}
|
||||
* {{ box-sizing:border-box; }}
|
||||
body {{ margin:0; background:var(--bg); color:var(--fg);
|
||||
font:15px/1.5 -apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif; }}
|
||||
header {{ padding:28px 20px 14px; border-bottom:1px solid #1f2533; text-align:center; }}
|
||||
header h1 {{ margin:0; font-size:28px; letter-spacing:.5px; }}
|
||||
header .sub {{ color:var(--mut); font-size:13px; margin-top:6px; }}
|
||||
main {{ max-width:1000px; margin:0 auto; padding:20px; }}
|
||||
h2.sech {{ font-size:18px; margin:26px 0 12px; border-left:3px solid var(--acc); padding-left:10px; }}
|
||||
.grid {{ display:grid; grid-template-columns:repeat(auto-fill,minmax(320px,1fr)); gap:14px; }}
|
||||
.card {{ background:var(--card); border:1px solid #1f2533; border-radius:12px; padding:16px; }}
|
||||
.card.big {{ grid-column:1/-1; }}
|
||||
.meta {{ display:flex; gap:10px; align-items:center; font-size:12px; color:var(--mut); }}
|
||||
.src {{ background:#1f2533; padding:2px 8px; border-radius:20px; text-transform:uppercase; }}
|
||||
.score {{ color:#ff9d5b; font-weight:600; margin-left:auto; }}
|
||||
.card h3 {{ font-size:16px; margin:10px 0 8px; line-height:1.35; }}
|
||||
.card.big h3 {{ font-size:20px; }}
|
||||
.card h3 a {{ color:var(--fg); text-decoration:none; }}
|
||||
.card h3 a:hover {{ color:var(--acc); }}
|
||||
.summary {{ color:var(--mut); font-size:13px; margin:0; }}
|
||||
.day {{ font-size:15px; color:var(--mut); margin:28px 0 10px; border-bottom:1px solid #1f2533; padding-bottom:6px; }}
|
||||
.stack {{ display:grid; grid-template-columns:repeat(auto-fill,minmax(320px,1fr)); gap:12px; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>Athena AI News</h1>
|
||||
<div class="sub">Auto-ranked by Clickability Index · decays with age so the stack flows top → bottom · generated {now} · {len(items)} stories</div>
|
||||
</header>
|
||||
<main>
|
||||
<h2 class="sech">\U0001f534 Top News</h2>
|
||||
<div class="grid">{top_html}</div>
|
||||
<h2 class="sech">\U0001f4f0 The Stack</h2>
|
||||
{stack_html}
|
||||
</main>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
def render(dry_run=False, webroot=None):
|
||||
"""Run the full render pipeline.
|
||||
|
||||
Returns (output_path, item_count).
|
||||
"""
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
items = fetch_items(conn)
|
||||
conn.close()
|
||||
|
||||
items = compute_index(items)
|
||||
items = decay_index(items, HALF_LIFE_H)
|
||||
page = build_html(items)
|
||||
|
||||
if dry_run:
|
||||
out = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "_preview.html")
|
||||
with open(out, "w") as f:
|
||||
f.write(page)
|
||||
fresh = [it for it in items if it.get("fresh")]
|
||||
top = sorted(fresh, key=lambda x: x["clickability_decayed"], reverse=True)[:TOP_N]
|
||||
print(f"[dry-run] wrote {out} ({len(items)} items, {len(fresh)} fresh today)")
|
||||
print(f"TOP {TOP_N} FRESH (today only) by decayed clickability:")
|
||||
for i, it in enumerate(top, 1):
|
||||
print(f" {i}. [{it['clickability_decayed']:.2f} | age {it['age_hours']:.0f}h] {it['source']:10} {it['title'][:55]}")
|
||||
return out, len(items)
|
||||
|
||||
target = webroot or (WEBROOT if os.path.isdir(WEBROOT) else str(FALLBACK_WEBROOT))
|
||||
os.makedirs(target, exist_ok=True)
|
||||
|
||||
with open(os.path.join(target, "index.html"), "w") as f:
|
||||
f.write(page)
|
||||
|
||||
with open(os.path.join(target, "feed.json"), "w") as f:
|
||||
json.dump([
|
||||
{"title": i["title"], "url": i["url"], "source": i["source"],
|
||||
"clickability_decayed": i["clickability_decayed"], "age_hours": i["age_hours"],
|
||||
"first_seen": i.get("first_seen")}
|
||||
for i in sorted(items, key=lambda x: x["clickability_decayed"], reverse=True)
|
||||
], f, indent=2)
|
||||
|
||||
where = "WEBROOT" if target == WEBROOT else "fallback(~oracle/site)"
|
||||
print(f"[render] wrote {target}/index.html ({len(items)} items) -> {where}")
|
||||
return os.path.join(target, "index.html"), len(items)
|
||||
|
||||
|
||||
# ── Variant rendering ──────────────────────────────────────────────────────
|
||||
|
||||
def _variant_card(it: dict, display: dict, big: bool = False) -> str:
|
||||
"""Build an HTML card for a variant edition."""
|
||||
title = html.escape(it.get("title") or "(untitled)")
|
||||
url = html.escape(it.get("url") or "#")
|
||||
src = html.escape(it.get("source", ""))
|
||||
t = _fmt_time(it.get("first_seen"))
|
||||
summary_raw = _clean_summary(it.get("summary") or "")
|
||||
summary = html.escape(summary_raw[:200])
|
||||
|
||||
score = it.get("signal_score")
|
||||
tier = it.get("source_tier")
|
||||
verdict = it.get("verdict", "")
|
||||
|
||||
cls = "card big" if big else "card"
|
||||
summary_html = ('<p class="summary">{}</p>'.format(summary)) if (summary and display.get("show_summary") and big) else ""
|
||||
|
||||
# Build meta badges
|
||||
badges = f'<span class="src">{src}</span>'
|
||||
if t:
|
||||
badges += f'<span class="time">{t}</span>'
|
||||
if display.get("show_score") and score is not None:
|
||||
badges += f'<span class="sig">sig {float(score):.1f}</span>'
|
||||
if display.get("show_tier") and tier is not None:
|
||||
badges += f'<span class="tier">T{tier}</span>'
|
||||
if display.get("show_verdict") and verdict:
|
||||
badges += f'<span class="verdict verdict-{verdict.lower()}">{verdict}</span>'
|
||||
|
||||
return f"""
|
||||
<article class="{cls}">
|
||||
<div class="meta">{badges}</div>
|
||||
<h3><a href="{url}" target="_blank" rel="noopener">{title}</a></h3>
|
||||
{summary_html}
|
||||
</article>"""
|
||||
|
||||
|
||||
def render_variant_html(items: list[dict], variant: dict) -> str:
|
||||
"""Build the full HTML page for a variant edition."""
|
||||
display = variant.get("display", {})
|
||||
name = variant.get("name", "Athena")
|
||||
desc = variant.get("description", "")
|
||||
logo = display.get("logo", "🏛️")
|
||||
top_n = display.get("top_n", 8)
|
||||
half_life = variant.get("ranking", {}).get("half_life_h", 18)
|
||||
|
||||
now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
||||
|
||||
top = items[:top_n]
|
||||
stack = items[top_n:]
|
||||
|
||||
# Group stack by day
|
||||
by_day = OrderedDict()
|
||||
for it in stack:
|
||||
day = (it.get("first_seen") or "")[:10] or "unknown"
|
||||
by_day.setdefault(day, []).append(it)
|
||||
|
||||
top_html = "\n".join(_variant_card(it, display, big=True) for it in top)
|
||||
stack_html = ""
|
||||
for day, rows in by_day.items():
|
||||
cards = "".join(_variant_card(it, display) for it in rows)
|
||||
stack_html += f"""
|
||||
<h3 class="day">🗒️ {html.escape(day)}</h3>
|
||||
<div class="stack">{cards}</div>"""
|
||||
|
||||
theme = _theme_css(display)
|
||||
|
||||
return f"""<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{html.escape(name)}</title>
|
||||
<style>
|
||||
{theme}
|
||||
* {{ box-sizing:border-box; }}
|
||||
body {{ margin:0; background:var(--bg); color:var(--fg);
|
||||
font:15px/1.5 -apple-system,Segoe UI,Roboto,Helvetica,Arial,sans-serif; }}
|
||||
header {{ padding:28px 20px 14px; border-bottom:1px solid var(--border); text-align:center; }}
|
||||
header h1 {{ margin:0; font-size:28px; letter-spacing:.5px; }}
|
||||
header .sub {{ color:var(--mut); font-size:13px; margin-top:6px; }}
|
||||
main {{ max-width:1000px; margin:0 auto; padding:20px; }}
|
||||
h2.sech {{ font-size:18px; margin:26px 0 12px; border-left:3px solid var(--acc); padding-left:10px; }}
|
||||
.grid {{ display:grid; grid-template-columns:repeat(auto-fill,minmax(320px,1fr)); gap:14px; }}
|
||||
.card {{ background:var(--card); border:1px solid var(--border); border-radius:12px; padding:16px; }}
|
||||
.card.big {{ grid-column:1/-1; }}
|
||||
.meta {{ display:flex; gap:10px; align-items:center; font-size:12px; color:var(--mut); flex-wrap:wrap; }}
|
||||
.src {{ background:var(--border); padding:2px 8px; border-radius:20px; text-transform:uppercase; }}
|
||||
.sig {{ color:var(--acc); font-weight:600; }}
|
||||
.tier {{ color:var(--mut); }}
|
||||
.verdict {{ padding:2px 6px; border-radius:4px; font-weight:600; text-transform:uppercase; font-size:10px; }}
|
||||
.verdict-publish {{ background:#065f46; color:#a7f3d0; }}
|
||||
.verdict-watch {{ background:#1e3a5f; color:#93c5fd; }}
|
||||
.verdict-archive {{ background:#4a3b1f; color:#fcd34d; }}
|
||||
.verdict-drop {{ background:#4a1f1f; color:#fca5a5; }}
|
||||
.card h3 {{ font-size:16px; margin:10px 0 8px; line-height:1.35; }}
|
||||
.card.big h3 {{ font-size:20px; }}
|
||||
.card h3 a {{ color:var(--fg); text-decoration:none; }}
|
||||
.card h3 a:hover {{ color:var(--acc); }}
|
||||
.summary {{ color:var(--mut); font-size:13px; margin:0; }}
|
||||
.day {{ font-size:15px; color:var(--mut); margin:28px 0 10px; border-bottom:1px solid var(--border); padding-bottom:6px; }}
|
||||
.stack {{ display:grid; grid-template-columns:repeat(auto-fill,minmax(320px,1fr)); gap:12px; }}
|
||||
footer {{ text-align:center; padding:20px; color:var(--mut); font-size:12px; border-top:1px solid var(--border); margin-top:40px; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>{logo} {html.escape(name)}</h1>
|
||||
<div class="sub">{html.escape(desc)} · generated {now} · {len(items)} stories</div>
|
||||
</header>
|
||||
<main>
|
||||
<h2 class="sech">🔥 Top News</h2>
|
||||
<div class="grid">{top_html}</div>
|
||||
{('<h2 class="sech">🗄️ The Stack</h2>' + stack_html) if stack else ''}
|
||||
</main>
|
||||
<footer>
|
||||
Athena AI Research Oracle · <a href="feed.json">feed.json</a>
|
||||
</footer>
|
||||
</body>
|
||||
</html>"""
|
||||
|
||||
|
||||
def render_variant_json(items: list[dict], variant: dict, path: str) -> None:
|
||||
"""Write a JSON feed for a variant edition."""
|
||||
display = variant.get("display", {})
|
||||
feed = []
|
||||
for i in items:
|
||||
entry = {
|
||||
"title": i.get("title", ""),
|
||||
"url": i.get("url", ""),
|
||||
"source": i.get("source", ""),
|
||||
"signal_score": float(i.get("signal_score") or 0),
|
||||
"first_seen": i.get("first_seen", ""),
|
||||
}
|
||||
if display.get("show_tier"):
|
||||
entry["source_tier"] = i.get("source_tier")
|
||||
if display.get("show_verdict"):
|
||||
entry["verdict"] = i.get("verdict", "")
|
||||
feed.append(entry)
|
||||
|
||||
with open(path, "w") as f:
|
||||
json.dump(feed, f, indent=2)
|
||||
@@ -0,0 +1,332 @@
|
||||
"""Athena scoring engine — Sprint 1: Pure Rule-Based Bucket Classifier + Scorer.
|
||||
|
||||
DESIGN CONSTRAINT (founder directive, 2026-07-15):
|
||||
Pure rules only. No embeddings, no semantic similarity, no LLM classification.
|
||||
|
||||
Pipeline position:
|
||||
ingestion/dedup -> [attach_scoring] -> rendering
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
|
||||
from oracle.config import DB_PATH
|
||||
|
||||
# ── Score weights ──────────────────────────────────────────────────────────
|
||||
WEIGHTS = {
|
||||
"shipping": 0.20,
|
||||
"utility": 0.20,
|
||||
"replication": 0.25,
|
||||
"enthusiast": 0.20,
|
||||
"novelty": 0.15,
|
||||
}
|
||||
HYPE_CAP = 0.45
|
||||
|
||||
# ── Keyword sets ───────────────────────────────────────────────────────────
|
||||
KW_SHIPPING = [
|
||||
"released", "launch", "v1.0", "v2.0", "v3.0", "shipping", "now available",
|
||||
"open source", "open-source", "open weights", "weights released", "live now",
|
||||
"beta", "public beta", "ga release", "general availability", "ships", "deployed",
|
||||
"production", "now in", "available today", "download", "gradio", "demo", "playground",
|
||||
]
|
||||
KW_LOCAL_AI = [
|
||||
"local llm", "local model", "local ai", "run locally", "run it locally", "on-device",
|
||||
"on device", "ollama", "llama.cpp", "llamacpp", "gguf", "ggml", "lm studio",
|
||||
"consumer hardware", "consumer gpu", "rtx", "your own gpu", "offline", "private ai",
|
||||
"local-only", "self-host", "self-hosted", "home server", "edge device", "edge inference",
|
||||
"quantized", "quantization", "q4", "q8", "int4", "fp16", "fine-tune at home",
|
||||
"train at home", "local inference", "local deployment", "no api", "no cloud",
|
||||
]
|
||||
KW_PROBLEM_SOLVED = [
|
||||
"how to", "how i", "solved", "fix", "fixed", "workaround", "benchmark", "improves",
|
||||
"improved", "speedup", "speed-up", "reduces", "reduce", "cut", "cuts", "boost",
|
||||
"optimize", "optimized", "optimisation", "faster", "3x", "10x", "2x", "latency",
|
||||
"throughput", "roi", "cost", "cheaper", "save", "saves", "eliminate", "eliminated",
|
||||
"from 117s to 30s", "p95", "memory usage", "vram", "token cost", "bottleneck",
|
||||
"case study", "results", "we measured", "we tested", "showdown", "comparison",
|
||||
]
|
||||
KW_MODEL_RELEASE = [
|
||||
"releases", "released", "unveils", "introduces", "new model", "new flagship",
|
||||
"gpt-", "claude", "gemini", "llama", "mistral", "qwen", "deepseek", "grok",
|
||||
"phi-", "command-r", "api access", "weights", "open model", "open-models",
|
||||
"frontier", "checkpoint", "fine-tune", "finetune", "rl-trained", "rl train",
|
||||
"trained", "post-training", "post training", "distilled", "distillation",
|
||||
]
|
||||
KW_RESEARCH = [
|
||||
"paper", "arxiv", "preprint", "study", "research", "we propose", "we present",
|
||||
"we introduce", "we show", "method", "framework", "theorem", "analysis of",
|
||||
"survey", "benchmark", "dataset", "neural", "transformer", "diffusion",
|
||||
"gradient", "ablation", "we find", "our approach", "novel", "state-of-the-art",
|
||||
"sota", "cs.lg", "cs.cl", "cs.cv", "cs.ai",
|
||||
]
|
||||
KW_BUSINESS = [
|
||||
"raises", "raised", "$", "valuation", "series a", "series b", "funding", "round",
|
||||
"ipo", "acquisition", "acquires", "merger", "deal", "revenue", "layoff", "hiring",
|
||||
"partnership", "invests", "investment", "market", "vc", "compute deal",
|
||||
"billion", "million", "forecast", "miss", "earnings", "stock",
|
||||
]
|
||||
KW_INFRA = [
|
||||
"gpu", "tpu", "data center", "datacenter", "data centre", "cluster", "cuda",
|
||||
"rocm", "vllm", "tensorrt", "inference server", "serving", "kubernetes", "docker",
|
||||
"pipeline", "mlops", "ci/cd", "rag", "vector db", "vector database", "agent",
|
||||
"agents", "orchestration", "observability", "evaluation", "eval", "guardrail",
|
||||
"safety", "red team", "jailbreak", "prompt injection", "fine-tuning stack",
|
||||
]
|
||||
KW_CULTURE = [
|
||||
"says", "argues", "opinion", "essay", "think", "thinks", "the real", "why we",
|
||||
"the future of", "dystopia", "utopia", "philosophy", "ethics", "regulation",
|
||||
"policy", "ban", "lawsuit", "eu", "senate", "congress", "interview", "podcast",
|
||||
"controversy", "controversial", "debate", "critic", "criticism",
|
||||
"creepy", "creeping", "not sexy", "vibe", "hot take", "unpopular",
|
||||
]
|
||||
|
||||
BUCKETS = {
|
||||
"SHIPPING": {"kw": KW_SHIPPING, "source_whitelist": None, "order": 0},
|
||||
"LOCAL AI": {"kw": KW_LOCAL_AI, "source_whitelist": None, "order": 1},
|
||||
"PROBLEM SOLVED": {"kw": KW_PROBLEM_SOLVED, "source_whitelist": None, "order": 2},
|
||||
"MODEL RELEASE": {"kw": KW_MODEL_RELEASE, "source_whitelist": None, "order": 3},
|
||||
"RESEARCH": {"kw": KW_RESEARCH, "source_whitelist": ["arxiv"], "order": 4},
|
||||
"BUSINESS": {"kw": KW_BUSINESS, "source_whitelist": None, "order": 5},
|
||||
"INFRASTRUCTURE": {"kw": KW_INFRA, "source_whitelist": None, "order": 6},
|
||||
"CULTURE": {"kw": KW_CULTURE, "source_whitelist": None, "order": 7},
|
||||
}
|
||||
BUCKET_ORDER = sorted(BUCKETS.keys(), key=lambda b: BUCKETS[b]["order"])
|
||||
|
||||
HYPE_TERMS = [
|
||||
"revolutionary", "game-changing", "game changer", "breakthrough", "mind-blowing",
|
||||
"insane", "crazy", "unbelievable", "shocking", "the future is here", "omg",
|
||||
"you won't believe", "secret", "they don't want you to know", "leaked", "viral",
|
||||
"hype", "buzzword", "disrupt", "disrupting everything", "ai will replace",
|
||||
"will change everything", "paradigm shift", "godlike", "magic", "miracle",
|
||||
]
|
||||
|
||||
ENTHUSIAST_SIGNALS = [
|
||||
"github", "repo", "repository", "self-host", "local", "ollama", "llamacpp",
|
||||
"hugging face", "huggingface", "colab", "notebook", "pip install", "docker",
|
||||
"cli", "open source", "open-source", "diy", "build your own", "tutorial",
|
||||
"how to", "implementation", "agent", "agents", "fine-tune", "finetune",
|
||||
"quantiz", "vllm", "rtx", "gpu", "consumer", "homelab", "self-hosted",
|
||||
"machine-learning", "machine learning", "deep learning", "python", "rust",
|
||||
"benchmark", "reproduc", "weights", "gguf",
|
||||
]
|
||||
SOURCE_ENTHUSIAST_BONUS = {
|
||||
"github": 0.20, "huggingface": 0.20, "arxiv": 0.10,
|
||||
"hackernews": 0.10, "reddit": 0.05, "rss": 0.0,
|
||||
}
|
||||
|
||||
NEW_COLUMNS = [
|
||||
"bucket TEXT",
|
||||
"shipping_score REAL DEFAULT 0",
|
||||
"utility_score REAL DEFAULT 0",
|
||||
"replication_score REAL DEFAULT 0",
|
||||
"enthusiast_score REAL DEFAULT 0",
|
||||
"novelty_score REAL DEFAULT 0",
|
||||
"hype_penalty REAL DEFAULT 0",
|
||||
"final_score REAL DEFAULT 0",
|
||||
"actionability_score REAL DEFAULT 0",
|
||||
"narrative_id TEXT",
|
||||
"topic_id TEXT",
|
||||
"relation_json TEXT",
|
||||
]
|
||||
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────────
|
||||
def _norm(text):
|
||||
if not text:
|
||||
return ""
|
||||
if isinstance(text, bytes):
|
||||
text = text.decode("utf-8", "replace")
|
||||
return " " + re.sub(r"\s+", " ", text.lower()) + " "
|
||||
|
||||
|
||||
def _summary_text(raw):
|
||||
if not raw:
|
||||
return ""
|
||||
try:
|
||||
d = json.loads(raw)
|
||||
if isinstance(d, dict):
|
||||
return " ".join(str(v) for v in d.values() if isinstance(v, str))
|
||||
except Exception:
|
||||
pass
|
||||
return raw
|
||||
|
||||
|
||||
# ── Classification ─────────────────────────────────────────────────────────
|
||||
def classify(entry: dict) -> tuple[str, list[str]]:
|
||||
"""Pure rule classification.
|
||||
|
||||
Returns (bucket, matched_list) where matched_list is human-readable proof.
|
||||
"""
|
||||
title = _norm(entry.get("title") or "")
|
||||
summary = _norm(_summary_text(entry.get("summary")))
|
||||
tags_raw = entry.get("category_tags") or ""
|
||||
try:
|
||||
tags = " ".join(json.loads(tags_raw)) if tags_raw else ""
|
||||
except Exception:
|
||||
tags = tags_raw
|
||||
tags = _norm(tags)
|
||||
source = (entry.get("source") or "").lower()
|
||||
haystack = title + " " + summary + " " + tags
|
||||
|
||||
matched = [f"source={source}"]
|
||||
best_bucket = "UNCATEGORIZED"
|
||||
best_hits = 0
|
||||
|
||||
for bucket in BUCKET_ORDER:
|
||||
spec = BUCKETS[bucket]
|
||||
whitelist = spec["source_whitelist"]
|
||||
if whitelist and source not in whitelist:
|
||||
continue
|
||||
hits = []
|
||||
for kw in spec["kw"]:
|
||||
if f" {kw.lower()} " in haystack:
|
||||
hits.append(kw)
|
||||
if hits:
|
||||
matched.extend(f"kw:{h}" for h in hits[:8])
|
||||
if len(hits) > best_hits:
|
||||
best_hits = len(hits)
|
||||
best_bucket = bucket
|
||||
|
||||
if best_bucket == "UNCATEGORIZED":
|
||||
matched.append("(no rule fired)")
|
||||
|
||||
return best_bucket, matched
|
||||
|
||||
|
||||
# ── Scoring ────────────────────────────────────────────────────────────────
|
||||
def score_entry(bucket: str, matched: list, entry: dict) -> dict:
|
||||
"""Return dict of component scores (0..1) + final (0..1)."""
|
||||
source = (entry.get("source") or "").lower()
|
||||
haystack = _norm(entry.get("title") or "") + " " + _norm(_summary_text(entry.get("summary")))
|
||||
|
||||
tags_raw = entry.get("category_tags") or ""
|
||||
try:
|
||||
tags = " ".join(json.loads(tags_raw)) if tags_raw else ""
|
||||
except Exception:
|
||||
tags = tags_raw
|
||||
haystack += _norm(tags)
|
||||
|
||||
# Enthusiast score
|
||||
ent_hits = sum(1 for s in ENTHUSIAST_SIGNALS if f" {s} " in haystack)
|
||||
enthusiast = min(ent_hits / 5.0 + SOURCE_ENTHUSIAST_BONUS.get(source, 0.0), 1.0)
|
||||
|
||||
# Shipping score
|
||||
ship_kw = [k for k in KW_SHIPPING if f" {k} " in haystack]
|
||||
shipping = 0.0
|
||||
if bucket == "SHIPPING":
|
||||
shipping = 0.9
|
||||
elif ship_kw:
|
||||
shipping = min(0.4 + 0.1 * len(ship_kw), 0.8)
|
||||
if source in ("github", "huggingface"):
|
||||
shipping = max(shipping, 0.7)
|
||||
|
||||
# Utility score
|
||||
util_kw = [k for k in KW_PROBLEM_SOLVED if f" {k} " in haystack]
|
||||
utility = 0.0
|
||||
if bucket == "PROBLEM SOLVED":
|
||||
utility = 0.85
|
||||
elif util_kw:
|
||||
utility = min(0.4 + 0.1 * len(util_kw), 0.8)
|
||||
if any(s in haystack for s in [" github ", " huggingface ", " demo "]):
|
||||
utility = max(utility, 0.6)
|
||||
|
||||
# Replication score
|
||||
repl_kw = [k for k in KW_LOCAL_AI if f" {k} " in haystack]
|
||||
replication = 0.0
|
||||
if bucket == "LOCAL AI":
|
||||
replication = 1.0
|
||||
elif repl_kw:
|
||||
replication = min(0.5 + 0.1 * len(repl_kw), 0.9)
|
||||
if source in ("github", "huggingface"):
|
||||
replication = max(replication, 0.7)
|
||||
if any(s in haystack for s in [" open source ", " open-source ", " weights "]):
|
||||
replication = max(replication, 0.6)
|
||||
|
||||
# Novelty score
|
||||
novelty = 0.0
|
||||
if bucket in ("RESEARCH", "MODEL RELEASE"):
|
||||
novelty = 0.6
|
||||
nov_kw = ["new", "novel", "first", "breakthrough-method", "we propose",
|
||||
"we introduce", "we present", "state-of-the-art", "sota", "unveils"]
|
||||
if any(f" {k} " in haystack for k in nov_kw):
|
||||
novelty = min(novelty + 0.2, 0.9)
|
||||
if bucket == "CULTURE":
|
||||
novelty = min(novelty, 0.3)
|
||||
|
||||
# Hype penalty
|
||||
hype_penalty = min(0.1 * sum(1 for t in HYPE_TERMS if f" {t} " in haystack), 0.6)
|
||||
|
||||
# Final score
|
||||
raw = (
|
||||
WEIGHTS["shipping"] * shipping
|
||||
+ WEIGHTS["utility"] * utility
|
||||
+ WEIGHTS["replication"] * replication
|
||||
+ WEIGHTS["enthusiast"] * enthusiast
|
||||
+ WEIGHTS["novelty"] * novelty
|
||||
)
|
||||
final = min(max(raw - hype_penalty, 0.0), 1.0)
|
||||
|
||||
return {
|
||||
"shipping_score": round(shipping, 3),
|
||||
"utility_score": round(utility, 3),
|
||||
"replication_score": round(replication, 3),
|
||||
"enthusiast_score": round(enthusiast, 3),
|
||||
"novelty_score": round(novelty, 3),
|
||||
"hype_penalty": round(hype_penalty, 3),
|
||||
"final_score": round(final, 4),
|
||||
}
|
||||
|
||||
|
||||
# ── DB Operations ──────────────────────────────────────────────────────────
|
||||
def migrate(db_path: Optional[str] = None) -> list[str]:
|
||||
"""Idempotent schema migration — only adds missing columns."""
|
||||
conn = sqlite3.connect(db_path or str(DB_PATH))
|
||||
cur = conn.cursor()
|
||||
cur.execute("PRAGMA table_info(entries)")
|
||||
existing = {row[1] for row in cur.fetchall()}
|
||||
added = []
|
||||
for col in NEW_COLUMNS:
|
||||
name = col.split(" ")[0]
|
||||
if name not in existing:
|
||||
cur.execute(f"ALTER TABLE entries ADD COLUMN {col}")
|
||||
added.append(name)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"[migrate] added columns: {', '.join(added) if added else 'none (already present)'}")
|
||||
return added
|
||||
|
||||
|
||||
def fetch_unscored(conn: sqlite3.Connection) -> list[dict]:
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT id, source, source_id, url, title, summary, category_tags, raw_metadata
|
||||
FROM entries WHERE bucket IS NULL OR bucket = ''
|
||||
""")
|
||||
cols = ["id", "source", "source_id", "url", "title", "summary", "category_tags", "raw_metadata"]
|
||||
return [dict(zip(cols, row)) for row in cur.fetchall()]
|
||||
|
||||
|
||||
def attach_scoring(db_path: Optional[str] = None, dry_run: bool = False) -> None:
|
||||
"""Score every unscored entry. Call after ingestion/dedup, before render."""
|
||||
conn = sqlite3.connect(db_path or str(DB_PATH))
|
||||
rows = fetch_unscored(conn)
|
||||
print(f"[attach] scoring {len(rows)} unscored entries")
|
||||
for e in rows:
|
||||
bucket, matched = classify(e)
|
||||
scores = score_entry(bucket, matched, e)
|
||||
if not dry_run:
|
||||
conn.execute(
|
||||
"""UPDATE entries SET bucket=?, shipping_score=?, utility_score=?,
|
||||
replication_score=?, enthusiast_score=?, novelty_score=?,
|
||||
hype_penalty=?, final_score=?, actionability_score=?,
|
||||
narrative_id=?, topic_id=?, relation_json=? WHERE id=?""",
|
||||
(bucket, scores["shipping_score"], scores["utility_score"],
|
||||
scores["replication_score"], scores["enthusiast_score"],
|
||||
scores["novelty_score"], scores["hype_penalty"], scores["final_score"],
|
||||
0.0, None, None, json.dumps({"matched_rules": matched}), e["id"]),
|
||||
)
|
||||
if not dry_run:
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print("[attach] done.")
|
||||
@@ -0,0 +1,301 @@
|
||||
"""Summarization engine for the AI Research Oracle.
|
||||
|
||||
Generates structured summaries for entries where summary IS NULL.
|
||||
Uses source-specific extraction logic (no LLM required).
|
||||
|
||||
Output schema: {one_liner, key_technical_point, potential_use_case, confidence}
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
|
||||
from oracle.config import DB_PATH
|
||||
|
||||
|
||||
def extract_github_summary(title: str, content: str) -> dict:
|
||||
"""Extract summary from GitHub README content."""
|
||||
text = re.sub(r'<p[^>]*>', '\n', content)
|
||||
text = re.sub(r'</p>', '\n', content)
|
||||
text = re.sub(r'<h[1-6][^>]*>', '\n## ', text)
|
||||
text = re.sub(r'</h[1-6]>', '\n', text)
|
||||
text = re.sub(r'<[^>]+>', '', text)
|
||||
text = re.sub(r'&', '&', text)
|
||||
text = re.sub(r'—', '—', text)
|
||||
text = re.sub(r''', "'", text)
|
||||
text = re.sub(r'·', '·', text)
|
||||
text = re.sub(r'```[\s\S]*?```', '', text)
|
||||
text = re.sub(r'\n\s*\n+', '\n\n', text)
|
||||
text = text.strip()
|
||||
|
||||
source_confidence = "low"
|
||||
if len(text) > 2000:
|
||||
source_confidence = "high"
|
||||
elif len(text) > 500:
|
||||
source_confidence = "medium"
|
||||
|
||||
one_liner = _find_project_description(text, title) or title[:200]
|
||||
key_tech = _extract_technical_point(text, source_confidence)
|
||||
use_case = _extract_use_case(text, title)
|
||||
confidence = _assess_extraction_quality(one_liner, key_tech, use_case, source_confidence)
|
||||
|
||||
if _is_security_tooling(title, one_liner, key_tech):
|
||||
use_case = use_case + " [security:dual-use]"
|
||||
|
||||
return {
|
||||
"one_liner": one_liner[:200],
|
||||
"key_technical_point": key_tech[:200],
|
||||
"potential_use_case": use_case[:200],
|
||||
"confidence": confidence,
|
||||
}
|
||||
|
||||
|
||||
def extract_arxiv_summary(title: str, content: str) -> dict:
|
||||
"""Extract summary from arXiv abstract."""
|
||||
text = re.sub(r'<[^>]+>', ' ', content)
|
||||
text = re.sub(r'\s+', ' ', text).strip()
|
||||
|
||||
confidence = "high" if len(text) > 300 else "medium"
|
||||
one_liner = _find_contribution(text) or f"This paper presents {title.lower()}"
|
||||
key_tech = _extract_method(text)
|
||||
use_case = _extract_application(text)
|
||||
|
||||
return {
|
||||
"one_liner": one_liner[:200],
|
||||
"key_technical_point": key_tech[:200],
|
||||
"potential_use_case": use_case[:200],
|
||||
"confidence": confidence,
|
||||
}
|
||||
|
||||
|
||||
def extract_reddit_summary(title: str, content: str) -> dict:
|
||||
"""Extract summary from Reddit post."""
|
||||
text = re.sub(r'<[^>]+>', ' ', content)
|
||||
text = re.sub(r'\s+', ' ', text).strip()
|
||||
|
||||
if len(text) > 500:
|
||||
confidence = "high"
|
||||
elif len(text) > 100:
|
||||
confidence = "medium"
|
||||
else:
|
||||
confidence = "low"
|
||||
|
||||
return {
|
||||
"one_liner": (title or text[:150])[:200],
|
||||
"key_technical_point": (text or "No additional content in post")[:200],
|
||||
"potential_use_case": "AI community discussion",
|
||||
"confidence": confidence,
|
||||
}
|
||||
|
||||
|
||||
# ── Extraction helpers ─────────────────────────────────────────────────────
|
||||
def _assess_extraction_quality(one_liner, key_tech, use_case, source_confidence) -> str:
|
||||
score = 0
|
||||
penalties = 0
|
||||
ol = one_liner.strip()
|
||||
ol_len = len(ol)
|
||||
|
||||
if 40 <= ol_len <= 200:
|
||||
score += 2
|
||||
elif 20 <= ol_len < 40:
|
||||
score += 1
|
||||
elif ol_len > 200:
|
||||
penalties += 1
|
||||
|
||||
if ol.endswith(('.', '!', '?', '…')):
|
||||
score += 1
|
||||
else:
|
||||
penalties += 1
|
||||
|
||||
if re.search(r'\b(?:is|are|provides|enables|implements|makes|allows|builds|creates|runs|uses)\b', ol, re.I):
|
||||
score += 1
|
||||
elif re.match(r'^[A-Z]\w+', ol) and ol_len > 30:
|
||||
score += 0.5
|
||||
|
||||
open_brackets = ol.count('[') + ol.count('(')
|
||||
close_brackets = ol.count(']') + ol.count(')')
|
||||
if abs(open_brackets - close_brackets) > 0:
|
||||
penalties += 1
|
||||
if open_brackets > 2:
|
||||
penalties += 1
|
||||
|
||||
kt = key_tech.strip()
|
||||
if kt and len(kt) > 20 and not kt.startswith('See '):
|
||||
score += 1
|
||||
else:
|
||||
penalties += 0.5
|
||||
|
||||
uc = use_case.strip()
|
||||
if uc and len(uc) > 10 and not uc.startswith('Relevant for'):
|
||||
score += 1
|
||||
else:
|
||||
penalties += 0.5
|
||||
|
||||
net = score - penalties
|
||||
if net >= 3:
|
||||
return source_confidence
|
||||
elif net >= 1:
|
||||
return "medium"
|
||||
return "low"
|
||||
|
||||
|
||||
def _is_security_tooling(title: str, one_liner: str, key_tech: str) -> bool:
|
||||
combined = f"{title} {one_liner} {key_tech}".lower()
|
||||
return any(sig in combined for sig in [
|
||||
"offensive", "pentest", "red team", "exploit", "kill chain",
|
||||
"attack surface", "vulnerability scan", "zero-day",
|
||||
"reverse engineer", "c2", "command and control",
|
||||
])
|
||||
|
||||
|
||||
def _find_project_description(text: str, title: str) -> Optional[str]:
|
||||
proj_name = title.split(':')[0].split('/')[0].strip().lower()
|
||||
for para in text.split('\n\n'):
|
||||
para = para.strip()
|
||||
if not para or para.startswith('##') or len(para) < 20:
|
||||
continue
|
||||
if 'img' in para.lower() or 'badge' in para.lower() or 'shields' in para.lower():
|
||||
continue
|
||||
if re.match(r'^[~$#€£¥*»\d]', para):
|
||||
continue
|
||||
special_chars = sum(1 for c in para if not c.isalnum() and not c.isspace() and c not in ',.!?;:\'\"-()[]')
|
||||
if special_chars / max(len(para), 1) > 0.4:
|
||||
continue
|
||||
sentence = re.split(r'[.!?]', para)[0].strip()
|
||||
if len(sentence) > 30:
|
||||
return sentence + '.'
|
||||
|
||||
for pattern in [
|
||||
rf'{re.escape(proj_name[:20])}\s+(?:is|enables|provides|implements)\s+[^.]+\.?',
|
||||
r'(?:This\s+)?(?:project|library|framework|tool|package)\s+(?:is|enables|provides)\s+[^.]+\.?',
|
||||
]:
|
||||
match = re.search(pattern, text, re.I)
|
||||
if match:
|
||||
return match.group(0)[:200]
|
||||
return None
|
||||
|
||||
|
||||
def _find_contribution(text: str) -> Optional[str]:
|
||||
for pattern in [
|
||||
r'(?:we|this\s+paper)\s+(?:propose|introduce|present|propose and evaluate)\s+[^.]{10,150}\.',
|
||||
r'(?:we\s+(?:show|demonstrate|find|discover|observe))\s+[^.]{10,150}\.',
|
||||
]:
|
||||
match = re.search(pattern, text, re.I)
|
||||
if match:
|
||||
return match.group(0)[:200]
|
||||
first = re.split(r'[.!?]', text)[0].strip()
|
||||
return first if first else None
|
||||
|
||||
|
||||
def _extract_technical_point(text: str, confidence: str) -> str:
|
||||
for pattern in [
|
||||
r'architecture(?:\s+designed)?\s+(?:for|to|that)\s+[^.]+\.?',
|
||||
r'(?:using|via|based\s+on|through)\s+[a-z][^.]{10,100}\.',
|
||||
]:
|
||||
match = re.search(pattern, text, re.I)
|
||||
if match:
|
||||
return match.group(0)[:200]
|
||||
if confidence == "low":
|
||||
return "Technical details not available in extracted content"
|
||||
return "See README for technical details"
|
||||
|
||||
|
||||
def _extract_method(text: str) -> str:
|
||||
for pattern in [
|
||||
r'(?:method|approach|framework|technique|model|system)\s+(?:based|using|via|through|with)\s+[a-z][^.]{10,120}\.',
|
||||
r'(?:combining|leveraging|exploiting)\s+[a-z][^.]{10,120}\.',
|
||||
]:
|
||||
match = re.search(pattern, text, re.I)
|
||||
if match:
|
||||
return match.group(0)[:200]
|
||||
return "See full paper for methodology"
|
||||
|
||||
|
||||
def _extract_use_case(text: str, title: str) -> str:
|
||||
for pattern in [
|
||||
r'(?:for|to)\s+(?:developers|engineers|researchers|teams)\s+who?\s+[^.]{5,80}\.',
|
||||
r'(?:enables|allows|helps)\s+[^\s]+\s+to\s+[^.]{10,80}\.',
|
||||
]:
|
||||
match = re.search(pattern, text, re.I)
|
||||
if match:
|
||||
return match.group(0)[:200]
|
||||
return f"Relevant for {title.lower()[:50]} developers and users"
|
||||
|
||||
|
||||
def _extract_application(text: str) -> str:
|
||||
title_lower = text[:200].lower()
|
||||
if any(k in title_lower for k in ["agent", "agentic"]):
|
||||
return "Building AI agent systems"
|
||||
if any(k in title_lower for k in ["verification", "verify"]):
|
||||
return "LLM output verification and reliability"
|
||||
if any(k in title_lower for k in ["embodied", "robot"]):
|
||||
return "Embodied AI and robotics applications"
|
||||
if any(k in title_lower for k in ["distill"]):
|
||||
return "Model distillation and knowledge transfer"
|
||||
return "See paper for specific applications"
|
||||
|
||||
|
||||
# ── Pipeline functions ─────────────────────────────────────────────────────
|
||||
def summarize_entry(entry: dict, conn: sqlite3.Connection) -> bool:
|
||||
"""Summarize a single entry using rule-based extraction."""
|
||||
source = entry["source"]
|
||||
title = entry["title"]
|
||||
content = entry.get("extracted_text", "")
|
||||
eid = entry["id"]
|
||||
|
||||
if not content or len(content) < 50:
|
||||
return False
|
||||
|
||||
if source == "github":
|
||||
summary = extract_github_summary(title, content)
|
||||
elif source == "arxiv":
|
||||
summary = extract_arxiv_summary(title, content)
|
||||
elif source == "reddit":
|
||||
summary = extract_reddit_summary(title, content)
|
||||
else:
|
||||
summary = extract_reddit_summary(title, content)
|
||||
|
||||
conn.execute("UPDATE entries SET summary = ? WHERE id = ?",
|
||||
(json.dumps(summary), eid))
|
||||
conn.commit()
|
||||
return True
|
||||
|
||||
|
||||
def run_summarization(source: Optional[str] = None, limit: int = 0) -> None:
|
||||
"""Summarize all pending entries."""
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
cur = conn.cursor()
|
||||
|
||||
where = "summary IS NULL"
|
||||
params = []
|
||||
if source:
|
||||
where += " AND source = ?"
|
||||
params.append(source)
|
||||
|
||||
cur.execute(f"SELECT COUNT(*) FROM entries WHERE {where}", params)
|
||||
total_pending = cur.fetchone()[0]
|
||||
print(f"[summarize] {total_pending} pending entries")
|
||||
|
||||
if limit:
|
||||
limit_clause = f"LIMIT {limit}"
|
||||
else:
|
||||
limit_clause = ""
|
||||
|
||||
cur.execute(f"""
|
||||
SELECT id, source, title, extracted_text, summary
|
||||
FROM entries WHERE {where}
|
||||
ORDER BY first_seen DESC
|
||||
{limit_clause}
|
||||
""", params)
|
||||
|
||||
summarized = 0
|
||||
for row in cur.fetchall():
|
||||
entry = {
|
||||
"id": row[0], "source": row[1], "title": row[2],
|
||||
"extracted_text": row[3], "summary": row[4],
|
||||
}
|
||||
if summarize_entry(entry, conn):
|
||||
summarized += 1
|
||||
|
||||
conn.close()
|
||||
print(f"[summarize] done — {summarized} entries summarized")
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Theme-based trend tracking for Athena.
|
||||
|
||||
Tag by THEME, not by entry ID. Count NEW theme-tagged arrivals per cron cycle.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
from collections import Counter
|
||||
|
||||
from oracle.config import DB_PATH
|
||||
|
||||
# Theme -> regex over title+summary+extracted text
|
||||
THEME_PATTERNS = {
|
||||
"tool-call": re.compile(
|
||||
r"\b(tool[- ]?call|tool[- ]?use|competence gate|confidence gate|"
|
||||
r"gate[d]? tool|action gate|tool reliability|function call gate)\b",
|
||||
re.I),
|
||||
"context": re.compile(
|
||||
r"\b(context (compress|window|ceiling|summar)|semantic compress|"
|
||||
r"token (compress|budget)|compress (context|session)|context (limit|overflow))\b",
|
||||
re.I),
|
||||
"compute": re.compile(
|
||||
r"\b(small(er|est)? model|route to|inference cost|cpu (tts|infer)|"
|
||||
r"cheap(er)? model|model routing|tiny model|on[- ]device (llm|model))\b",
|
||||
re.I),
|
||||
"trust": re.compile(
|
||||
r"\b(trust(ed)? (adapter|lora)|vetted adapter|learn (only|what).*adapter|"
|
||||
r"trust boundary|what a model (can|may) learn|auditable (adapter|skill))\b",
|
||||
re.I),
|
||||
}
|
||||
|
||||
|
||||
def scan(conn=None, history=False) -> dict:
|
||||
"""Classify fresh entries and report new theme arrivals.
|
||||
|
||||
Returns dict with counts and cumulative totals.
|
||||
"""
|
||||
if conn is None:
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
conn.row_factory = sqlite3.Row
|
||||
own_conn = True
|
||||
else:
|
||||
own_conn = False
|
||||
|
||||
cur = conn.cursor()
|
||||
cur.execute("""CREATE TABLE IF NOT EXISTS theme_tags (
|
||||
entry_id INTEGER NOT NULL,
|
||||
theme TEXT NOT NULL,
|
||||
first_seen_cycle TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')),
|
||||
PRIMARY KEY (entry_id, theme))""")
|
||||
|
||||
cur.execute("""
|
||||
SELECT e.id, e.source, e.title,
|
||||
COALESCE(e.summary,'') AS summary,
|
||||
COALESCE(e.extracted_text,'') AS extracted
|
||||
FROM entries e
|
||||
WHERE e.id NOT IN (SELECT entry_id FROM theme_tags)
|
||||
""")
|
||||
fresh = cur.fetchall()
|
||||
|
||||
new_counts = Counter()
|
||||
for row in fresh:
|
||||
blob = f"{row['title']} {row['summary']} {row['extracted']}"
|
||||
for theme, pat in THEME_PATTERNS.items():
|
||||
if pat.search(blob):
|
||||
cur.execute(
|
||||
"INSERT OR IGNORE INTO theme_tags (entry_id, theme) VALUES (?, ?)",
|
||||
(row["id"], theme))
|
||||
new_counts[theme] += 1
|
||||
|
||||
conn.commit()
|
||||
|
||||
cur.execute("SELECT theme, COUNT(*) AS c FROM theme_tags GROUP BY theme")
|
||||
cum = {r["theme"]: r["c"] for r in cur.fetchall()}
|
||||
|
||||
result = {
|
||||
"fresh_count": len(fresh),
|
||||
"new_arrivals": dict(new_counts),
|
||||
"cumulative": cum,
|
||||
}
|
||||
|
||||
if history:
|
||||
cur.execute("""
|
||||
SELECT substr(first_seen_cycle,1,10) AS day, theme, COUNT(*) AS c
|
||||
FROM theme_tags GROUP BY day, theme ORDER BY day, theme
|
||||
""")
|
||||
result["history"] = [(r["day"], r["theme"], r["c"]) for r in cur.fetchall()]
|
||||
|
||||
if own_conn:
|
||||
conn.close()
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,254 @@
|
||||
"""Multi-variant edition engine for Athena.
|
||||
|
||||
One data pipeline → multiple audience-specific editions.
|
||||
Each variant is a YAML config that defines filters, ranking, and display.
|
||||
|
||||
World Monitor pattern: 6 variants from 1 codebase.
|
||||
"""
|
||||
import os
|
||||
import sqlite3
|
||||
import yaml
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from oracle.config import ROOT, DB_PATH, VERDICT_THRESHOLDS
|
||||
|
||||
|
||||
# ── Variant config defaults ────────────────────────────────────────────────
|
||||
|
||||
DEFAULT_CONFIG = {
|
||||
"name": "Athena AI News",
|
||||
"description": "",
|
||||
"output": "index.html",
|
||||
"filters": {
|
||||
"verdicts": ["PUBLISH", "WATCH", "ARCHIVE", "DROP"],
|
||||
"sources": [],
|
||||
"min_score": 0,
|
||||
"max_age_h": 0,
|
||||
"max_items": 0,
|
||||
},
|
||||
"ranking": {
|
||||
"by": "clickability",
|
||||
"half_life_h": 18,
|
||||
},
|
||||
"display": {
|
||||
"top_n": 8,
|
||||
"show_summary": True,
|
||||
"show_score": True,
|
||||
"show_tier": False,
|
||||
"show_verdict": False,
|
||||
"theme": "dark",
|
||||
"accent": "#5b8cff",
|
||||
"logo": "🏛️",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def load_variant(name: str) -> dict:
|
||||
"""Load a variant config by name, merging with defaults.
|
||||
|
||||
Looks in variants/<name>.yaml relative to project root.
|
||||
Returns merged config dict.
|
||||
"""
|
||||
variants_dir = ROOT / "variants"
|
||||
variant_file = variants_dir / f"{name}.yaml"
|
||||
|
||||
if not variant_file.exists():
|
||||
available = [p.stem for p in sorted(variants_dir.glob("*.yaml"))]
|
||||
raise FileNotFoundError(
|
||||
f"Variant '{name}' not found. Available: {', '.join(available)}"
|
||||
)
|
||||
|
||||
with open(variant_file) as f:
|
||||
raw = yaml.safe_load(f) or {}
|
||||
|
||||
# Deep merge with defaults
|
||||
config = _deep_merge(DEFAULT_CONFIG, raw)
|
||||
config["_name"] = name
|
||||
config["_file"] = str(variant_file)
|
||||
return config
|
||||
|
||||
|
||||
def list_variants() -> list[str]:
|
||||
"""List all available variant names."""
|
||||
variants_dir = ROOT / "variants"
|
||||
if not variants_dir.exists():
|
||||
return []
|
||||
return sorted(p.stem for p in variants_dir.glob("*.yaml"))
|
||||
|
||||
|
||||
def apply_filters(conn: sqlite3.Connection, variant: dict) -> list[dict]:
|
||||
"""Fetch entries from DB filtered by variant config.
|
||||
|
||||
Returns list of entry dicts matching the variant's filter criteria.
|
||||
"""
|
||||
f = variant["filters"]
|
||||
|
||||
# Build query
|
||||
clauses = []
|
||||
params = []
|
||||
|
||||
# Verdict filter
|
||||
if f.get("verdicts"):
|
||||
verdicts = [v for v in f["verdicts"] if v] # strip empties
|
||||
if verdicts:
|
||||
placeholders = ", ".join("?" for _ in verdicts)
|
||||
clauses.append(f"verdict IN ({placeholders})")
|
||||
params.extend(verdicts)
|
||||
|
||||
# Source filter (empty = all)
|
||||
if f.get("sources"):
|
||||
sources = [s for s in f["sources"] if s]
|
||||
if sources:
|
||||
placeholders = ", ".join("?" for _ in sources)
|
||||
clauses.append(f"source IN ({placeholders})")
|
||||
params.extend(sources)
|
||||
|
||||
# Min signal score
|
||||
if f.get("min_score", 0) > 0:
|
||||
clauses.append("COALESCE(signal_score, 0) >= ?")
|
||||
params.append(f["min_score"])
|
||||
|
||||
# Max age
|
||||
if f.get("max_age_h", 0) > 0:
|
||||
max_age = f["max_age_h"]
|
||||
cutoff = datetime.now(timezone.utc).timestamp() - (max_age * 3600)
|
||||
clauses.append("first_seen >= datetime(?, 'unixepoch')")
|
||||
params.append(cutoff)
|
||||
|
||||
# Base query
|
||||
sql = "SELECT * FROM entries"
|
||||
if clauses:
|
||||
sql += " WHERE " + " AND ".join(clauses)
|
||||
|
||||
sql += " ORDER BY first_seen DESC"
|
||||
|
||||
# Max items
|
||||
if f.get("max_items", 0) > 0:
|
||||
sql += f" LIMIT {int(f['max_items'])}"
|
||||
|
||||
conn.row_factory = sqlite3.Row
|
||||
cur = conn.cursor()
|
||||
cur.execute(sql, params)
|
||||
rows = cur.fetchall()
|
||||
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def rank_items(items: list[dict], variant: dict) -> list[dict]:
|
||||
"""Rank items according to variant ranking config.
|
||||
|
||||
Supports: clickability, signal_score, verdict_priority, freshness
|
||||
"""
|
||||
ranking = variant["ranking"]
|
||||
by = ranking.get("by", "clickability")
|
||||
half_life = ranking.get("half_life_h", 18)
|
||||
|
||||
if by == "clickability":
|
||||
# Use clickability module for decayed scoring
|
||||
from oracle.clickability import compute_index, decay_index
|
||||
items = compute_index(items)
|
||||
items = decay_index(items, half_life)
|
||||
items.sort(key=lambda x: x.get("clickability_decayed", 0), reverse=True)
|
||||
|
||||
elif by == "signal_score":
|
||||
items.sort(key=lambda x: float(x.get("signal_score") or 0), reverse=True)
|
||||
|
||||
elif by == "verdict_priority":
|
||||
# PUBLISH > WATCH > ARCHIVE > DROP
|
||||
priority = {"PUBLISH": 0, "WATCH": 1, "ARCHIVE": 2, "DROP": 3}
|
||||
items.sort(key=lambda x: (
|
||||
priority.get(x.get("verdict", "DROP"), 4),
|
||||
float(x.get("signal_score") or 0),
|
||||
), reverse=False)
|
||||
|
||||
elif by == "freshness":
|
||||
items.sort(key=lambda x: x.get("first_seen", ""), reverse=True)
|
||||
|
||||
else:
|
||||
# Default fallback: signal score
|
||||
items.sort(key=lambda x: float(x.get("signal_score") or 0), reverse=True)
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def render_variant(variant: dict, dry_run: bool = False, webroot: Optional[str] = None):
|
||||
"""Full render pipeline for one variant.
|
||||
|
||||
Load config → filter → rank → render HTML.
|
||||
Returns (output_path, item_count).
|
||||
"""
|
||||
from oracle.render import render_variant_html, render_variant_json
|
||||
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
conn.row_factory = sqlite3.Row
|
||||
|
||||
items = apply_filters(conn, variant)
|
||||
items = rank_items(items, variant)
|
||||
conn.close()
|
||||
|
||||
if not items:
|
||||
print(f"[variant:{variant['_name']}] No items matched filters")
|
||||
return None, 0
|
||||
|
||||
page = render_variant_html(items, variant)
|
||||
|
||||
# Determine output path
|
||||
output_rel = variant.get("output", "index.html")
|
||||
|
||||
if dry_run:
|
||||
out_dir = ROOT / "_preview" / variant["_name"]
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Use just the filename for dry-run (output_rel may have subdirs)
|
||||
dry_filename = os.path.basename(output_rel)
|
||||
out_path = str(out_dir / dry_filename)
|
||||
|
||||
with open(out_path, "w") as f:
|
||||
f.write(page)
|
||||
|
||||
# Also write JSON feed
|
||||
json_path = str(out_dir / "feed.json")
|
||||
render_variant_json(items, variant, json_path)
|
||||
|
||||
display = variant["display"]
|
||||
top_n = display.get("top_n", 8)
|
||||
top = items[:top_n]
|
||||
print(f"[variant:{variant['_name']}] wrote {out_path} ({len(items)} items)")
|
||||
print(f" TOP {top_n}:")
|
||||
for i, it in enumerate(top, 1):
|
||||
score = f" sig={it.get('signal_score', 0):.1f}" if display.get("show_score") else ""
|
||||
tier = f" tier={it.get('source_tier', '?')}" if display.get("show_tier") else ""
|
||||
verdict = f" [{it.get('verdict', '?')}]" if display.get("show_verdict") else ""
|
||||
print(f" {i}. {it['title'][:60]}{score}{tier}{verdict}")
|
||||
|
||||
return out_path, len(items)
|
||||
|
||||
# Production render
|
||||
target = webroot or str(ROOT / "site")
|
||||
out_path = os.path.join(target, output_rel)
|
||||
os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
|
||||
|
||||
with open(out_path, "w") as f:
|
||||
f.write(page)
|
||||
|
||||
# JSON feed alongside
|
||||
json_path = os.path.join(os.path.dirname(out_path), "feed.json")
|
||||
render_variant_json(items, variant, json_path)
|
||||
|
||||
print(f"[variant:{variant['_name']}] wrote {out_path} ({len(items)} items)")
|
||||
return out_path, len(items)
|
||||
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
def _deep_merge(base: dict, override: dict) -> dict:
|
||||
"""Recursively merge override into base dict."""
|
||||
merged = base.copy()
|
||||
for key, value in override.items():
|
||||
if key in merged and isinstance(merged[key], dict) and isinstance(value, dict):
|
||||
merged[key] = _deep_merge(merged[key], value)
|
||||
else:
|
||||
merged[key] = value
|
||||
return merged
|
||||
Reference in New Issue
Block a user