Sprint 0+1: Package restructure, source tiers, verdicts, multi-variant editions
- New oracle/ package (11 modules) with unified CLI (python -m oracle) - Source tiers: Tier 1 (arxiv/github/hf), Tier 2 (rss/hn), Tier 3 (reddit) - Composite verdicts: PUBLISH/WATCH/ARCHIVE/DROP based on signal score + age - Content-hash dedup: SHA-256[:16] normalized, atomic at insert time - Multi-variant editions: 4 YAML configs (default/research/devops/brief) - Variant engine: filter → rank → render (HTML + JSON, themed) - Per-adapter timeout (10s) + threading fallback - Consolidated 12 root scripts → thin wrappers + oracle/ package - Archived stale scripts (_engagement, _live_compare, reddit_proof) - Updated .gitignore, README.md, schema.sql
This commit is contained in:
Epictetus
@@ -0,0 +1,254 @@
|
||||
"""Multi-variant edition engine for Athena.
|
||||
|
||||
One data pipeline → multiple audience-specific editions.
|
||||
Each variant is a YAML config that defines filters, ranking, and display.
|
||||
|
||||
World Monitor pattern: 6 variants from 1 codebase.
|
||||
"""
|
||||
import os
|
||||
import sqlite3
|
||||
import yaml
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from oracle.config import ROOT, DB_PATH, VERDICT_THRESHOLDS
|
||||
|
||||
|
||||
# ── Variant config defaults ────────────────────────────────────────────────
|
||||
|
||||
DEFAULT_CONFIG = {
|
||||
"name": "Athena AI News",
|
||||
"description": "",
|
||||
"output": "index.html",
|
||||
"filters": {
|
||||
"verdicts": ["PUBLISH", "WATCH", "ARCHIVE", "DROP"],
|
||||
"sources": [],
|
||||
"min_score": 0,
|
||||
"max_age_h": 0,
|
||||
"max_items": 0,
|
||||
},
|
||||
"ranking": {
|
||||
"by": "clickability",
|
||||
"half_life_h": 18,
|
||||
},
|
||||
"display": {
|
||||
"top_n": 8,
|
||||
"show_summary": True,
|
||||
"show_score": True,
|
||||
"show_tier": False,
|
||||
"show_verdict": False,
|
||||
"theme": "dark",
|
||||
"accent": "#5b8cff",
|
||||
"logo": "🏛️",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def load_variant(name: str) -> dict:
|
||||
"""Load a variant config by name, merging with defaults.
|
||||
|
||||
Looks in variants/<name>.yaml relative to project root.
|
||||
Returns merged config dict.
|
||||
"""
|
||||
variants_dir = ROOT / "variants"
|
||||
variant_file = variants_dir / f"{name}.yaml"
|
||||
|
||||
if not variant_file.exists():
|
||||
available = [p.stem for p in sorted(variants_dir.glob("*.yaml"))]
|
||||
raise FileNotFoundError(
|
||||
f"Variant '{name}' not found. Available: {', '.join(available)}"
|
||||
)
|
||||
|
||||
with open(variant_file) as f:
|
||||
raw = yaml.safe_load(f) or {}
|
||||
|
||||
# Deep merge with defaults
|
||||
config = _deep_merge(DEFAULT_CONFIG, raw)
|
||||
config["_name"] = name
|
||||
config["_file"] = str(variant_file)
|
||||
return config
|
||||
|
||||
|
||||
def list_variants() -> list[str]:
|
||||
"""List all available variant names."""
|
||||
variants_dir = ROOT / "variants"
|
||||
if not variants_dir.exists():
|
||||
return []
|
||||
return sorted(p.stem for p in variants_dir.glob("*.yaml"))
|
||||
|
||||
|
||||
def apply_filters(conn: sqlite3.Connection, variant: dict) -> list[dict]:
|
||||
"""Fetch entries from DB filtered by variant config.
|
||||
|
||||
Returns list of entry dicts matching the variant's filter criteria.
|
||||
"""
|
||||
f = variant["filters"]
|
||||
|
||||
# Build query
|
||||
clauses = []
|
||||
params = []
|
||||
|
||||
# Verdict filter
|
||||
if f.get("verdicts"):
|
||||
verdicts = [v for v in f["verdicts"] if v] # strip empties
|
||||
if verdicts:
|
||||
placeholders = ", ".join("?" for _ in verdicts)
|
||||
clauses.append(f"verdict IN ({placeholders})")
|
||||
params.extend(verdicts)
|
||||
|
||||
# Source filter (empty = all)
|
||||
if f.get("sources"):
|
||||
sources = [s for s in f["sources"] if s]
|
||||
if sources:
|
||||
placeholders = ", ".join("?" for _ in sources)
|
||||
clauses.append(f"source IN ({placeholders})")
|
||||
params.extend(sources)
|
||||
|
||||
# Min signal score
|
||||
if f.get("min_score", 0) > 0:
|
||||
clauses.append("COALESCE(signal_score, 0) >= ?")
|
||||
params.append(f["min_score"])
|
||||
|
||||
# Max age
|
||||
if f.get("max_age_h", 0) > 0:
|
||||
max_age = f["max_age_h"]
|
||||
cutoff = datetime.now(timezone.utc).timestamp() - (max_age * 3600)
|
||||
clauses.append("first_seen >= datetime(?, 'unixepoch')")
|
||||
params.append(cutoff)
|
||||
|
||||
# Base query
|
||||
sql = "SELECT * FROM entries"
|
||||
if clauses:
|
||||
sql += " WHERE " + " AND ".join(clauses)
|
||||
|
||||
sql += " ORDER BY first_seen DESC"
|
||||
|
||||
# Max items
|
||||
if f.get("max_items", 0) > 0:
|
||||
sql += f" LIMIT {int(f['max_items'])}"
|
||||
|
||||
conn.row_factory = sqlite3.Row
|
||||
cur = conn.cursor()
|
||||
cur.execute(sql, params)
|
||||
rows = cur.fetchall()
|
||||
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def rank_items(items: list[dict], variant: dict) -> list[dict]:
|
||||
"""Rank items according to variant ranking config.
|
||||
|
||||
Supports: clickability, signal_score, verdict_priority, freshness
|
||||
"""
|
||||
ranking = variant["ranking"]
|
||||
by = ranking.get("by", "clickability")
|
||||
half_life = ranking.get("half_life_h", 18)
|
||||
|
||||
if by == "clickability":
|
||||
# Use clickability module for decayed scoring
|
||||
from oracle.clickability import compute_index, decay_index
|
||||
items = compute_index(items)
|
||||
items = decay_index(items, half_life)
|
||||
items.sort(key=lambda x: x.get("clickability_decayed", 0), reverse=True)
|
||||
|
||||
elif by == "signal_score":
|
||||
items.sort(key=lambda x: float(x.get("signal_score") or 0), reverse=True)
|
||||
|
||||
elif by == "verdict_priority":
|
||||
# PUBLISH > WATCH > ARCHIVE > DROP
|
||||
priority = {"PUBLISH": 0, "WATCH": 1, "ARCHIVE": 2, "DROP": 3}
|
||||
items.sort(key=lambda x: (
|
||||
priority.get(x.get("verdict", "DROP"), 4),
|
||||
float(x.get("signal_score") or 0),
|
||||
), reverse=False)
|
||||
|
||||
elif by == "freshness":
|
||||
items.sort(key=lambda x: x.get("first_seen", ""), reverse=True)
|
||||
|
||||
else:
|
||||
# Default fallback: signal score
|
||||
items.sort(key=lambda x: float(x.get("signal_score") or 0), reverse=True)
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def render_variant(variant: dict, dry_run: bool = False, webroot: Optional[str] = None):
|
||||
"""Full render pipeline for one variant.
|
||||
|
||||
Load config → filter → rank → render HTML.
|
||||
Returns (output_path, item_count).
|
||||
"""
|
||||
from oracle.render import render_variant_html, render_variant_json
|
||||
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
conn.row_factory = sqlite3.Row
|
||||
|
||||
items = apply_filters(conn, variant)
|
||||
items = rank_items(items, variant)
|
||||
conn.close()
|
||||
|
||||
if not items:
|
||||
print(f"[variant:{variant['_name']}] No items matched filters")
|
||||
return None, 0
|
||||
|
||||
page = render_variant_html(items, variant)
|
||||
|
||||
# Determine output path
|
||||
output_rel = variant.get("output", "index.html")
|
||||
|
||||
if dry_run:
|
||||
out_dir = ROOT / "_preview" / variant["_name"]
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Use just the filename for dry-run (output_rel may have subdirs)
|
||||
dry_filename = os.path.basename(output_rel)
|
||||
out_path = str(out_dir / dry_filename)
|
||||
|
||||
with open(out_path, "w") as f:
|
||||
f.write(page)
|
||||
|
||||
# Also write JSON feed
|
||||
json_path = str(out_dir / "feed.json")
|
||||
render_variant_json(items, variant, json_path)
|
||||
|
||||
display = variant["display"]
|
||||
top_n = display.get("top_n", 8)
|
||||
top = items[:top_n]
|
||||
print(f"[variant:{variant['_name']}] wrote {out_path} ({len(items)} items)")
|
||||
print(f" TOP {top_n}:")
|
||||
for i, it in enumerate(top, 1):
|
||||
score = f" sig={it.get('signal_score', 0):.1f}" if display.get("show_score") else ""
|
||||
tier = f" tier={it.get('source_tier', '?')}" if display.get("show_tier") else ""
|
||||
verdict = f" [{it.get('verdict', '?')}]" if display.get("show_verdict") else ""
|
||||
print(f" {i}. {it['title'][:60]}{score}{tier}{verdict}")
|
||||
|
||||
return out_path, len(items)
|
||||
|
||||
# Production render
|
||||
target = webroot or str(ROOT / "site")
|
||||
out_path = os.path.join(target, output_rel)
|
||||
os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
|
||||
|
||||
with open(out_path, "w") as f:
|
||||
f.write(page)
|
||||
|
||||
# JSON feed alongside
|
||||
json_path = os.path.join(os.path.dirname(out_path), "feed.json")
|
||||
render_variant_json(items, variant, json_path)
|
||||
|
||||
print(f"[variant:{variant['_name']}] wrote {out_path} ({len(items)} items)")
|
||||
return out_path, len(items)
|
||||
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
def _deep_merge(base: dict, override: dict) -> dict:
|
||||
"""Recursively merge override into base dict."""
|
||||
merged = base.copy()
|
||||
for key, value in override.items():
|
||||
if key in merged and isinstance(merged[key], dict) and isinstance(value, dict):
|
||||
merged[key] = _deep_merge(merged[key], value)
|
||||
else:
|
||||
merged[key] = value
|
||||
return merged
|
||||
Reference in New Issue
Block a user