Epictetus
07c5f9a5c2
- New oracle/ package (11 modules) with unified CLI (python -m oracle) - Source tiers: Tier 1 (arxiv/github/hf), Tier 2 (rss/hn), Tier 3 (reddit) - Composite verdicts: PUBLISH/WATCH/ARCHIVE/DROP based on signal score + age - Content-hash dedup: SHA-256[:16] normalized, atomic at insert time - Multi-variant editions: 4 YAML configs (default/research/devops/brief) - Variant engine: filter → rank → render (HTML + JSON, themed) - Per-adapter timeout (10s) + threading fallback - Consolidated 12 root scripts → thin wrappers + oracle/ package - Archived stale scripts (_engagement, _live_compare, reddit_proof) - Updated .gitignore, README.md, schema.sql
94 lines
3.0 KiB
Python
94 lines
3.0 KiB
Python
"""Theme-based trend tracking for Athena.
|
|
|
|
Tag by THEME, not by entry ID. Count NEW theme-tagged arrivals per cron cycle.
|
|
"""
|
|
import argparse
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
from collections import Counter
|
|
|
|
from oracle.config import DB_PATH
|
|
|
|
# Theme -> regex over title+summary+extracted text
|
|
THEME_PATTERNS = {
|
|
"tool-call": re.compile(
|
|
r"\b(tool[- ]?call|tool[- ]?use|competence gate|confidence gate|"
|
|
r"gate[d]? tool|action gate|tool reliability|function call gate)\b",
|
|
re.I),
|
|
"context": re.compile(
|
|
r"\b(context (compress|window|ceiling|summar)|semantic compress|"
|
|
r"token (compress|budget)|compress (context|session)|context (limit|overflow))\b",
|
|
re.I),
|
|
"compute": re.compile(
|
|
r"\b(small(er|est)? model|route to|inference cost|cpu (tts|infer)|"
|
|
r"cheap(er)? model|model routing|tiny model|on[- ]device (llm|model))\b",
|
|
re.I),
|
|
"trust": re.compile(
|
|
r"\b(trust(ed)? (adapter|lora)|vetted adapter|learn (only|what).*adapter|"
|
|
r"trust boundary|what a model (can|may) learn|auditable (adapter|skill))\b",
|
|
re.I),
|
|
}
|
|
|
|
|
|
def scan(conn=None, history=False) -> dict:
|
|
"""Classify fresh entries and report new theme arrivals.
|
|
|
|
Returns dict with counts and cumulative totals.
|
|
"""
|
|
if conn is None:
|
|
conn = sqlite3.connect(str(DB_PATH))
|
|
conn.row_factory = sqlite3.Row
|
|
own_conn = True
|
|
else:
|
|
own_conn = False
|
|
|
|
cur = conn.cursor()
|
|
cur.execute("""CREATE TABLE IF NOT EXISTS theme_tags (
|
|
entry_id INTEGER NOT NULL,
|
|
theme TEXT NOT NULL,
|
|
first_seen_cycle TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')),
|
|
PRIMARY KEY (entry_id, theme))""")
|
|
|
|
cur.execute("""
|
|
SELECT e.id, e.source, e.title,
|
|
COALESCE(e.summary,'') AS summary,
|
|
COALESCE(e.extracted_text,'') AS extracted
|
|
FROM entries e
|
|
WHERE e.id NOT IN (SELECT entry_id FROM theme_tags)
|
|
""")
|
|
fresh = cur.fetchall()
|
|
|
|
new_counts = Counter()
|
|
for row in fresh:
|
|
blob = f"{row['title']} {row['summary']} {row['extracted']}"
|
|
for theme, pat in THEME_PATTERNS.items():
|
|
if pat.search(blob):
|
|
cur.execute(
|
|
"INSERT OR IGNORE INTO theme_tags (entry_id, theme) VALUES (?, ?)",
|
|
(row["id"], theme))
|
|
new_counts[theme] += 1
|
|
|
|
conn.commit()
|
|
|
|
cur.execute("SELECT theme, COUNT(*) AS c FROM theme_tags GROUP BY theme")
|
|
cum = {r["theme"]: r["c"] for r in cur.fetchall()}
|
|
|
|
result = {
|
|
"fresh_count": len(fresh),
|
|
"new_arrivals": dict(new_counts),
|
|
"cumulative": cum,
|
|
}
|
|
|
|
if history:
|
|
cur.execute("""
|
|
SELECT substr(first_seen_cycle,1,10) AS day, theme, COUNT(*) AS c
|
|
FROM theme_tags GROUP BY day, theme ORDER BY day, theme
|
|
""")
|
|
result["history"] = [(r["day"], r["theme"], r["c"]) for r in cur.fetchall()]
|
|
|
|
if own_conn:
|
|
conn.close()
|
|
|
|
return result
|