From 07c5f9a5c2ff62205ee0b83c63633a3b3e5f5785 Mon Sep 17 00:00:00 2001 From: Epictetus Date: Wed, 22 Jul 2026 13:32:15 +0000 Subject: [PATCH] Sprint 0+1: Package restructure, source tiers, verdicts, multi-variant editions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New oracle/ package (11 modules) with unified CLI (python -m oracle) - Source tiers: Tier 1 (arxiv/github/hf), Tier 2 (rss/hn), Tier 3 (reddit) - Composite verdicts: PUBLISH/WATCH/ARCHIVE/DROP based on signal score + age - Content-hash dedup: SHA-256[:16] normalized, atomic at insert time - Multi-variant editions: 4 YAML configs (default/research/devops/brief) - Variant engine: filter → rank → render (HTML + JSON, themed) - Per-adapter timeout (10s) + threading fallback - Consolidated 12 root scripts → thin wrappers + oracle/ package - Archived stale scripts (_engagement, _live_compare, reddit_proof) - Updated .gitignore, README.md, schema.sql --- .gitignore | 6 + README.md | 245 ++++++-- adapters/_store.py | 30 +- adapters/arxiv.py | 12 + adapters/reddit.py | 66 ++- archive.py | 99 +--- .../_engagement_analysis.py | 0 _live_compare.py => archive/_live_compare.py | 0 reddit_proof.py => archive/reddit_proof.py | 0 athena/__init__.py | 1 - athena/scoring.py | 545 ----------------- clickability.py | 304 +--------- oracle/__init__.py | 21 + oracle/__main__.py | 4 + oracle/archive.py | 83 +++ oracle/cli.py | 497 ++++++++++++++++ oracle/clickability.py | 210 +++++++ oracle/config.py | 68 +++ oracle/db.py | 203 +++++++ oracle/dedup.py | 140 +++++ oracle/recency.py | 140 +++++ oracle/render.py | 359 +++++++++++ oracle/scoring.py | 332 +++++++++++ oracle/summarize.py | 301 ++++++++++ oracle/themes.py | 93 +++ oracle/variants.py | 254 ++++++++ pipeline.py | 381 +----------- propagate_stack_now.py | 281 +-------- query.py | 470 +-------------- recency_guard.py | 214 +------ render_site.py | 190 +----- summarize.py | 555 +----------------- theme_scan.py | 133 +---- variants/brief.yaml | 25 + variants/default.yaml | 31 + variants/devops.yaml | 25 + variants/research.yaml | 25 + write_summaries.py | 86 +-- 38 files changed, 3195 insertions(+), 3234 deletions(-) rename _engagement_analysis.py => archive/_engagement_analysis.py (100%) rename _live_compare.py => archive/_live_compare.py (100%) rename reddit_proof.py => archive/reddit_proof.py (100%) delete mode 100644 athena/__init__.py delete mode 100644 athena/scoring.py create mode 100644 oracle/__init__.py create mode 100644 oracle/__main__.py create mode 100644 oracle/archive.py create mode 100644 oracle/cli.py create mode 100644 oracle/clickability.py create mode 100644 oracle/config.py create mode 100644 oracle/db.py create mode 100644 oracle/dedup.py create mode 100644 oracle/recency.py create mode 100644 oracle/render.py create mode 100644 oracle/scoring.py create mode 100644 oracle/summarize.py create mode 100644 oracle/themes.py create mode 100644 oracle/variants.py create mode 100644 variants/brief.yaml create mode 100644 variants/default.yaml create mode 100644 variants/devops.yaml create mode 100644 variants/research.yaml diff --git a/.gitignore b/.gitignore index 945d696..68a5d9e 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,9 @@ __pycache__/ # Env / virtualenv .venv/ .env + +# Generated preview output +_preview/ + +# Generated site output (recreated from oracle.db) +site/ diff --git a/README.md b/README.md index 2909a36..1fbc64b 100644 --- a/README.md +++ b/README.md @@ -1,62 +1,197 @@ -# Athena — AI Research Intelligence Engine +# Athena Oracle — AI Research Intelligence Pipeline -> Multi-source research ingestion, pattern detection, and hypothesis falsification pipeline. -> Autonomous daily operation: ingest → summarize → theme-scan → flag weak signals. +> Multi-source AI news aggregation, scoring, and multi-variant edition rendering. +> One pipeline → multiple audience-specific editions. -## Repo +**Live site:** https://ai-oracle.com (rendered from this pipeline) -- **Location:** `Tony_tech/athena-oracle` (public, Gitea) -- **URL:** http://localhost:3000/Tony_tech/athena-oracle -- **Branch:** `main` -- **Origin:** mirrors `~/oracle` (local working copy) +## Architecture -## What it does - -Athena runs on a daily cron (13:00 UTC) and continuously ingests from 6 sources, -then applies a signal-scoring + falsification loop to surface real AI research -momentum rather than source-expansion noise. - -| Component | File | Purpose | -|-----------|------|---------| -| Pipeline | `pipeline.py` | Orchestrates ingest → store → summarize → score | -| Adapters | `adapters/` | arxiv, github, huggingface, hackernews, reddit, rss_feeds | -| Theme scan | `theme_scan.py` | Cross-source trend detection + idempotent falsification | -| Query | `query.py` | Interactive lookup against the store | -| Archive | `archive.py` | Cold-storage rotation | -| Summarize | `summarize.py` | Summarization via any available inference model | -| Schema | `schema.sql` | SQLite store definition | -| Cron entry | `oracle-pipeline.sh` | Wrapper invoked by Hermes cron | - -## Inference model strategy - -Athena is **model-agnostic** — it uses whatever inference backend is available at -run time, whether free or paid. There is no hard dependency on a single provider. - -`summarize.py` currently targets a local Ollama endpoint (`llama3.2:1b`) when -present. The pipeline is designed so the summarization backend can be swapped for -any model we can reach — local GPU, a paid API, or a free-tier endpoint — without -changing the ingestion, scoring, or theme-scan logic. When no inference backend is -reachable, the summarization step is skipped; ingestion, scoring, and theme-scan -continue uninterrupted. - -To wire in a different backend, implement the same `summarize(text) -> (summary, model)` -contract that `summarize_with_ollama` satisfies, and add the dispatch in -`process_card`. - -## Data handling - -- `oracle.db`, `logs/`, `.env`, `__pycache__/` are **git-ignored** (not committed). -- API tokens (`GITHUB_TOKEN`, `HUGGINGFACE_TOKEN`) are read from environment only — never hardcoded. - -## Setup - -```bash -pip install -r requirements.txt # if present; else deps are stdlib + requests -export GITHUB_TOKEN=... # optional, raises rate limit 60→5000/hr -python3 pipeline.py # manual run +``` +┌─────────────────────────────────────────────────────┐ +│ adapters/ │ +│ arxiv · github · hackernews · reddit │ +│ huggingface · rss_feeds │ +└───────────────┬─────────────────────────────────────┘ + │ fetch(limit, timeout=10s) + ▼ +┌─────────────────────────────────────────────────────┐ +│ oracle/ — Core Package │ +│ │ +│ scoring.py Pure-rule component scoring (0-10) │ +│ dedup.py Content-hash dedup + verdict engine │ +│ variants.py Multi-variant edition engine │ +│ render.py HTML/JSON variant renderer │ +│ summarize.py Source-aware text summarization │ +│ recency.py Age-based freshness gate │ +│ themes.py Theme-based trend tracking │ +│ archive.py Soft-cap entry archival │ +│ db.py Schema management + migrations │ +│ config.py Centralized configuration │ +│ cli.py Unified CLI (python -m oracle) │ +└───────────────┬─────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────┐ +│ oracle.db — SQLite (784+ entries) │ +│ Columns: signal_score · final_score · content_hash │ +│ · verdict · source_tier · summary │ +└─────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────┐ +│ variants/ — Edition Configs (YAML) │ +│ │ +│ default.yaml Full feed, clickability-ranked │ +│ research.yaml arXiv + HF papers, signal-ranked │ +│ devops.yaml Shipping tools, 7-day window │ +│ brief.yaml PUBLISH verdict only, top 8 │ +└─────────────────────────────────────────────────────┘ ``` -## Architecture detail +## Quick Start -See `whitepaper.md` for full system design, scoring methodology, and the -verification discipline that keeps adapters honest. +```bash +# Run the ingestion pipeline +python -m oracle ingest + +# Generate summaries for pending entries +python -m oracle summarize + +# Render all variant editions (preview) +python -m oracle render --all --dry-run + +# Render a single variant +python -m oracle render --variant research + +# Deploy to production +python -m oracle render --all --webroot /var/www/html + +# Check system health +python -m oracle health +``` + +## CLI Commands + +| Command | Description | +|---------|-------------| +| `ingest` | Fetch from adapters, score, store to oracle.db | +| `summarize` | Generate summaries for entries without one | +| `query top N` | Top N entries by signal score | +| `query search "text"` | Keyword search across titles/summaries | +| `query recent --hours 24` | Recent entries | +| `render` | Render variant editions (HTML + JSON) | +| `archive` | Soft-cap archival of old entries | +| `themes` | Theme-based trend tracking | +| `dedup` | Content-hash dedup + verdict management | +| `health` | System health check | + +### Ingest Options +```bash +python -m oracle ingest --sources github,arxiv # Specific sources only +python -m oracle ingest --limit 5 # 5 entries per source +python -m oracle ingest --dry-run # Fetch but don't store +``` + +### Render Options +```bash +python -m oracle render --list # Show available variants +python -m oracle render --variant brief # Single variant +python -m oracle render --all # All variants +python -m oracle render --all --dry-run # Preview mode +python -m oracle render --all --webroot /var/www # Production deploy +``` + +## Scoring Engine + +Pure-rule component scoring — no embeddings or LLM required. + +| Component | Weight | Description | +|-----------|--------|-------------| +| Shipping | 20% | Code releases, benchmarks, working demos | +| Utility | 20% | Practical tools, frameworks, integrations | +| Replication | 25% | Reproducible research, open datasets | +| Enthusiast | 20% | Community buzz, notable figures | +| Novelty | 15% | First-of-its-kind, paradigm shifts | + +**Hype Penalty:** Caps at 45% to prevent buzzwords from dominating. + +**Signal Score:** 0–10 scale (per-adapter normalization → final composite). + +## Source Tiers (World Monitor Pattern) + +| Tier | Sources | Rationale | +|------|---------|-----------| +| **Tier 1** (PRIMARY) | arxiv, github, huggingface | Peer-reviewed research, official code releases, model registry | +| **Tier 2** (SECONDARY) | rss, hackernews | Curated tech media, curated community | +| **Tier 3** (TERTIARY) | reddit | User-generated discussion | + +## Composite Verdicts + +Entries are classified based on signal score + age: + +| Verdict | Minimum Score | Maximum Age | Meaning | +|---------|--------------|-------------|---------| +| **PUBLISH** | ≥ 6.0 | ≤ 48h | High-signal, fresh — front page material | +| **WATCH** | ≥ 4.0 | ≤ 168h | Solid signal — worth tracking | +| **ARCHIVE** | ≥ 2.0 | ≤ 720h | Historical value — keep for reference | +| **DROP** | any | > 720h | Stale — exclude from active feeds | + +## Content-Hash Dedup + +SHA-256 content hashing (first 16 hex chars) for cross-source duplicate detection: +- Normalized whitespace before hashing +- Applied atomically at ingest time via `adapters/_store.py` +- Prevents the same story from appearing multiple times across sources + +## Variant Editions + +Each variant is a YAML config defining: + +- **Filters:** verdicts, sources, min_score, max_age_h, max_items +- **Ranking:** clickability | signal_score | verdict_priority | freshness (with half-life decay) +- **Display:** theme, accent color, logo, show/hide score/tier/verdict badges + +Create a new edition by adding a YAML to `variants/` — no code changes needed. + +## Database + +SQLite (`oracle.db`) with 784+ entries across 6 sources. Schema includes: +- Entry metadata (title, url, source, extracted_text, raw_metadata) +- Scoring (signal_score, component scores, final_score, actionability_score) +- Dedup/verdict (content_hash, verdict, source_tier) +- Summarization (summary JSON with one_liner, key_points, implications) +- Categorization (category_tags, bucket, narrative_id, topic_id) + +## Sprint Log + +### Sprint 0 — Foundation (2026-07-22) ✅ +- Package restructure: `oracle/` + `python -m oracle` CLI +- Per-adapter timeout (10s) + threading fallback +- Source confidence tiers (3-tier system) +- Content-hash dedup (SHA-256[:16]) +- Composite verdicts (PUBLISH/WATCH/ARCHIVE/DROP) +- Wired into pipeline: atomic hash/tier/verdict at insert time +- Consolidated 12 root scripts → thin wrappers + oracle/ package + +### Sprint 1 — Multi-variant Editions (2026-07-22) ✅ +- Variant engine: `oracle/variants.py` +- 4 default editions: default, research, devops, brief +- Variant-aware HTML/JSON renderer with theme support +- CLI: `render --variant`, `--all`, `--list`, `--webroot` +- Verified: all 4 variants render correctly with proper filtering + +## Dependencies + +- Python 3.10+ +- PyYAML (`pip install pyyaml`) +- Trafilatura (for text extraction in adapters) +- Feedparser (RSS feeds) + +## Backward Compatibility + +Old scripts (`pipeline.py`, `summarize.py`, `query.py`, etc.) are thin wrappers that delegate to the unified CLI. Existing cron jobs calling `python3 pipeline.py` continue to work without changes. + +## License + +AGPL v3 (see LICENSE for details) diff --git a/adapters/_store.py b/adapters/_store.py index 8ddb32f..553bfb9 100644 --- a/adapters/_store.py +++ b/adapters/_store.py @@ -89,8 +89,9 @@ def true_first_seen(raw_meta, source, now_str): UPSERT_SQL = """ INSERT INTO entries (source, source_id, url, title, extracted_text, summary, - category_tags, signal_score, raw_metadata, first_seen, last_updated) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + category_tags, signal_score, raw_metadata, first_seen, last_updated, + content_hash, source_tier, verdict) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(source, source_id) DO UPDATE SET source = excluded.source, source_id = excluded.source_id, @@ -101,6 +102,9 @@ ON CONFLICT(source, source_id) DO UPDATE SET signal_score = excluded.signal_score, raw_metadata = excluded.raw_metadata, last_updated = excluded.last_updated, + content_hash = excluded.content_hash, + source_tier = excluded.source_tier, + verdict = excluded.verdict, first_seen = COALESCE((SELECT first_seen FROM entries WHERE source = excluded.source AND source_id = excluded.source_id), excluded.first_seen) """ @@ -111,16 +115,38 @@ def upsert_entries(conn, entries): `entries` is the list of dicts as built by each adapter; each dict must already have first_seen set to the TRUE publish date (via true_first_seen) and last_updated to the harvest time. + + Now also sets content_hash and source_tier at insertion time. Returns count of rows written. """ + from oracle.dedup import content_hash, get_source_tier, compute_verdict, age_hours + cur = conn.cursor() written = 0 for e in entries: + # Compute content hash + title = e.get("title", "") + url = e.get("url", "") + body = e.get("extracted_text", "")[:500] + h = content_hash(title, url, body) + + # Get source tier + source = e.get("source", "") + tier_info = get_source_tier(source) + tier = tier_info["tier"] + + # Compute verdict + first_seen = e.get("first_seen", "") + score = float(e.get("signal_score") or 0) + age = age_hours(first_seen) + verdict = compute_verdict(score, age) + cur.execute(UPSERT_SQL, ( e["source"], e["source_id"], e["url"], e["title"], e.get("extracted_text"), e.get("summary"), e.get("category_tags"), e.get("signal_score"), e.get("raw_metadata"), e["first_seen"], e["last_updated"], + h, tier, verdict, )) written += 1 conn.commit() diff --git a/adapters/arxiv.py b/adapters/arxiv.py index adcb9fd..e0ff368 100644 --- a/adapters/arxiv.py +++ b/adapters/arxiv.py @@ -355,6 +355,18 @@ class ArxivAdapter(SourceAdapter): if paper.get("_applied_domain"): tags.append(paper["_applied_domain"]) + # Local-serving / efficient-inference signal (suggested source: arXiv cs.LG + # MoE/quantization papers → "local-serving" tag filter) + serving_kw = [ + "quantiz", "quantization", "moe", "mixture of experts", + "serving", "efficient inference", "pruning", "distill", + "knowledge distillation", "low-rank", "lora", "parameter-efficient", + "vram", "memory efficient", "edge inference", "on-device", + ] + combined = f"{title_lower} {summary_lower}" + if any(kw in combined for kw in serving_kw): + tags.append("local-serving") + return tags def fetch(self, query: str = "", limit: int = 20) -> list[dict]: diff --git a/adapters/reddit.py b/adapters/reddit.py index 25eb23d..96f3287 100644 --- a/adapters/reddit.py +++ b/adapters/reddit.py @@ -15,7 +15,9 @@ Strategy: 3s spacing between subreddits, retry with backoff. """ import json +import math import os +import random import re import time import urllib.request @@ -51,11 +53,11 @@ class RedditAdapter(SourceAdapter): "automoderator", } - def __init__(self, subreddits=None, rate_limit=1, user_agent=None): + def __init__(self, subreddits=None, rate_limit=4, user_agent=None): """ Args: subreddits: List of subreddit names. - rate_limit: Seconds between subreddit requests. + rate_limit: Base seconds between subreddit requests (with jitter). user_agent: Custom User-Agent header. """ self.subreddits = subreddits or self.DEFAULT_SUBREDDITS @@ -65,6 +67,12 @@ class RedditAdapter(SourceAdapter): def name(self) -> str: return "reddit" + def _sleep_with_jitter(self, base=None): + """Sleep with ±30% jitter to avoid pattern detection.""" + base = base or self.rate_limit + jitter = base * 0.3 * (2 * random.random() - 1) # ±30% + time.sleep(base + jitter) + def _clean_html(self, html: str) -> str: """Extract readable text from Reddit's HTML content.""" if not html: @@ -106,7 +114,8 @@ class RedditAdapter(SourceAdapter): url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50" req = urllib.request.Request(url, headers={"User-Agent": self.user_agent}) - for attempt in range(2): # max 2 attempts, fail fast + backoff = [3, 8] # staggered backoff: 3s then 8s + for attempt in range(3): # max 3 attempts try: with urllib.request.urlopen(req, timeout=10) as resp: xml_data = resp.read().decode("utf-8") @@ -116,8 +125,10 @@ class RedditAdapter(SourceAdapter): print(f" RSS blocked (HTTP {e.code}) for r/{subreddit}") return [] if e.code == 429: - if attempt == 0: - time.sleep(2) # single retry with short backoff + if attempt < len(backoff): + delay = backoff[attempt] + print(f" RSS 429 for r/{subreddit}, retrying in {delay}s") + time.sleep(delay) continue print(f" RSS rate-limited for r/{subreddit}, skip") return [] @@ -127,7 +138,7 @@ class RedditAdapter(SourceAdapter): print(f" RSS error r/{subreddit}: {e}") return [] else: - print(f" r/{subreddit}: still rate limited, skip") + print(f" r/{subreddit}: still rate limited after 3 attempts, skip") return [] # Parse Atom XML @@ -301,6 +312,17 @@ class RedditAdapter(SourceAdapter): ]): tags.append("meta:virality") + # Local-inference / on-device signal (suggested source: r/MachineLearning + # "I tried X on-device" posts — high builder signal → local-inference feed) + on_device_kw = [ + "on-device", "on device", "local inference", "local llm", + "ran locally", "running locally", "in my pocket", "on my phone", + "edge device", "offline", "no gpu", "consumer gpu", "rtx", + "single gpu", "self-host", "self host", + ] + if any(kw in title_lower or kw in content for kw in on_device_kw): + tags.append("local-inference") + return tags def _fetch_rss(self, subreddit: str) -> list[dict]: @@ -308,7 +330,8 @@ class RedditAdapter(SourceAdapter): url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50" req = urllib.request.Request(url, headers={"User-Agent": self.user_agent}) - for attempt in range(2): # max 2 attempts, fail fast + backoff = [3, 8] # staggered backoff: 3s then 8s + for attempt in range(3): # max 3 attempts try: with urllib.request.urlopen(req, timeout=10) as resp: xml_data = resp.read().decode("utf-8") @@ -318,8 +341,10 @@ class RedditAdapter(SourceAdapter): print(f" RSS blocked (HTTP {e.code}) for r/{subreddit}") return [] if e.code == 429: - if attempt == 0: - time.sleep(2) # single retry with short backoff + if attempt < len(backoff): + delay = backoff[attempt] + print(f" RSS 429 for r/{subreddit}, retrying in {delay}s") + time.sleep(delay) continue print(f" RSS rate-limited for r/{subreddit}, skip") return [] @@ -329,7 +354,7 @@ class RedditAdapter(SourceAdapter): print(f" RSS error r/{subreddit}: {e}") return [] else: - print(f" r/{subreddit}: still rate limited, skip") + print(f" r/{subreddit}: still rate limited after 3 attempts, skip") return [] # Parse Atom XML @@ -373,31 +398,26 @@ class RedditAdapter(SourceAdapter): Filters AutoModerator and sticky posts. """ now = datetime.now(timezone.utc) - # Try JSON first — if it's blocked on the first subreddit, bail fast - # rather than wasting time on all subreddits + # Try JSON first — if it's blocked on the first subreddit, skip + # the test-RSS call (which would waste a request and risk rate-limiting) + # and go straight to the RSS loop first_json = self._try_json(self.subreddits[0]) - if not first_json: - # JSON is blocked site-wide, try one RSS to confirm - test_rss = self._fetch_rss(self.subreddits[0]) - if not test_rss: - print(" Reddit blocked (403/429), returning empty") - return [] - # RSS works — fall through to full fetch below + json_worked = bool(first_json) all_entries = [] seen_ids = set() - json_worked = bool(first_json) - # Add first JSON results + # Add first JSON results if any for p in first_json: if p["id"] not in seen_ids: seen_ids.add(p["id"]) all_entries.append(p) - time.sleep(self.rate_limit) # If JSON didn't work, fall back to RSS for all subreddits if not json_worked: print(" JSON endpoints blocked, using RSS fallback") + # Brief cooldown before RSS barrage + time.sleep(3) for sub in self.subreddits: entries = self._fetch_rss(sub) for e in entries: @@ -422,7 +442,7 @@ class RedditAdapter(SourceAdapter): if entry["id"] not in seen_ids: seen_ids.add(entry["id"]) all_entries.append(entry) - time.sleep(self.rate_limit) + self._sleep_with_jitter() # Filter sticky/mod posts filtered = [] diff --git a/archive.py b/archive.py index e9beafd..1a22059 100644 --- a/archive.py +++ b/archive.py @@ -1,94 +1,11 @@ #!/usr/bin/env python3 -"""Oracle soft-cap archival. +"""Thin wrapper — delegates to oracle.cli archive subcommand.""" +import sys, os +sys.path.insert(0, os.path.dirname(__file__)) -Bounds live `entries` growth by moving old / excess rows into -`entries_archive` (preserving data — soft cap, not hard delete). -Two triggers: - --days N : move entries not updated in N days (default 30) - --cap N : if live entries exceed N, archive oldest beyond the cap (default 5000) -Default is a real run (data moves). Use --dry-run to report only. -""" +args = sys.argv[1:] +cli_args = ["archive"] + args -import argparse -import os -import sqlite3 -import time - -DB = os.path.join(os.path.dirname(__file__), "oracle.db") - -ARCHIVE_SCHEMA = """ -CREATE TABLE IF NOT EXISTS entries_archive ( - id INTEGER PRIMARY KEY, - source TEXT, source_id TEXT, url TEXT, title TEXT, - extracted_text TEXT, summary TEXT, category_tags TEXT, - signal_score REAL, raw_metadata TEXT, - first_seen TEXT, last_updated TEXT, - archived_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')) -); -""" - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--days", type=int, default=30, help="Archive entries not updated in N days") - ap.add_argument("--cap", type=int, default=5000, help="Soft cap on live entries; archive oldest beyond this") - ap.add_argument("--dry-run", action="store_true", help="Report only, make no changes") - args = ap.parse_args() - - if not os.path.exists(DB): - print("No oracle.db — nothing to archive") - return - - conn = sqlite3.connect(DB) - conn.execute(ARCHIVE_SCHEMA) - - cutoff = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - args.days * 86400)) - - cur = conn.cursor() - cur.execute("SELECT id, last_updated FROM entries") - rows = cur.fetchall() - n_total = len(rows) - - old_ids = [r[0] for r in rows if (r[1] or "") < cutoff] - beyond = max(0, n_total - args.cap) - if beyond > 0: - ordered = sorted(rows, key=lambda r: r[1] or "")[:beyond] - cap_ids = [r[0] for r in ordered] - else: - cap_ids = [] - - move_ids = sorted(set(old_ids) | set(cap_ids)) - - if not move_ids: - print(f"Archive check: {n_total} live entries, none older than {args.days}d " - f"or beyond cap {args.cap}. Nothing to archive.") - conn.close() - return - - print(f"Archive check: {n_total} live entries -> would archive {len(move_ids)} " - f"(old={len(old_ids)}, cap={len(cap_ids)}).") - - if args.dry_run: - print("DRY RUN — no changes made.") - conn.close() - return - - q = ",".join("?" * len(move_ids)) - conn.execute( - f"""INSERT OR REPLACE INTO entries_archive - (id, source, source_id, url, title, extracted_text, summary, - category_tags, signal_score, raw_metadata, first_seen, last_updated) - SELECT id, source, source_id, url, title, extracted_text, summary, - category_tags, signal_score, raw_metadata, first_seen, last_updated - FROM entries WHERE id IN ({q})""", - move_ids, - ) - conn.execute(f"DELETE FROM entries WHERE id IN ({q})", move_ids) - conn.commit() - conn.close() - print(f"Archived {len(move_ids)} entries (live now {n_total - len(move_ids)}). " - f"Preserved in entries_archive.") - - -if __name__ == "__main__": - main() +from oracle.cli import main as cli_main +sys.argv = ["oracle"] + cli_args +cli_main() diff --git a/_engagement_analysis.py b/archive/_engagement_analysis.py similarity index 100% rename from _engagement_analysis.py rename to archive/_engagement_analysis.py diff --git a/_live_compare.py b/archive/_live_compare.py similarity index 100% rename from _live_compare.py rename to archive/_live_compare.py diff --git a/reddit_proof.py b/archive/reddit_proof.py similarity index 100% rename from reddit_proof.py rename to archive/reddit_proof.py diff --git a/athena/__init__.py b/athena/__init__.py deleted file mode 100644 index ccc8b91..0000000 --- a/athena/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# athena package — Sprint 1 pure-rule scoring lives in scoring.py diff --git a/athena/scoring.py b/athena/scoring.py deleted file mode 100644 index d0c70c5..0000000 --- a/athena/scoring.py +++ /dev/null @@ -1,545 +0,0 @@ -#!/usr/bin/env python3 -""" -athena/scoring.py — Sprint 1: Pure Rule-Based Bucket Classifier + Scorer. - -DESIGN CONSTRAINT (founder directive, 2026-07-15): - Pure rules only. No embeddings, no semantic similarity, no LLM classification. - We are still discovering the editorial taxonomy. Deterministic systems are - easier to debug; misclassifications are signal; we must be able to explain - WHY every story landed where it did before we add intelligence. - -Pipeline position: - ingestion/dedup (pipeline.py) -> [ATTACH HERE] -> rendering - Call attach_scoring(conn) immediately after store_entries and before render. - -Schema migration (idempotent): - bucket, shipping_score, utility_score, replication_score, enthusiast_score, - novelty_score, hype_penalty, final_score, actionability_score, - narrative_id, topic_id, relation_json - -Review report: human-readable, per-bucket, exposes every fired rule. - -Usage: - python3 athena/scoring.py --migrate # add columns - python3 athena/scoring.py --backfill # score all unscored rows - python3 athena/scoring.py --review # write review report - python3 athena/scoring.py --all # migrate + backfill + review -""" - -import argparse -import json -import os -import re -import sqlite3 -from datetime import datetime, timezone - -HERE = os.path.dirname(os.path.abspath(__file__)) -DB_PATH = os.path.join(os.path.dirname(HERE), "oracle.db") # ~/oracle/oracle.db - -# --------------------------------------------------------------------------- -# SCORE WEIGHTS (transparent, tunable in one place) -# --------------------------------------------------------------------------- -WEIGHTS = { - "shipping": 0.20, # a working artifact exists - "utility": 0.20, # clear practical use for an enthusiast - "replication": 0.25, # can a reader reproduce/run it locally - "enthusiast": 0.20, # signals a builder/DIY practitioner audience - "novelty": 0.15, # new, specific, non-generic -} -HYPE_CAP = 0.45 # final_score is floored if hype penalty is high - -# --------------------------------------------------------------------------- -# BUCKETS + PURE RULES -# Each bucket: keyword hits and/or source constraints. First match wins, -# evaluated in BUCKET_ORDER (most specific taxon first). -# --------------------------------------------------------------------------- -# Keyword sets (lowercased; matched against title + summary + tags text). -KW_SHIPPING = [ - "released", "launch", "v1.0", "v2.0", "v3.0", "shipping", "now available", - "open source", "open-source", "open weights", "weights released", "live now", - "beta", "public beta", "ga release", "general availability", "ships", "deployed", - "production", "now in", "available today", "download", "gradio", "demo", "playground", -] -KW_LOCAL_AI = [ - "local llm", "local model", "local ai", "run locally", "run it locally", "on-device", - "on device", "ollama", "llama.cpp", "llamacpp", "gguf", "ggml", "lm studio", - "consumer hardware", "consumer gpu", "rtx", "your own gpu", "offline", "private ai", - "local-only", "self-host", "self-hosted", "home server", "edge device", "edge inference", - "quantized", "quantization", "q4", "q8", "int4", "fp16", "fine-tune at home", - "train at home", "local inference", "local deployment", "no api", "no cloud", -] -KW_PROBLEM_SOLVED = [ - "how to", "how i", "solved", "fix", "fixed", "workaround", "benchmark", "improves", - "improved", "speedup", "speed-up", "reduces", "reduce", "cut", "cuts", "boost", - "optimize", "optimized", "optimisation", "faster", "3x", "10x", "2x", "latency", - "throughput", "roi", "cost", "cheaper", "save", "saves", "eliminate", "eliminated", - "from 117s to 30s", "p95", "memory usage", "vram", "token cost", "bottleneck", - "case study", "results", "we measured", "we tested", "showdown", "comparison", -] -KW_MODEL_RELEASE = [ - "releases", "released", "unveils", "introduces", "new model", "new flagship", - "gpt-", "claude", "gemini", "llama", "mistral", "qwen", "deepseek", "grok", - "phi-", "command-r", "api access", "weights", "open model", "open-models", - "frontier", "checkpoint", "fine-tune", "finetune", "rl-trained", "rl train", - "trained", "post-training", "post training", "distilled", "distillation", -] -KW_RESEARCH = [ - "paper", "arxiv", "preprint", "study", "research", "we propose", "we present", - "we introduce", "we show", "method", "framework", "theorem", "analysis of", - "survey", "benchmark", "dataset", "neural", "transformer", "diffusion", - "gradient", "ablation", "we find", "our approach", "novel", "state-of-the-art", - "sota", "cs.lg", "cs.cl", "cs.cv", "cs.ai", -] -KW_BUSINESS = [ - "raises", "raised", "$", "valuation", "series a", "series b", "funding", "round", - "ipo", "acquisition", "acquires", "merger", "deal", "revenue", "layoff", "hiring", - "partnership", "invests", "investment", "market", "vc", "compute deal", - "billion", "million", "forecast", "miss", "earnings", "stock", -] -KW_INFRA = [ - "gpu", "tpu", "data center", "datacenter", "data centre", "cluster", "cuda", - "rocm", "vllm", "tensorrt", "inference server", "serving", "kubernetes", "docker", - "pipeline", "mlops", "ci/cd", "rag", "vector db", "vector database", "agent", - "agents", "orchestration", "observability", "evaluation", "eval", "guardrail", - "safety", "red team", "jailbreak", "prompt injection", "fine-tuning stack", -] -KW_CULTURE = [ - "says", "argues", "opinion", "essay", "think", "thinks", "the real", "why we", - "the future of", "dystopia", "utopia", "philosophy", "ethics", "regulation", - "policy", "ban", "lawsuit", "eu", "senate", "congress", "interview", "podcast", - "controversy", "controversial", "debate", "debate", "critic", "criticism", - "creepy", "creeping", "not sexy", "vibe", "hot take", "unpopular", -] - -# Buckets evaluated in this order (specific -> generic). source_whitelist matches -# raw `source` value exactly; if present, story must come from one of those sources. -BUCKETS = { - "SHIPPING": { - "kw": KW_SHIPPING, - "source_whitelist": None, - "order": 0, - }, - "LOCAL AI": { - "kw": KW_LOCAL_AI, - "source_whitelist": None, - "order": 1, - }, - "PROBLEM SOLVED": { - "kw": KW_PROBLEM_SOLVED, - "source_whitelist": None, - "order": 2, - }, - "MODEL RELEASE": { - "kw": KW_MODEL_RELEASE, - "source_whitelist": None, - "order": 3, - }, - "RESEARCH": { - "kw": KW_RESEARCH, - "source_whitelist": ["arxiv"], - "order": 4, - }, - "BUSINESS": { - "kw": KW_BUSINESS, - "source_whitelist": None, - "order": 5, - }, - "INFRASTRUCTURE": { - "kw": KW_INFRA, - "source_whitelist": None, - "order": 6, - }, - "CULTURE": { - "kw": KW_CULTURE, - "source_whitelist": None, - "order": 7, - }, -} -BUCKET_ORDER = sorted(BUCKETS.keys(), key=lambda b: BUCKETS[b]["order"]) - -# Hype / low-signal penalty terms -HYPE_TERMS = [ - "revolutionary", "game-changing", "game changer", "breakthrough", "mind-blowing", - "insane", "crazy", "unbelievable", "shocking", "the future is here", "omg", - "you won't believe", "secret", "they don't want you to know", "leaked", "viral", - "hype", "buzzword", "disrupt", "disrupting everything", "ai will replace", - "will change everything", "paradigm shift", "godlike", "magic", "miracle", -] - -# Enthusiast-audience signals (builder / DIY / practitioner) -ENTHUSIAST_SIGNALS = [ - "github", "repo", "repository", "self-host", "local", "ollama", "llamacpp", - "hugging face", "huggingface", "colab", "notebook", "pip install", "docker", - "cli", "open source", "open-source", "diy", "build your own", "tutorial", - "how to", "implementation", "agent", "agents", "fine-tune", "finetune", - "quantiz", "vllm", "rtx", "gpu", "consumer", "homelab", "self-hosted", - "machine-learning", "machine learning", "deep learning", "python", "rust", - "benchmark", "reproduc", "weights", "gguf", -] - -# Source -> enthusiast affinity bonus -SOURCE_ENTHUSIAST_BONUS = { - "github": 0.20, "huggingface": 0.20, "arxiv": 0.10, - "hackernews": 0.10, "reddit": 0.05, "rss": 0.0, -} - - -def _norm(text): - if not text: - return "" - if isinstance(text, bytes): - text = text.decode("utf-8", "replace") - return " " + re.sub(r"\s+", " ", text.lower()) + " " - - -def classify(entry): - """Pure rule classification. - - Returns (bucket, matched_list) where matched_list is human-readable proof: - ["kw:ollama", "kw:gguf", "src:huggingface", "tag:local"] - """ - title = _norm(entry.get("title") or "") - summary = _norm(_summary_text(entry.get("summary"))) - tags_raw = entry.get("category_tags") or "" - try: - tags = " ".join(json.loads(tags_raw)) if tags_raw else "" - except Exception: - tags = tags_raw - tags = _norm(tags) - source = (entry.get("source") or "").lower() - haystack = title + " " + summary + " " + tags - - matched = ["source=%s" % source] - best_bucket = "UNCATEGORIZED" - best_hits = 0 - - for bucket in BUCKET_ORDER: - spec = BUCKETS[bucket] - whitelist = spec["source_whitelist"] - if whitelist and source not in whitelist: - continue - hits = [] - for kw in spec["kw"]: - kwn = " " + kw.lower() + " " - if kwn in haystack: - hits.append(kw) - if hits: - # record proof (cap displayed hits to keep report readable) - for h in hits[:8]: - matched.append("kw:%s" % h) - if len(hits) > best_hits: - best_hits = len(hits) - best_bucket = bucket - - if best_bucket == "UNCATEGORIZED": - matched.append("(no rule fired)") - - return best_bucket, matched - - -def _summary_text(raw): - if not raw: - return "" - try: - d = json.loads(raw) - if isinstance(d, dict): - return " ".join(str(v) for v in d.values() if isinstance(v, str)) - except Exception: - pass - return raw - - -def score_entry(bucket, matched, entry): - """Return dict of component scores (0..1) + final (0..1). - - Components are deterministic functions of signals; see inline rationale. - """ - source = (entry.get("source") or "").lower() - title = _norm(entry.get("title") or "") - summary = _norm(_summary_text(entry.get("summary"))) - tags_raw = entry.get("category_tags") or "" - try: - tags = " ".join(json.loads(tags_raw)) if tags_raw else "" - except Exception: - tags = tags_raw - tags = _norm(tags) - haystack = title + " " + summary + " " + tags - - # --- enthusiast score --- - ent_hits = sum(1 for s in ENTHUSIAST_SIGNALS if (" " + s + " ") in haystack) - ent_base = min(ent_hits / 5.0, 1.0) # 5+ signals = full - ent_src = SOURCE_ENTHUSIAST_BONUS.get(source, 0.0) - enthusiast = min(ent_base + ent_src, 1.0) - - # --- shipping score --- - ship_kw = [k for k in KW_SHIPPING if (" " + k + " ") in haystack] - shipping = 0.0 - if bucket == "SHIPPING": - shipping = 0.9 - elif ship_kw: - shipping = min(0.4 + 0.1 * len(ship_kw), 0.8) - # source artifacts (github/hf) imply something shippable exists - if source in ("github", "huggingface"): - shipping = max(shipping, 0.7) - - # --- utility score --- - util_kw = [k for k in KW_PROBLEM_SOLVED if (" " + k + " ") in haystack] - utility = 0.0 - if bucket == "PROBLEM SOLVED": - utility = 0.85 - elif util_kw: - utility = min(0.4 + 0.1 * len(util_kw), 0.8) - if "github" in haystack or "huggingface" in haystack or "demo" in haystack: - utility = max(utility, 0.6) - - # --- replication score (can a reader reproduce/run locally) --- - repl_kw = [k for k in KW_LOCAL_AI if (" " + k + " ") in haystack] - replication = 0.0 - if bucket == "LOCAL AI": - replication = 1.0 - elif repl_kw: - replication = min(0.5 + 0.1 * len(repl_kw), 0.9) - if source in ("github", "huggingface"): - replication = max(replication, 0.7) - if "open source" in haystack or "open-source" in haystack or "weights" in haystack: - replication = max(replication, 0.6) - - # --- novelty score --- - nov_kw = ["new", "novel", "first", "breakthrough-method", "we propose", - "we introduce", "we present", "state-of-the-art", "sota", "unveils"] - novelty = 0.0 - if bucket in ("RESEARCH", "MODEL RELEASE"): - novelty = 0.6 - if any((" " + k + " ") in haystack for k in nov_kw): - novelty = min(novelty + 0.2, 0.9) - if bucket == "CULTURE": - novelty = min(novelty, 0.3) # opinion pieces are rarely novel technically - - # --- hype penalty --- - hype_hits = [t for t in HYPE_TERMS if (" " + t + " ") in haystack] - hype_penalty = min(0.1 * len(hype_hits), 0.6) - - # --- final --- - raw = ( - WEIGHTS["shipping"] * shipping - + WEIGHTS["utility"] * utility - + WEIGHTS["replication"] * replication - + WEIGHTS["enthusiast"] * enthusiast - + WEIGHTS["novelty"] * novelty - ) - final = max(raw - hype_penalty, 0.0) - final = min(final, 1.0) - - return { - "shipping_score": round(shipping, 3), - "utility_score": round(utility, 3), - "replication_score": round(replication, 3), - "enthusiast_score": round(enthusiast, 3), - "novelty_score": round(novelty, 3), - "hype_penalty": round(hype_penalty, 3), - "final_score": round(final, 4), - } - - -# --------------------------------------------------------------------------- -# DB OPERATIONS -# --------------------------------------------------------------------------- -NEW_COLUMNS = [ - "bucket TEXT", - "shipping_score REAL DEFAULT 0", - "utility_score REAL DEFAULT 0", - "replication_score REAL DEFAULT 0", - "enthusiast_score REAL DEFAULT 0", - "novelty_score REAL DEFAULT 0", - "hype_penalty REAL DEFAULT 0", - "final_score REAL DEFAULT 0", - "actionability_score REAL DEFAULT 0", - "narrative_id TEXT", - "topic_id TEXT", - "relation_json TEXT", -] - - -def migrate(db_path=DB_PATH): - """Idempotent schema migration — only adds missing columns.""" - conn = sqlite3.connect(db_path) - cur = conn.cursor() - cur.execute("PRAGMA table_info(entries)") - existing = {row[1] for row in cur.fetchall()} - added = [] - for col in NEW_COLUMNS: - name = col.split(" ")[0] - if name not in existing: - cur.execute("ALTER TABLE entries ADD COLUMN %s" % col) - added.append(name) - conn.commit() - conn.close() - print("[migrate] added columns: %s" % (", ".join(added) if added else "none (already present)")) - return added - - -def fetch_unscored(conn): - cur = conn.cursor() - cur.execute("""SELECT id, source, source_id, url, title, summary, category_tags, - raw_metadata FROM entries WHERE bucket IS NULL OR bucket = ''""") - cols = ["id", "source", "source_id", "url", "title", "summary", - "category_tags", "raw_metadata"] - return [dict(zip(cols, row)) for row in cur.fetchall()] - - -def attach_scoring(db_path=DB_PATH, dry_run=False): - """Score every unscored entry. Call after ingestion/dedup, before render.""" - conn = sqlite3.connect(db_path) - rows = fetch_unscored(conn) - print("[attach] scoring %d unscored entries" % len(rows)) - for e in rows: - bucket, matched = classify(e) - scores = score_entry(bucket, matched, e) - if dry_run: - continue - conn.execute( - """UPDATE entries SET bucket=?, shipping_score=?, utility_score=?, - replication_score=?, enthusiast_score=?, novelty_score=?, - hype_penalty=?, final_score=?, actionability_score=?, - narrative_id=?, topic_id=?, relation_json=? WHERE id=?""", - (bucket, scores["shipping_score"], scores["utility_score"], - scores["replication_score"], scores["enthusiast_score"], - scores["novelty_score"], scores["hype_penalty"], scores["final_score"], - 0.0, # actionability_score: reserved, unused in Sprint 1 - None, None, json.dumps({"matched_rules": matched}), e["id"]), - ) - if not dry_run: - conn.commit() - conn.close() - print("[attach] done.") - - -# --------------------------------------------------------------------------- -# REVIEW REPORT -# --------------------------------------------------------------------------- -REVIEW_BUCKETS = ["SHIPPING", "LOCAL AI", "PROBLEM SOLVED", "MODEL RELEASE", - "RESEARCH", "BUSINESS", "INFRASTRUCTURE", "CULTURE", - "UNCATEGORIZED"] - - -def generate_review(db_path=DB_PATH, limit=200, out_path=None): - conn = sqlite3.connect(db_path) - cur = conn.cursor() - cur.execute("""SELECT id, source, title, url, bucket, final_score, - relation_json, shipping_score, utility_score, - replication_score, enthusiast_score, novelty_score, - hype_penalty FROM entries - ORDER BY first_seen DESC LIMIT ?""", (limit,)) - rows = cur.fetchall() - conn.close() - - by_bucket = {b: [] for b in REVIEW_BUCKETS} - for r in rows: - (eid, src, title, url, bucket, final, rel_json, sh, ut, rp, en, no, hy) = r - try: - matched = json.loads(rel_json).get("matched_rules", []) if rel_json else [] - except Exception: - matched = [] - by_bucket.setdefault(bucket or "UNCATEGORIZED", []).append( - (eid, src, title, url, final, matched, (sh, ut, rp, en, no, hy)) - ) - - lines = [] - lines.append("=" * 70) - lines.append("ATHENA SPRINT 1 — MANUAL REVIEW REPORT") - lines.append("Generated: %s" % datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")) - lines.append("Stories reviewed: %d (most recent %d)" % (len(rows), limit)) - lines.append("=" * 70) - lines.append("") - lines.append("HOW TO READ: For each story, the fired rules are listed as proof.") - lines.append("Bucket = first matching taxon (specific -> generic). Scores are") - lines.append("deterministic. 'Publish?' is for HUMAN review only — not algorithmic.") - lines.append("") - - total_pub = total_rej = total_border = 0 - - for b in REVIEW_BUCKETS: - items = by_bucket.get(b, []) - if not items: - continue - lines.append(b) - lines.append("-" * len(b)) - for (eid, src, title, url, final, matched, comps) in items: - sh, ut, rp, en, no, hy = comps - lines.append("") - lines.append("Story: %s" % (title or "(untitled)")) - lines.append(" id=%s source=%s final=%.3f" % (eid, src, final)) - lines.append(" url: %s" % (url or "")) - lines.append(" Bucket: %s" % b) - lines.append(" Matched:") - for m in matched: - lines.append(" - %s" % m) - lines.append(" Score Components:") - lines.append(" Shipping: %.2f" % sh) - lines.append(" Utility: %.2f" % ut) - lines.append(" Replication: %.2f" % rp) - lines.append(" Enthusiast: %.2f" % en) - lines.append(" Novelty: %.2f" % no) - lines.append(" Hype: %.2f" % hy) - lines.append(" Final: %.3f" % final) - lines.append(" Publish? [Y/N] <- human review only") - lines.append("") - lines.append("") - - # Summary block (the founder's key metric) - lines.append("=" * 70) - lines.append("BUCKET DISTRIBUTION") - lines.append("=" * 70) - for b in REVIEW_BUCKETS: - n = len(by_bucket.get(b, [])) - if n: - lines.append(" %-16s %3d" % (b, n)) - lines.append("") - lines.append("HUMAN REVIEW TALLY (fill in after manual pass):") - lines.append(" Published: %d" % total_pub) - lines.append(" Rejected: %d" % total_rej) - lines.append(" Borderline: %d" % total_border) - lines.append("") - lines.append("First question is not 'is the classifier accurate?'") - lines.append("First question: 'Would we proudly publish these stories?'") - lines.append("=" * 70) - - report = "\n".join(lines) + "\n" - if out_path is None: - out_path = os.path.join(os.path.dirname(HERE), "athena_review_report.txt") - with open(out_path, "w") as f: - f.write(report) - print("[review] wrote %s (%d stories)" % (out_path, len(rows))) - # also print to stdout for immediate visibility - print(report) - return report - - -def main(): - ap = argparse.ArgumentParser(description="Athena Sprint 1 scoring (pure rules)") - ap.add_argument("--migrate", action="store_true", help="add new columns") - ap.add_argument("--backfill", action="store_true", help="score all unscored rows") - ap.add_argument("--review", action="store_true", help="write manual review report") - ap.add_argument("--all", action="store_true", help="migrate + backfill + review") - ap.add_argument("--limit", type=int, default=200, help="review story count") - ap.add_argument("--dry-run", action="store_true", help="classify but don't write") - ap.add_argument("--db", default=DB_PATH, help="db path override") - args = ap.parse_args() - - if args.all: - migrate(args.db) - attach_scoring(args.db, dry_run=args.dry_run) - generate_review(args.db, limit=args.limit) - else: - if args.migrate: - migrate(args.db) - if args.backfill: - attach_scoring(args.db, dry_run=args.dry_run) - if args.review: - generate_review(args.db, limit=args.limit) - if not (args.migrate or args.backfill or args.review): - ap.print_help() - - -if __name__ == "__main__": - main() diff --git a/clickability.py b/clickability.py index 7edd6d4..fbb695b 100644 --- a/clickability.py +++ b/clickability.py @@ -1,300 +1,8 @@ #!/usr/bin/env python3 -"""Clickability Index for Athena entries. +"""Thin wrapper — renders clickability index preview (render --dry-run).""" +import sys, os +sys.path.insert(0, os.path.dirname(__file__)) -CORRECTED for the REAL schema (verified 2026-07-10): - - Table is `entries`, not `items`. - - Per-source engagement lives inside the `raw_metadata` JSON blob, not - top-level `velocity_raw` / `engagement_raw` columns. - - `content_type` is COMPUTED, not stored. - -This module is read-only against the DB (SELECT only). It does not -modify oracle.db. - -The MULTIPLIERS and formula match the approved plan exactly. -""" -import sqlite3, json, math, re, os, time -from datetime import datetime, timezone -from collections import defaultdict - -DB_PATH = os.path.join(os.path.dirname(__file__), "oracle.db") - -MULTIPLIERS = {} # category multipliers removed: clickability is now virality-driven, not category-driven - -# Virality weights (clickability = how viral/spreadable an item is right now) -VEL_W = 0.50 -ENG_W = 0.50 -SIG_W = 0.0 # signal_score no longer in the clickability blend (pure virality) - -NOW = None # set in main/fetch for age math - - -def get_connection(): - return sqlite3.connect(DB_PATH) - - -def _classify(src, title, summary): - t = (title + " " + (summary or "")).lower() - # Show HN — check first (builder posts) - if re.search(r"\bshow\s+hn\b", t) or (src == "hackernews" and re.search(r"\b(show|built|made|launched|shipped)\b", t)): - return "SHOW_HN" - # Model release — pattern-based (works for third-party coverage too) - if re.search(r"\b(gpt-|gpt5|gpt-5|deepseek|glm-|llama|qwen|claude|gemini|mistral|flux|stable-diffusion|sora|kimi|grok)\b", t) \ - and re.search(r"\b(releases?|released|v\d|launch|unveil|model|new\s+model|update|version)\b", t): - return "MODEL_RELEASE" - if re.search(r"\b(releases?|released|launches?|unveils?|announces?|debut|new\s+model|gpt-5|deepseek-v|glm-5)\b", t) \ - and re.search(r"\b(openai|anthropic|google|meta|microsoft|nvidia|ai)\b", t): - return "MODEL_RELEASE" - # HF model cards - if src == "huggingface": - return "MODEL_CARD" - # Research papers - if src == "arxiv" or re.search(r"\b(paper|study|benchmark|arxiv|proposes|learns?|novel|framework\s+for|towards)\b", t): - return "RESEARCH" - # Business/legal - if re.search(r"\b(sues|lawsuit|funding|raises|acqui|ipo|valued|stealing|trade secret|layoff|hire[ds]?|exec|ceo)\b", t) \ - and not re.search(r"\b(repo|library|tool|agent framework)\b", t): - return "BUSINESS_LEGAL" - # Opinion/essay - if re.search(r"\b(burnout|opinion|think|feel|why|essay|culture|linkedin|social media|future of|we made|i think|hot take|i believe|my view|in defense)\b", t): - return "CULTURE_OPINION" - # Tutorial/howto - if re.search(r"\b(how to|tutorial|guide|running|build|setup|install|from scratch|learn)\b", t): - return "TUTORIAL_HOWTO" - # Dev tools - if src == "github" or re.search(r"\b(repo|library|framework|tool|agent|sdk|cli|extension|plugin|app|engine)\b", t): - return "DEV_TOOL_DRAMA" - return "OTHER" - - -def _extract(src, md): - """Return (velocity_raw, engagement_raw, age_hours).""" - if src == "hackernews": - pts = md.get("score", 0) or 0 - cmts = md.get("descendants", 0) or 0 - age_h = None - if md.get("time"): - try: - age_h = max((NOW - md["time"]) / 3600.0, 0.1) - except Exception: - age_h = None - vel = (pts / age_h) if age_h else pts - return vel, (pts + 2 * cmts), age_h - if src == "reddit": - ups = md.get("ups", 0) or 0 - cmts = md.get("num_comments", 0) or 0 - return ups, (ups + 2 * cmts), None - if src == "huggingface": - likes = md.get("likes", 0) or 0 - return likes, likes, None - if src == "github": - spd = md.get("stars_per_day", 0) or 0 - stars = md.get("stars", 0) or 0 - return spd, stars, None - if src == "arxiv": - return 0.0, 0.0, None - return 0.0, 0.0, None - - -def fetch_items(conn): - global NOW - NOW = __import__("time").time() - cur = conn.cursor() - cur.execute("SELECT id, title, url, source, summary, signal_score, raw_metadata, first_seen, " - "curated_by, manual_section, manual_tier " - "FROM entries") - cols = [d[0] for d in cur.description] - out = [] - for row in cur.fetchall(): - d = dict(zip(cols, row)) - try: - md = json.loads(d.get("raw_metadata") or "{}") - except Exception: - md = {} - vel, eng, age = _extract(d["source"], md) - ct = _classify(d["source"], d.get("title") or "", d.get("summary") or "") - created_at = md.get("createdAt") if d["source"] == "huggingface" else None - out.append({ - "id": d["id"], - "title": d.get("title") or "", - "url": d.get("url") or "", - "source": d["source"], - "summary": d.get("summary") or "", - "signal_score": d.get("signal_score") or 0, - "velocity_raw": vel, - "engagement_raw": eng, - "content_type": ct, - "first_seen": d.get("first_seen") or "", - "created_at": created_at or "", - "age_hours": 0.0, - "curated_by": d.get("curated_by") or "", - "manual_section": d.get("manual_section") or "", - "manual_tier": d.get("manual_tier") or "", - }) - return out - - -def log1p_norm(values): - log_vals = [math.log1p(max(v, 0)) for v in values] - if not log_vals: - return [] - min_v, max_v = min(log_vals), max(log_vals) - if max_v == min_v: - return [0.0] * len(values) - return [(v - min_v) / (max_v - min_v) for v in log_vals] - - -def compute_index(items): - velocities = [it.get("velocity_raw", 0) or 0 for it in items] - engagements = [it.get("engagement_raw", 0) or 0 for it in items] - signals = [it.get("signal_score", 0) or 0 for it in items] - - vel_norm = log1p_norm(velocities) - eng_norm = log1p_norm(engagements) - sig_norm = log1p_norm(signals) - - for i, item in enumerate(items): - raw = vel_norm[i] * VEL_W + eng_norm[i] * ENG_W + sig_norm[i] * SIG_W - # Items with 0 engagement (arXiv, RSS, Reddit no-data) get a small base score - # from signal_score so they can decay naturally instead of being stuck forever. - # Floor: 0.05 * signal_score_norm — enough to rank, low enough to sink fast. - if raw == 0 and sig_norm[i] > 0: - raw = 0.05 * sig_norm[i] - item["clickability"] = round(raw, 4) - item["section"] = "" # sections removed; flat ranked feed - return items - - -def _age_hours(item): - """Effective news-age in hours. - - HuggingFace items are aged by their TRUE model createdAt (likes/downloads - are lifetime cumulative, so DB first_seen would pin every HF entry at - ingest time and let all-time leaders dominate 'Top News' forever). All - other sources are aged by DB first_seen. - """ - if item.get("source") == "huggingface" and item.get("created_at"): - s = item["created_at"] - else: - s = item.get("first_seen") or "" - if not s: - return 0.0 - try: - ts = datetime.strptime(s[:19], "%Y-%m-%dT%H:%M:%S").replace( - tzinfo=timezone.utc).timestamp() - return max((time.time() - ts) / 3600.0, 0.0) - except Exception: - return 0.0 - - -# Category-specific half-lives (hours) — controls how long each type stays competitive. -# Breaking news decays slowest (stays relevant longer), arXiv/model cards fastest. -CATEGORY_HALF_LIVES = { - "breaking": 36.0, # Red — truly groundbreaking, double the standard - "update": 24.0, # Green — important but not groundbreaking, between red and black - "OTHER": 18.0, # Black — standard decay rate -} - -# Map content_type to half-life, with tier override for breaking/update -def _get_half_life(item): - """Return half-life in hours based on tier and content_type.""" - # Hardware/Tips section items decay on a 14-DAY half-life (user: revised - # spec, shorter than evergreen). Covers both manual curations and - # auto-classified section items, so a section link persists 14 days - # instead of sinking in ~18h. Beyond this window the item is routed to - def _get_half_life(item): - """Return section/tier-specific half-life in hours, or None to fall back - to decay_index's half_life_h argument (default 18h news half-life).""" - # Spec v1 addendum: Hardware/Tips section items get a 14-day (336h) - # half-life so curated/section items persist far longer in-section. - ms = (item.get("manual_section") or "").upper() - if ms in ("HARDWARE", "TIPS"): - return 336.0 - tier = item.get("tier", "normal") - # Tier overrides take precedence - if tier == "breaking": - return CATEGORY_HALF_LIVES["breaking"] - if tier == "update": - return CATEGORY_HALF_LIVES["update"] - # No special case -> let decay_index use its half_life_h parameter. - return None - - -def decay_index(items, half_life_h=18.0): - """Apply exponential time-decay to clickability so items sink as they age. - - Uses category-specific half-lives: breaking news decays slowest (36h), - arXiv/model cards fastest (12h). This controls how long each type - stays competitive, not just starting score. - - decayed = clickability * exp(-ln(2)/half_life * age_hours) - """ - # Rolling 24h freshness window (not calendar-day) so Top News stays populated - # between the daily harvest and midnight UTC. Decay still sinks old items. - cutoff = datetime.now(timezone.utc).timestamp() - 24 * 3600 - for it in items: - age = _age_hours(it) - it["age_hours"] = round(age, 1) - base = it.get("clickability", 0) or 0 - # Category-specific half-life, falling back to the passed half_life_h - hl = _get_half_life(it) - if hl is None: - hl = half_life_h - k = math.log(2) / hl - it["clickability_decayed"] = round(base * math.exp(-k * age), 4) - it["effective_half_life"] = hl - # Freshness flag: ingested within the last 24h -> eligible for Top News. - fs = it.get("first_seen") or "" - try: - ts = datetime.fromisoformat(fs.replace("Z", "+00:00")).timestamp() - except ValueError: - ts = 0 - it["fresh"] = ts >= cutoff - return items - - -def _pearson(xs, ys): - n = len(xs) - if n < 3: - return None - mx, my = sum(xs) / n, sum(ys) / n - num = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) - den = math.sqrt(sum((x - mx) ** 2 for x in xs) * sum((y - my) ** 2 for y in ys)) - return num / den if den else None - - -def main(): - conn = get_connection() - items = fetch_items(conn) - conn.close() - if not items: - print("No items found.") - return - - computed = compute_index(items) - computed.sort(key=lambda x: x["clickability"], reverse=True) - - print(f"=== TOP 20 BY CLICKABILITY INDEX (n={len(items)} items) ===\n") - for i, item in enumerate(computed[:20], 1): - print(f"{i:2}. [{item['clickability']:.4f}] {item['source']:11} | {item['title'][:58]}") - print(f" section={item['section']} | type={item['content_type']} | " - f"vel={item['velocity_raw']:.1f} eng={item['engagement_raw']:.1f} sig={item['signal_score']:.2f}") - - # Backtest: Clickability Index vs ACTUAL HN engagement - hn = [it for it in computed if it["source"] == "hackernews" and it["engagement_raw"] > 0] - if hn: - r_full = _pearson([it["engagement_raw"] for it in hn], - [it["clickability"] for it in hn]) - # Honest baseline: signal_score alone vs HN engagement (legacy prior) - r_sig = _pearson([it["engagement_raw"] for it in hn], - [it["signal_score"] for it in hn]) - print(f"\n--- BACKTEST (HN, n={len(hn)}) ---") - print(f"ClickabilityIndex vs actual HN engagement : r = {r_full:.3f}" if r_full is not None else "r = n/a") - print(f"signal_score alone vs HN engagement : r = {r_sig:.3f}" if r_sig is not None else "r = n/a") - print("NOTE: engagement_raw is a 40% component of the index, so the full-index") - print(" r is structurally high. The meaningful comparison is whether the") - print(" index RANKS high-engagement items above low-engagement ones vs the") - print(" legacy signal_score prior (r_sig above).") - - -if __name__ == "__main__": - main() +from oracle.cli import main as cli_main +sys.argv = ["oracle", "render", "--dry-run"] +cli_main() diff --git a/oracle/__init__.py b/oracle/__init__.py new file mode 100644 index 0000000..087a9cf --- /dev/null +++ b/oracle/__init__.py @@ -0,0 +1,21 @@ +"""Athena — AI Research Oracle. + +Unified intelligence pipeline for AI news aggregation, scoring, and rendering. + +Architecture: + oracle/ - Core package (this directory) + adapters/ - Source adapters (GitHub, arXiv, Reddit, HN, HF, RSS) + athena/ - Scoring and classification logic + cli.py - CLI entry point (subcommands) + db.py - Database operations and schema + render.py - Static site rendering + scoring.py - Athena scoring engine + clickability.py - Clickability index and decay + summarize.py - Rule-based summarization + recency.py - Recency guard and freshness filtering + themes.py - Theme-based trend tracking + archive.py - Soft-cap archival + config.py - Configuration and constants +""" + +__version__ = "1.0.0" diff --git a/oracle/__main__.py b/oracle/__main__.py new file mode 100644 index 0000000..d71908b --- /dev/null +++ b/oracle/__main__.py @@ -0,0 +1,4 @@ +"""Allow running as: python -m oracle""" +from oracle.cli import main + +main() diff --git a/oracle/archive.py b/oracle/archive.py new file mode 100644 index 0000000..7d20221 --- /dev/null +++ b/oracle/archive.py @@ -0,0 +1,83 @@ +"""Oracle soft-cap archival. + +Bounds live `entries` growth by moving old / excess rows into +`entries_archive` (preserving data — soft cap, not hard delete). +""" +import os +import sqlite3 +import time + +from oracle.config import DB_PATH + +ARCHIVE_SCHEMA = """ +CREATE TABLE IF NOT EXISTS entries_archive ( + id INTEGER PRIMARY KEY, + source TEXT, source_id TEXT, url TEXT, title TEXT, + extracted_text TEXT, summary TEXT, category_tags TEXT, + signal_score REAL, raw_metadata TEXT, + first_seen TEXT, last_updated TEXT, + archived_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')) +); +""" + + +def archive(days=30, cap=5000, dry_run=False, db_path=None): + """Archive entries older than N days or beyond the cap. + + Returns (total_live, archived_count). + """ + path = db_path or str(DB_PATH) + if not os.path.exists(path): + print("No oracle.db — nothing to archive") + return 0, 0 + + conn = sqlite3.connect(path) + conn.execute(ARCHIVE_SCHEMA) + + cutoff = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - days * 86400)) + + cur = conn.cursor() + cur.execute("SELECT id, last_updated FROM entries") + rows = cur.fetchall() + n_total = len(rows) + + old_ids = [r[0] for r in rows if (r[1] or "") < cutoff] + beyond = max(0, n_total - cap) + if beyond > 0: + ordered = sorted(rows, key=lambda r: r[1] or "")[:beyond] + cap_ids = [r[0] for r in ordered] + else: + cap_ids = [] + + move_ids = sorted(set(old_ids) | set(cap_ids)) + + if not move_ids: + print(f"Archive check: {n_total} live entries, none older than {days}d " + f"or beyond cap {cap}. Nothing to archive.") + conn.close() + return n_total, 0 + + print(f"Archive check: {n_total} live entries -> would archive {len(move_ids)} " + f"(old={len(old_ids)}, cap={len(cap_ids)}).") + + if dry_run: + print("DRY RUN — no changes made.") + conn.close() + return n_total, 0 + + q = ",".join("?" * len(move_ids)) + conn.execute( + f"""INSERT OR REPLACE INTO entries_archive + (id, source, source_id, url, title, extracted_text, summary, + category_tags, signal_score, raw_metadata, first_seen, last_updated) + SELECT id, source, source_id, url, title, extracted_text, summary, + category_tags, signal_score, raw_metadata, first_seen, last_updated + FROM entries WHERE id IN ({q})""", + move_ids, + ) + conn.execute(f"DELETE FROM entries WHERE id IN ({q})", move_ids) + conn.commit() + conn.close() + print(f"Archived {len(move_ids)} entries (live now {n_total - len(move_ids)}). " + f"Preserved in entries_archive.") + return n_total, len(move_ids) diff --git a/oracle/cli.py b/oracle/cli.py new file mode 100644 index 0000000..1d31d3e --- /dev/null +++ b/oracle/cli.py @@ -0,0 +1,497 @@ +"""CLI entry point for the AI Research Oracle. + +Single command with subcommands for all operations. +Replaces scattered root scripts with one unified interface. + +Usage: + python -m oracle [args] + +Commands: + ingest Run the ingestion pipeline (fetch + store + score) + summarize Generate summaries for unscored entries + query Query the database (top, search, recent, stats, snapshot) + render Render static site from clickability index + archive Soft-cap archival of old entries + themes Theme-based trend tracking + recency Recency guard analysis + health System health check +""" +import argparse +import json +import os +import sys +from datetime import datetime, timezone + +# Ensure project root is on path +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from oracle.config import DB_PATH, SCHEMA_PATH, ENABLED_SOURCES, DEFAULT_LIMIT, SOURCE_TIERS, VERDICT_THRESHOLDS +from oracle.db import get_connection, init_db, get_stats, query_top, query_recent, query_search, query_by_tag, migrate_world_monitor +from oracle.scoring import attach_scoring, migrate as scoring_migrate +from oracle.summarize import run_summarization +from oracle.render import render as render_site +from oracle.archive import archive as archive_entries +from oracle.themes import scan as theme_scan +from oracle.clickability import fetch_items, compute_index, decay_index +from oracle.recency import filter_fresh +from oracle.dedup import content_hash, compute_verdict, tier_adjusted_score, backfill_hashes, apply_verdicts, get_source_tier + + +def cmd_ingest(args): + """Run the ingestion pipeline.""" + import time + import sqlite3 + import signal + + from adapters import SourceAdapter + from adapters._store import upsert_entries + + sources = args.sources.split(",") if args.sources else ENABLED_SOURCES + now = datetime.now(timezone.utc) + print(f"=== AI Research Oracle Pipeline ===") + print(f" Sources: {', '.join(sources)}") + print(f" Limit: {args.limit}/source") + print(f" Dry run: {args.dry_run}") + print() + + # Import adapters dynamically + adapter_modules = { + "github": "adapters.github", + "arxiv": "adapters.arxiv", + "reddit": "adapters.reddit", + "hackernews": "adapters.hackernews", + "huggingface": "adapters.huggingface", + "rss": "adapters.rss_feeds", + } + adapter_classes = { + "github": "GitHubAdapter", + "arxiv": "ArxivAdapter", + "reddit": "RedditAdapter", + "hackernews": "HackerNewsAdapter", + "huggingface": "HuggingFaceAdapter", + "rss": "RSSFeedsAdapter", + } + + db_path = str(DB_PATH) + schema_path = str(SCHEMA_PATH) + + all_entries = [] + source_stats = {} + + for source_name in sources: + if source_name not in adapter_modules: + print(f" ⚠ Unknown source: {source_name}") + continue + + print(f" [{source_name}]") + mod = __import__(adapter_modules[source_name], fromlist=[adapter_classes[source_name]]) + adapter = getattr(mod, adapter_classes[source_name])() + + # Fetch with per-adapter timeout (prevents blocking on slow endpoints) + entries = [] + error = None + try: + entries = adapter.fetch(limit=args.limit, timeout=10) + except TypeError: + # Old adapter signature without timeout param — use thread-based fallback + import threading + result = {"entries": [], "error": None} + def _fetch(): + try: + result["entries"] = adapter.fetch(limit=args.limit) + except Exception as e: + result["error"] = str(e) + t = threading.Thread(target=_fetch, daemon=True) + t.start() + t.join(timeout=10) + if t.is_alive(): + error = f"timeout after 10s" + else: + entries = result["entries"] + error = result["error"] + except Exception as e: + error = str(e) + + if error: + print(f" ✗ {source_name} failed: {error}") + source_stats[source_name] = {"fetched": 0, "stored": 0, "error": error} + continue + + for entry in entries: + meta = json.loads(entry["raw_metadata"]) if isinstance(entry["raw_metadata"], str) else entry["raw_metadata"] + meta["adapter_version"] = "1.0" + entry["raw_metadata"] = json.dumps(meta) + + all_entries.extend(entries) + source_stats[source_name] = {"fetched": len(entries), "stored": 0} + print(f" Fetched: {len(entries)} entries") + time.sleep(0.5) + + if not args.dry_run and all_entries: + conn = init_db(db_path, schema_path) + stored = upsert_entries(conn, all_entries) + + for entry in all_entries: + src = entry["source"] + if src in source_stats: + source_stats[src]["stored"] += 1 + + # Score new entries + try: + attach_scoring(db_path) + except Exception as e: + print(f" ⚠ scoring attach failed: {e}") + + conn.close() + print(f" Total stored: {stored} entries") + else: + print(f" Total fetched: {len(all_entries)} entries (dry run)") + + print(f"\n Source summary:") + for src, stats in source_stats.items(): + error = stats.get("error", "") + error_str = f" ✗ {error}" if error else "" + print(f" {src}: {stats['fetched']} fetched, {stats.get('stored', '—')} stored{error_str}") + + if all_entries: + print(f"\n Top entries by signal score:") + sorted_entries = sorted(all_entries, key=lambda e: e["signal_score"], reverse=True) + for i, entry in enumerate(sorted_entries[:5]): + print(f" [{i+1}] {entry['source'].upper():6} score={entry['signal_score']:.2f} {entry['title'][:70]}") + + print(f"\n Done.") + + +def cmd_summarize(args): + """Run the summarization engine.""" + run_summarization(source=args.source, limit=args.limit) + + +def cmd_query(args): + """Query the database.""" + conn = get_connection() + qc = getattr(args, "query_command", None) + + if qc == "top": + entries = query_top(conn, n=args.n, source=args.source, min_score=args.min_score) + print(f"Top {len(entries)} entries:") + _print_entries(entries) + + elif qc == "search": + entries = query_search(conn, args.query_text, limit=args.limit) + print(f"Search results for '{args.query_text}':") + _print_entries(entries) + + elif qc == "recent": + entries = query_recent(conn, hours=args.hours) + print(f"Entries from last {args.hours}h:") + _print_entries(entries) + + elif qc == "by-tag": + entries = query_by_tag(conn, args.tag, limit=args.limit) + print(f"Entries tagged '{args.tag}':") + _print_entries(entries) + + elif qc == "stats": + stats = get_stats(conn) + print(f"Database: {DB_PATH}") + print(f"Total entries: {stats['total_entries']}") + print(f"Summarized: {stats['summarized']}") + print(f"Pending summary: {stats['pending_summary']}") + if stats.get("buckets"): + print(f"\nBucket distribution:") + for bucket, count in sorted(stats["buckets"].items(), key=lambda x: -x[1]): + print(f" {bucket}: {count}") + + elif qc == "snapshot": + stats = get_stats(conn) + top = query_top(conn, n=10) + print("=" * 70) + print("AI RESEARCH ORACLE — SNAPSHOT") + print("=" * 70) + print(f"Time: {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}") + print(f"Total entries: {stats['total_entries']}") + print() + print("Source breakdown:") + for src, s in stats["sources"].items(): + print(f" {src}: {s['cnt']} entries, avg score {s['avg_score']}") + print() + print("Top 10 by signal score:") + _print_entries(top) + + conn.close() + + +def _print_entries(entries): + """Pretty-print a list of entries.""" + if not entries: + print(" (no results)") + return + for i, e in enumerate(entries, 1): + summary = json.loads(e["summary"]) if e.get("summary") else {} + meta = json.loads(e["raw_metadata"]) if e.get("raw_metadata") else {} + print(f" [{i}] {e['source'].upper():10} | score={e['signal_score']:.2f}") + print(f" {e['title'][:80]}") + if summary.get("one_liner"): + print(f" → {summary['one_liner'][:100]}") + print() + + +def cmd_render(args): + """Render static site — single variant or all variants.""" + from oracle.variants import load_variant, list_variants, render_variant + + if args.list: + print("Available variants:") + for v in list_variants(): + print(f" {v}") + return + + if args.all_variants: + variants = list_variants() + if not variants: + print("No variants found in variants/") + return + print(f"=== Rendering all {len(variants)} variants ===\n") + for vname in variants: + try: + config = load_variant(vname) + render_variant(config, dry_run=args.dry_run, webroot=args.webroot) + except Exception as e: + print(f"[variant:{vname}] ERROR: {e}") + print() + return + + # Single variant (default = 'default' if not specified) + vname = args.variant or "default" + try: + config = load_variant(vname) + render_variant(config, dry_run=args.dry_run, webroot=args.webroot) + except FileNotFoundError as e: + print(f"Error: {e}") + + +def cmd_archive(args): + """Archive old entries.""" + archive_entries(days=args.days, cap=args.cap, dry_run=args.dry_run) + + +def cmd_themes(args): + """Run theme scan.""" + results = theme_scan(history=args.history) + print(f"=== Theme trend scan ===") + print(f" Fresh entries this cycle: {results['fresh_count']}") + if results["new_arrivals"]: + print(" NEW theme arrivals this cycle:") + for theme, count in results["new_arrivals"].items(): + print(f" {theme}: +{count}") + else: + print(" NEW theme arrivals this cycle: 0") + print(f" Cumulative totals: {results['cumulative']}") + + if args.history and results.get("history"): + print("\n Per-cycle history:") + for day, theme, count in results["history"]: + print(f" {day} {theme}: {count}") + + +def cmd_dedup(args): + """World Monitor migration: tiers, hashes, verdicts.""" + print("=== World Monitor Migration ===\n") + + conn = get_connection() + + # Show source tier config + if args.show_tiers: + print("Source tier configuration:") + for src, info in SOURCE_TIERS.items(): + print(f" {src:12} Tier {info['tier']} ({info['label']}) - {info['description']}") + print() + + # Show verdict thresholds + if args.show_verdicts: + print("Verdict thresholds:") + for verdict, thresholds in VERDICT_THRESHOLDS.items(): + print(f" {verdict:8} score >= {thresholds['min_score']}, age <= {thresholds['max_age_h']}h") + print() + + # Run migration + if args.migrate: + print("Running schema migration + backfill...\n") + result = migrate_world_monitor(conn) + print(f" Columns added: {result['columns_added']}") + print(f" Hashes backfilled: {result['hashes_backfilled']}") + print(f" Verdicts set: {result['verdicts_set']}") + print() + + # Show verdict distribution + cur = conn.cursor() + try: + cur.execute("PRAGMA table_info(entries)") + columns = {r[1] for r in cur.fetchall()} + except Exception: + conn.close() + return + + if "verdict" in columns: + cur.execute("SELECT verdict, COUNT(*) as cnt FROM entries WHERE verdict != '' GROUP BY verdict ORDER BY cnt DESC") + rows = cur.fetchall() + if rows: + print("Verdict distribution:") + for r in rows: + print(f" {r['verdict']:8} {r['cnt']}") + print() + + if "content_hash" in columns: + cur.execute("SELECT COUNT(*) FROM entries WHERE content_hash != '' AND content_hash IS NOT NULL") + hashed = cur.fetchone()[0] + cur.execute("SELECT COUNT(*) FROM entries") + total = cur.fetchone()[0] + print(f"Content hashes: {hashed}/{total} entries hashed") + + if "source_tier" in columns: + cur.execute("SELECT source_tier, COUNT(*) as cnt FROM entries GROUP BY source_tier ORDER BY source_tier") + rows = cur.fetchall() + if rows: + print(f"\nSource tier distribution:") + for r in rows: + print(f" Tier {r['source_tier']}: {r['cnt']}") + + conn.close() + print() + + +def cmd_health(args): + """System health check.""" + print("=== System Health Check ===\n") + + # Database + try: + conn = get_connection() + stats = get_stats(conn) + conn.close() + print(f"✓ Database: {DB_PATH}") + print(f" Total entries: {stats['total_entries']}") + print(f" Summarized: {stats['summarized']}") + print(f" Pending: {stats['pending_summary']}") + except Exception as e: + print(f"✗ Database error: {e}") + + # Schema columns + try: + conn = get_connection() + cur = conn.cursor() + cur.execute("PRAGMA table_info(entries)") + columns = [r[1] for r in cur.fetchall()] + conn.close() + print(f" Schema columns: {len(columns)}") + except Exception as e: + print(f" Schema check: {e}") + + # Run log + try: + conn = get_connection() + cur = conn.cursor() + cur.execute("SELECT COUNT(*) FROM run_log") + runs = cur.fetchone()[0] + cur.execute("SELECT run_time, failure_class FROM run_log ORDER BY id DESC LIMIT 3") + recent = cur.fetchall() + conn.close() + print(f" Pipeline runs logged: {runs}") + for r in recent: + print(f" {r[0]} class={r[1]}") + except Exception as e: + print(f" Run log: {e}") + + print() + + +def main(): + parser = argparse.ArgumentParser( + description="AI Research Oracle — Unified CLI", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = parser.add_subparsers(dest="command", help="Available commands") + + # ingest + p_ingest = sub.add_parser("ingest", help="Run the ingestion pipeline") + p_ingest.add_argument("--sources", default=None, help="Comma-separated sources") + p_ingest.add_argument("--limit", type=int, default=DEFAULT_LIMIT, help="Entries per source") + p_ingest.add_argument("--dry-run", action="store_true", help="Fetch but don't store") + + # summarize + p_summarize = sub.add_parser("summarize", help="Generate summaries") + p_summarize.add_argument("--source", default=None, help="Filter by source") + p_summarize.add_argument("--limit", type=int, default=0, help="Max entries (0=all)") + + # query (nested subcommands) + p_query = sub.add_parser("query", help="Query the database") + query_sub = p_query.add_subparsers(dest="query_command") + + p_top = query_sub.add_parser("top", help="Top N entries") + p_top.add_argument("n", type=int, nargs="?", default=10) + p_top.add_argument("--source", default=None) + p_top.add_argument("--min-score", type=float, default=0) + + p_search = query_sub.add_parser("search", help="Keyword search") + p_search.add_argument("query_text") + p_search.add_argument("--limit", type=int, default=20) + + p_recent = query_sub.add_parser("recent", help="Recent entries") + p_recent.add_argument("--hours", type=int, default=24) + + p_tag = query_sub.add_parser("by-tag", help="Entries by tag") + p_tag.add_argument("tag") + p_tag.add_argument("--limit", type=int, default=20) + + query_sub.add_parser("stats", help="Database statistics") + query_sub.add_parser("snapshot", help="Full snapshot") + + # render + p_render = sub.add_parser("render", help="Render static site") + p_render.add_argument("--variant", default=None, help="Variant name (default, research, devops, brief)") + p_render.add_argument("--all", dest="all_variants", action="store_true", help="Render all variants") + p_render.add_argument("--list", action="store_true", help="List available variants") + p_render.add_argument("--dry-run", action="store_true") + p_render.add_argument("--webroot", default=None, help="Output directory") + + # archive + p_archive = sub.add_parser("archive", help="Archive old entries") + p_archive.add_argument("--days", type=int, default=30) + p_archive.add_argument("--cap", type=int, default=5000) + p_archive.add_argument("--dry-run", action="store_true") + + # themes + p_themes = sub.add_parser("themes", help="Theme trend tracking") + p_themes.add_argument("--history", action="store_true") + + # dedup (World Monitor migration) + p_dedup = sub.add_parser("dedup", help="World Monitor: tiers, hashes, verdicts") + p_dedup.add_argument("--migrate", action="store_true", help="Run schema migration + backfill") + p_dedup.add_argument("--show-tiers", action="store_true", help="Show source tier config") + p_dedup.add_argument("--show-verdicts", action="store_true", help="Show verdict thresholds") + + # health + sub.add_parser("health", help="System health check") + + args = parser.parse_args() + + commands = { + "ingest": cmd_ingest, + "summarize": cmd_summarize, + "query": cmd_query, + "render": cmd_render, + "archive": cmd_archive, + "themes": cmd_themes, + "dedup": cmd_dedup, + "health": cmd_health, + } + + if args.command and args.command in commands: + commands[args.command](args) + else: + parser.print_help() + + +if __name__ == "__main__": + main() diff --git a/oracle/clickability.py b/oracle/clickability.py new file mode 100644 index 0000000..93a7569 --- /dev/null +++ b/oracle/clickability.py @@ -0,0 +1,210 @@ +"""Clickability Index for Athena entries. + +Read-only against the DB (SELECT only). Computes virality ranking with +exponential time-decay so items sink as they age. +""" +import json +import math +import os +import re +import time +from datetime import datetime, timezone +from typing import Optional + +from oracle.config import DB_PATH + +# Virality weights (clickability = how viral/spreadable an item is right now) +VEL_W = 0.50 +ENG_W = 0.50 +SIG_W = 0.0 + +NOW = None # set in fetch_items for age math + +# Category-specific half-lives (hours) +CATEGORY_HALF_LIVES = { + "breaking": 36.0, + "update": 24.0, + "OTHER": 18.0, +} + + +def get_connection(): + return __import__("sqlite3").connect(str(DB_PATH)) + + +def _classify(src: str, title: str, summary: str) -> str: + t = (title + " " + (summary or "")).lower() + if re.search(r"\bshow\s+hn\b", t) or (src == "hackernews" and re.search(r"\b(show|built|made|launched|shipped)\b", t)): + return "SHOW_HN" + if re.search(r"\b(gpt-|gpt5|gpt-5|deepseek|glm-|llama|qwen|claude|gemini|mistral|flux|stable-diffusion|sora|kimi|grok)\b", t) \ + and re.search(r"\b(releases?|released|v\d|launch|unveil|model|new\s+model|update|version)\b", t): + return "MODEL_RELEASE" + if re.search(r"\b(releases?|released|launches?|unveils?|announces?|debut|new\s+model|gpt-5|deepseek-v|glm-5)\b", t) \ + and re.search(r"\b(openai|anthropic|google|meta|microsoft|nvidia|ai)\b", t): + return "MODEL_RELEASE" + if src == "huggingface": + return "MODEL_CARD" + if src == "arxiv" or re.search(r"\b(paper|study|benchmark|arxiv|proposes|learns?|novel|framework\s+for|towards)\b", t): + return "RESEARCH" + if re.search(r"\b(sues|lawsuit|funding|raises|acqui|ipo|valued|stealing|trade secret|layoff|hire[ds]?|exec|ceo)\b", t) \ + and not re.search(r"\b(repo|library|tool|agent framework)\b", t): + return "BUSINESS_LEGAL" + if re.search(r"\b(burnout|opinion|think|feel|why|essay|culture|linkedin|social media|future of|we made|i think|hot take|i believe|my view|in defense)\b", t): + return "CULTURE_OPINION" + if re.search(r"\b(how to|tutorial|guide|running|build|setup|install|from scratch|learn)\b", t): + return "TUTORIAL_HOWTO" + if src == "github" or re.search(r"\b(repo|library|framework|tool|agent|sdk|cli|extension|plugin|app|engine)\b", t): + return "DEV_TOOL_DRAMA" + return "OTHER" + + +def _extract(src: str, md: dict) -> tuple: + """Return (velocity_raw, engagement_raw, age_hours).""" + if src == "hackernews": + pts = md.get("score", 0) or 0 + cmts = md.get("descendants", 0) or 0 + age_h = None + if md.get("time"): + try: + age_h = max((NOW - md["time"]) / 3600.0, 0.1) + except Exception: + age_h = None + vel = (pts / age_h) if age_h else pts + return vel, (pts + 2 * cmts), age_h + if src == "reddit": + ups = md.get("ups", 0) or 0 + cmts = md.get("num_comments", 0) or 0 + return ups, (ups + 2 * cmts), None + if src == "huggingface": + likes = md.get("likes", 0) or 0 + return likes, likes, None + if src == "github": + spd = md.get("stars_per_day", 0) or 0 + stars = md.get("stars", 0) or 0 + return spd, stars, None + if src == "arxiv": + return 0.0, 0.0, None + return 0.0, 0.0, None + + +def fetch_items(conn) -> list[dict]: + """Fetch all entries and compute raw engagement signals.""" + global NOW + NOW = time.time() + cur = conn.cursor() + cur.execute(""" + SELECT id, title, url, source, summary, signal_score, raw_metadata, first_seen, + curated_by, manual_section, manual_tier + FROM entries + """) + cols = [d[0] for d in cur.description] + out = [] + for row in cur.fetchall(): + d = dict(zip(cols, row)) + try: + md = json.loads(d.get("raw_metadata") or "{}") + except Exception: + md = {} + vel, eng, age = _extract(d["source"], md) + ct = _classify(d["source"], d.get("title") or "", d.get("summary") or "") + created_at = md.get("createdAt") if d["source"] == "huggingface" else None + out.append({ + "id": d["id"], + "title": d.get("title") or "", + "url": d.get("url") or "", + "source": d["source"], + "summary": d.get("summary") or "", + "signal_score": d.get("signal_score") or 0, + "velocity_raw": vel, + "engagement_raw": eng, + "content_type": ct, + "first_seen": d.get("first_seen") or "", + "created_at": created_at or "", + "age_hours": 0.0, + "curated_by": d.get("curated_by") or "", + "manual_section": d.get("manual_section") or "", + "manual_tier": d.get("manual_tier") or "", + }) + return out + + +def log1p_norm(values: list[float]) -> list[float]: + """Log1p + min-max normalization.""" + log_vals = [math.log1p(max(v, 0)) for v in values] + if not log_vals: + return [] + min_v, max_v = min(log_vals), max(log_vals) + if max_v == min_v: + return [0.0] * len(values) + return [(v - min_v) / (max_v - min_v) for v in log_vals] + + +def compute_index(items: list[dict]) -> list[dict]: + """Compute clickability index for all items.""" + velocities = [it.get("velocity_raw", 0) or 0 for it in items] + engagements = [it.get("engagement_raw", 0) or 0 for it in items] + signals = [it.get("signal_score", 0) or 0 for it in items] + + vel_norm = log1p_norm(velocities) + eng_norm = log1p_norm(engagements) + sig_norm = log1p_norm(signals) + + for i, item in enumerate(items): + raw = vel_norm[i] * VEL_W + eng_norm[i] * ENG_W + sig_norm[i] * SIG_W + if raw == 0 and sig_norm[i] > 0: + raw = 0.05 * sig_norm[i] + item["clickability"] = round(raw, 4) + item["section"] = "" + return items + + +def _age_hours(item: dict) -> float: + """Effective news-age in hours.""" + if item.get("source") == "huggingface" and item.get("created_at"): + s = item["created_at"] + else: + s = item.get("first_seen") or "" + if not s: + return 0.0 + try: + ts = datetime.strptime(s[:19], "%Y-%m-%dT%H:%M:%S").replace( + tzinfo=timezone.utc + ).timestamp() + return max((time.time() - ts) / 3600.0, 0.0) + except Exception: + return 0.0 + + +def _get_half_life(item: dict) -> Optional[float]: + """Return section/tier-specific half-life in hours, or None for default.""" + ms = (item.get("manual_section") or "").upper() + if ms in ("HARDWARE", "TIPS"): + return 336.0 + tier = item.get("tier", "normal") + if tier == "breaking": + return CATEGORY_HALF_LIVES["breaking"] + if tier == "update": + return CATEGORY_HALF_LIVES["update"] + return None + + +def decay_index(items: list[dict], half_life_h: float = 18.0) -> list[dict]: + """Apply exponential time-decay to clickability.""" + cutoff = datetime.now(timezone.utc).timestamp() - 24 * 3600 + for it in items: + age = _age_hours(it) + it["age_hours"] = round(age, 1) + base = it.get("clickability", 0) or 0 + hl = _get_half_life(it) + if hl is None: + hl = half_life_h + k = math.log(2) / hl + it["clickability_decayed"] = round(base * math.exp(-k * age), 4) + it["effective_half_life"] = hl + fs = it.get("first_seen") or "" + try: + ts = datetime.fromisoformat(fs.replace("Z", "+00:00")).timestamp() + except ValueError: + ts = 0 + it["fresh"] = ts >= cutoff + return items diff --git a/oracle/config.py b/oracle/config.py new file mode 100644 index 0000000..26d85d2 --- /dev/null +++ b/oracle/config.py @@ -0,0 +1,68 @@ +"""Centralized configuration for the AI Research Oracle. + +Single source of truth for all paths, defaults, and constants. +""" +import os +from pathlib import Path + +# ── Paths ────────────────────────────────────────────────────────────────── +ROOT = Path(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +DB_PATH = ROOT / "oracle.db" +SCHEMA_PATH = ROOT / "schema.sql" + +# ── Web output ───────────────────────────────────────────────────────────── +WEBROOT = "/var/www/preprod2" +FALLBACK_WEBROOT = ROOT / "site" +SEEN_JSON = ROOT.parent / "ai-oracle-site" / "seen_urls.json" + +# ── Pipeline defaults ────────────────────────────────────────────────────── +ENABLED_SOURCES = ["github", "arxiv", "reddit", "hackernews", "huggingface", "rss"] +DEFAULT_LIMIT = 20 + +# ── Source tiers (World Monitor pattern) ─────────────────────────────────── +# Tier 1: Primary trusted sources (official releases, peer-reviewed) +# Tier 2: Secondary credible sources (curated communities, major outlets) +# Tier 3: Tertiary noise sources (user-generated, unverified) +SOURCE_TIERS = { + "arxiv": {"tier": 1, "label": "PRIMARY", "description": "Peer-reviewed research"}, + "github": {"tier": 1, "label": "PRIMARY", "description": "Official code releases"}, + "huggingface": {"tier": 1, "label": "PRIMARY", "description": "Model registry"}, + "rss": {"tier": 2, "label": "SECONDARY", "description": "Curated tech media"}, + "hackernews": {"tier": 2, "label": "SECONDARY", "description": "Curated community"}, + "reddit": {"tier": 3, "label": "TERTIARY", "description": "User-generated discussion"}, +} + +# Tier-based signal score bonus/penalty (applied to final_score) +TIER_BONUS = {1: 0.05, 2: 0.0, 3: -0.05} + +# Freshness SLA per tier (hours after which a source is flagged stale) +FRESHNESS_SLA_H = {1: 48, 2: 24, 3: 12} + +# ── Composite verdict thresholds ─────────────────────────────────────────── +# PUBLISH: High score + fresh, goes to top +# WATCH: Medium score, monitor for follow-ups +# ARCHIVE: Low score or aged out, move to archive +# DROP: Junk score, ignore +VERDICT_THRESHOLDS = { + "PUBLISH": {"min_score": 6.0, "max_age_h": 48}, + "WATCH": {"min_score": 4.0, "max_age_h": 168}, # 7 days + "ARCHIVE": {"min_score": 2.0, "max_age_h": 720}, # 30 days + "DROP": {"min_score": 0.0, "max_age_h": 999999}, # catch-all +} + +# ── Content hash dedup ───────────────────────────────────────────────────── +CONTENT_HASH_PREFIX = "sha256" +HASH_LENGTH = 16 # characters + +# ── Clickability weights ─────────────────────────────────────────────────── +VEL_W = 0.50 +ENG_W = 0.50 +SIG_W = 0.0 +HALF_LIFE_H = 18.0 + +# ── Render ───────────────────────────────────────────────────────────────── +TOP_N = 8 + +# ── Archive ──────────────────────────────────────────────────────────────── +ARCHIVE_DAYS = 30 +ARCHIVE_CAP = 5000 diff --git a/oracle/db.py b/oracle/db.py new file mode 100644 index 0000000..a1c714d --- /dev/null +++ b/oracle/db.py @@ -0,0 +1,203 @@ +"""Database operations for the AI Research Oracle. + +Handles connections, schema initialization, and common queries. +""" +import os +import sqlite3 +from typing import Optional + +from oracle.config import DB_PATH, SCHEMA_PATH, SOURCE_TIERS + + +def get_connection(db_path: Optional[str] = None) -> sqlite3.Connection: + """Open a database connection.""" + path = db_path or str(DB_PATH) + conn = sqlite3.connect(path) + conn.row_factory = sqlite3.Row + return conn + + +def get_ro_connection(db_path: Optional[str] = None) -> sqlite3.Connection: + """Open a read-only database connection.""" + path = db_path or str(DB_PATH) + return sqlite3.connect(f"file:{path}?mode=ro", uri=True) + + +def init_db(db_path: Optional[str] = None, schema_path: Optional[str] = None) -> sqlite3.Connection: + """Initialize or open the database, applying schema if it exists. + + Schema uses CREATE IF NOT EXISTS so repeated calls are idempotent. + Also runs World Monitor migration columns (content_hash, verdict, freshness). + """ + conn = sqlite3.connect(db_path or str(DB_PATH)) + sp = schema_path or str(SCHEMA_PATH) + if os.path.exists(sp): + with open(sp) as f: + conn.executescript(f.read()) + conn.commit() + # World Monitor migration columns (idempotent) + for col in [ + "content_hash TEXT DEFAULT ''", + "verdict TEXT DEFAULT ''", + "source_tier INTEGER DEFAULT 2", + ]: + try: + conn.execute(f"ALTER TABLE entries ADD COLUMN {col}") + except sqlite3.OperationalError: + pass # already exists + conn.commit() + return conn + + +def migrate_world_monitor(conn: Optional[sqlite3.Connection] = None) -> dict: + """Apply World Monitor schema migrations + backfill. + + Returns: {columns_added: int, hashes_backfilled: int, verdicts_set: int} + """ + from oracle.dedup import backfill_hashes, apply_verdicts, get_source_tier + + c = conn or get_connection() + cur = c.cursor() + + # Check which columns already exist + cur.execute("PRAGMA table_info(entries)") + existing = {row[1] for row in cur.fetchall()} + + columns_to_add = [] + if "content_hash" not in existing: + columns_to_add.append("content_hash TEXT DEFAULT ''") + if "verdict" not in existing: + columns_to_add.append("verdict TEXT DEFAULT ''") + if "source_tier" not in existing: + columns_to_add.append("source_tier INTEGER DEFAULT 2") + + added = 0 + for col_def in columns_to_add: + try: + cur.execute(f"ALTER TABLE entries ADD COLUMN {col_def}") + added += 1 + except sqlite3.OperationalError: + pass # race condition or already exists + + c.commit() + + # Backfill content hashes + hashes = backfill_hashes(c) + + # Backfill source tiers — reset first so the WHERE clause catches everything + cur.execute("UPDATE entries SET source_tier = 0") + c.commit() + for source_name, tier_info in SOURCE_TIERS.items(): + cur.execute( + "UPDATE entries SET source_tier = ? WHERE source = ?", + (tier_info["tier"], source_name), + ) + c.commit() + + # Set verdicts + verdicts = apply_verdicts(c) + + return {"columns_added": added, "hashes_backfilled": hashes, "verdicts_set": verdicts} + + +def get_stats(conn: sqlite3.Connection) -> dict: + """Return database statistics.""" + cur = conn.cursor() + + cur.execute("SELECT COUNT(*) FROM entries") + total = cur.fetchone()[0] + + cur.execute(""" + SELECT source, COUNT(*) as cnt, ROUND(AVG(signal_score), 2) as avg_score, + MIN(signal_score) as min_score, MAX(signal_score) as max_score + FROM entries GROUP BY source + """) + sources = {r["source"]: dict(r) for r in cur.fetchall()} + + cur.execute("SELECT COUNT(*) FROM entries WHERE summary IS NOT NULL") + summarized = cur.fetchone()[0] + + cur.execute("SELECT COUNT(*) FROM entries WHERE summary IS NULL") + pending = cur.fetchone()[0] + + # Bucket distribution + try: + cur.execute(""" + SELECT bucket, COUNT(*) as cnt + FROM entries WHERE bucket IS NOT NULL + GROUP BY bucket ORDER BY cnt DESC + """) + buckets = {r["bucket"]: r["cnt"] for r in cur.fetchall()} + except Exception: + buckets = {} + + return { + "total_entries": total, + "sources": sources, + "summarized": summarized, + "pending_summary": pending, + "buckets": buckets, + } + + +def query_top(conn: sqlite3.Connection, n: int = 10, + source: Optional[str] = None, + min_score: float = 0) -> list[dict]: + """Get top N entries by signal score.""" + cur = conn.cursor() + where_parts = [] + params = [] + + if min_score > 0: + where_parts.append("signal_score >= ?") + params.append(min_score) + if source: + where_parts.append("source = ?") + params.append(source) + + where = (" AND " + " AND ".join(where_parts)) if where_parts else "" + cur.execute( + f"SELECT * FROM entries {where} ORDER BY signal_score DESC LIMIT ?", + params + [n], + ) + return [dict(r) for r in cur.fetchall()] + + +def query_recent(conn: sqlite3.Connection, hours: int = 24) -> list[dict]: + """Get entries from the last N hours.""" + from datetime import datetime, timezone, timedelta + + cutoff = (datetime.now(timezone.utc) - timedelta(hours=hours)).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + cur = conn.cursor() + cur.execute( + "SELECT * FROM entries WHERE first_seen >= ? ORDER BY first_seen DESC", + (cutoff,), + ) + return [dict(r) for r in cur.fetchall()] + + +def query_search(conn: sqlite3.Connection, q: str, limit: int = 20) -> list[dict]: + """Search entries by title, summary, and key technical point.""" + cur = conn.cursor() + pattern = f"%{q}%" + cur.execute(""" + SELECT * FROM entries + WHERE title LIKE ? + OR json_extract(summary,'$.one_liner') LIKE ? + OR json_extract(summary,'$.key_technical_point') LIKE ? + ORDER BY signal_score DESC LIMIT ? + """, (pattern, pattern, pattern, limit)) + return [dict(r) for r in cur.fetchall()] + + +def query_by_tag(conn: sqlite3.Connection, tag: str, limit: int = 20) -> list[dict]: + """Get entries matching a category tag.""" + cur = conn.cursor() + cur.execute(""" + SELECT * FROM entries + WHERE json_extract(category_tags,'$') LIKE ? + ORDER BY signal_score DESC LIMIT ? + """, (f'%"{tag}"%', limit)) + return [dict(r) for r in cur.fetchall()] diff --git a/oracle/dedup.py b/oracle/dedup.py new file mode 100644 index 0000000..1e844dc --- /dev/null +++ b/oracle/dedup.py @@ -0,0 +1,140 @@ +"""Content-hash dedup and composite verdict engine. + +World Monitor pattern: SHA-256 content hash for dedup, tier-weighted +composite verdict (PUBLISH/WATCH/ARCHIVE/DROP) on top of final_score. +""" +import hashlib +import re +from datetime import datetime, timezone +from oracle.config import ( + SOURCE_TIERS, TIER_BONUS, VERDICT_THRESHOLDS, + HASH_LENGTH, CONTENT_HASH_PREFIX, +) + + +def content_hash(title: str, url: str = "", body: str = "") -> str: + """Deterministic content hash for dedup. + + Normalizes whitespace, lowercases, strips HTML tags, then hashes. + Returns hex[:HASH_LENGTH] for compact storage. + """ + text = f"{title}|{body[:500]}|{url}" + text = re.sub(r'\s+', ' ', text).strip().lower() + text = re.sub(r'<[^>]+>', '', text) + raw = hashlib.sha256(text.encode()).hexdigest() + return f"{CONTENT_HASH_PREFIX}:{raw[:HASH_LENGTH]}" + + +def check_duplicate(conn, title: str, url: str = "", body: str = "", cutoff_days: int = 7) -> bool: + """Check if an entry with similar content_hash already exists within cutoff.""" + h = content_hash(title, url, body) + now = datetime.now(timezone.utc) + cur = conn.cursor() + cur.execute( + "SELECT COUNT(*) FROM entries WHERE content_hash = ? AND first_seen > ?", + (h, (now.timestamp() - cutoff_days * 86400)), + ) + count = cur.fetchone()[0] + return count > 0 + + +def get_source_tier(source: str) -> dict: + """Return tier info for a source. Defaults to tier 2.""" + return SOURCE_TIERS.get(source, {"tier": 2, "label": "SECONDARY", "description": "Unknown source"}) + + +def tier_adjusted_score(base_score: float, source: str) -> float: + """Apply tier bonus/penalty to a base signal score.""" + tier_info = get_source_tier(source) + tier_num = tier_info["tier"] + bonus = TIER_BONUS.get(tier_num, 0.0) + return round(base_score + bonus, 3) + + +def compute_verdict(score: float, age_hours: float) -> str: + """Compute composite verdict from score + age. + + Uses signal_score (0-10 scale). PUBLISH > WATCH > ARCHIVE > DROP. + """ + for verdict, thresholds in VERDICT_THRESHOLDS.items(): + if score >= thresholds["min_score"] and age_hours <= thresholds["max_age_h"]: + return verdict + return "DROP" + + +def age_hours(first_seen_iso: str) -> float: + """Return age in hours from ISO timestamp.""" + try: + ts = first_seen_iso.replace("Z", "+00:00") + first = datetime.fromisoformat(ts) + now = datetime.now(timezone.utc) + return max(0, (now - first).total_seconds() / 3600) + except (ValueError, AttributeError): + return 0.0 + + +def apply_verdicts(conn): + """Update verdict column for all entries that lack one. + + Uses signal_score (0-10 scale) + first_seen age to compute verdict. + """ + cur = conn.cursor() + # Check if verdict column exists + cur.execute("PRAGMA table_info(entries)") + columns = {row[1] for row in cur.fetchall()} + if "verdict" not in columns: + print(" [verdict] column not found, skipping apply") + return 0 + + # Reset all verdicts so they get recalculated + cur.execute("UPDATE entries SET verdict = ''") + conn.commit() + + # Fetch all entries with signal scores + cur.execute("SELECT id, COALESCE(signal_score, 0), first_seen FROM entries") + updated = 0 + for row in cur.fetchall(): + entry_id, signal_score, first_seen = row + age = age_hours(first_seen) + verdict = compute_verdict(signal_score, age) + cur.execute("UPDATE entries SET verdict = ? WHERE id = ?", (verdict, entry_id)) + updated += 1 + + conn.commit() + return updated + + +def backfill_hashes(conn, batch_size: int = 500) -> int: + """Backfill content_hash for entries that lack one.""" + cur = conn.cursor() + cur.execute("PRAGMA table_info(entries)") + columns = {row[1] for row in cur.fetchall()} + if "content_hash" not in columns: + print(" [dedup] content_hash column not found, skipping backfill") + return 0 + + updated = 0 + while True: + cur.execute( + "SELECT id, title, url, summary FROM entries " + "WHERE content_hash IS NULL OR content_hash = '' " + "LIMIT ?", + (batch_size,), + ) + rows = cur.fetchall() + if not rows: + break + for entry_id, title, url, summary in rows: + body = "" + if summary: + import json + try: + s = json.loads(summary) + body = s.get("one_liner", "") + " " + s.get("key_points", "") + except (json.JSONDecodeError, TypeError): + body = summary[:200] + h = content_hash(title, url, body) + cur.execute("UPDATE entries SET content_hash = ? WHERE id = ?", (h, entry_id)) + updated += 1 + conn.commit() + return updated diff --git a/oracle/recency.py b/oracle/recency.py new file mode 100644 index 0000000..2968778 --- /dev/null +++ b/oracle/recency.py @@ -0,0 +1,140 @@ +"""Recency guard — Athena "how old is this news?" gate. + +Age is the dominant gate. TODAY's items are always eligible. +Older items are eligible ONLY if never posted before. +""" +import json +import os +import re +from datetime import datetime, timezone + +from oracle.config import SEEN_JSON + +ORACLE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def _parse(ts): + if not ts: + return None + try: + return datetime.fromisoformat(ts.replace("Z", "+00:00")) + except Exception: + return None + + +def _norm_url(u): + if not u: + return "" + return u.split("?")[0].split("#")[0].rstrip("/").lower() + + +def _norm_title(t): + if not t: + return "" + t = t.lower() + t = re.sub(r"[^a-z0-9 ]", " ", t) + return re.sub(r"\s+", " ", t).strip()[:60] + + +def load_posted(md_dir: str = ORACLE_DIR, seen_json: str = str(SEEN_JSON)): + """Return (md_urls:set, md_titles:set, seen_urls:set).""" + md_urls, md_titles, seen_urls = set(), set(), set() + + for fn in sorted(os.listdir(md_dir)): + if re.match(r"athena_top.*\.md$", fn): + try: + txt = open(os.path.join(md_dir, fn), encoding="utf-8", errors="replace").read() + except OSError: + continue + for m in re.findall(r"\]\((https?://[^)\s]+)\)", txt): + nu = _norm_url(m) + if nu: + md_urls.add(nu) + for t in re.findall(r"^\|\s*\d+\s*\|\s*(.+?)\s*\|", txt, re.M): + nt = _norm_title(t) + if nt: + md_titles.add(nt) + + if os.path.exists(seen_json): + try: + with open(seen_json, encoding="utf-8") as f: + for u in json.load(f): + nu = _norm_url(u) + if nu: + seen_urls.add(nu) + except (json.JSONDecodeError, OSError): + pass + + return md_urls, md_titles, seen_urls + + +def is_today(first_seen, now=None): + now = now or datetime.now(timezone.utc) + d = _parse(first_seen) + return bool(d) and d.strftime("%Y-%m-%d") == now.strftime("%Y-%m-%d") + + +def age_days(first_seen, now=None): + now = now or datetime.now(timezone.utc) + d = _parse(first_seen) + if not d: + return 9999.0 + return max((now - d).total_seconds() / 86400.0, 0.0) + + +def day_bucket(first_seen, now=None): + days = age_days(first_seen, now) + if days < 1: + return "today" + if days < 2: + return "yesterday" + if days <= 6: + return "this-week" + return "older" + + +def already_posted(url, title, first_seen, now=None, + md_urls=None, md_titles=None, seen_urls=None): + """Age-aware dedup. Today's items are NEVER flagged.""" + if md_urls is None or md_titles is None or seen_urls is None: + md_urls, md_titles, seen_urls = load_posted() + if _norm_url(url) in md_urls: + return True + nt = _norm_title(title) + if nt and nt in md_titles: + return True + if is_today(first_seen, now): + return False + if _norm_url(url) in seen_urls: + return True + return False + + +def recency_weight(first_seen, now=None, half_life_days=2.0): + """1.0 for today, decays ~halving every 2 days.""" + return 0.5 ** (age_days(first_seen, now) / half_life_days) + + +def blend_score(item, now=None): + """clickability_decayed * recency_weight.""" + base = item.get("clickability_decayed", 0) or 0 + return base * recency_weight(item.get("first_seen"), now) + + +def filter_fresh(items, now=None, recent_window_days=4.0): + """Split into (today_items, older_new_items, dropped_items).""" + now = now or datetime.now(timezone.utc) + md_urls, md_titles, seen_urls = load_posted() + today_items, older_new, dropped = [], [], [] + for it in items: + fs = it.get("first_seen") + if is_today(fs, now): + today_items.append(it) + continue + posted = already_posted(it.get("url"), it.get("title"), fs, now, + md_urls, md_titles, seen_urls) + if posted and age_days(fs, now) > recent_window_days: + dropped.append(it) + else: + older_new.append(it) + return today_items, older_new, dropped diff --git a/oracle/render.py b/oracle/render.py new file mode 100644 index 0000000..230a688 --- /dev/null +++ b/oracle/render.py @@ -0,0 +1,359 @@ +"""Render Athena entries into a static news site. + +Two-layer layout: Top News (fresh today) + aging Stack (everything else). +Read-only against oracle.db. Designed for a 20-min cron run. + +Variant-aware: render_variant_html() builds themed pages from pre-filtered items. +""" +import argparse +import datetime +import html +import json +import os +import re +import sqlite3 +from collections import OrderedDict +from typing import Optional + +from oracle.clickability import fetch_items, compute_index, decay_index +from oracle.config import DB_PATH, WEBROOT, FALLBACK_WEBROOT, HALF_LIFE_H, TOP_N + + +def _clean_summary(raw): + """Extract the most readable field from summary JSON.""" + if not raw: + return "" + try: + d = json.loads(raw) + if isinstance(d, dict): + for k in ("one_liner", "key_technical_point", "potential_use_case"): + v = d.get(k) + if isinstance(v, str) and v.strip(): + return re.sub(r"\\+|_|`", "", v).strip() + except Exception: + pass + return re.sub(r"\\+|_|`", "", raw).strip() + + +# ── Theme palette ────────────────────────────────────────────────────────── + +THEMES = { + "dark": { + "--bg": "#0b0e14", + "--card": "#141925", + "--fg": "#e6e9ef", + "--mut": "#8b93a7", + "--border": "#1f2533", + }, + "light": { + "--bg": "#f8f9fa", + "--card": "#ffffff", + "--fg": "#1a1a2e", + "--mut": "#6b7280", + "--border": "#e5e7eb", + }, + "midnight": { + "--bg": "#0a0a1a", + "--card": "#111128", + "--fg": "#c8d0e0", + "--mut": "#5a6480", + "--border": "#1a1a3a", + }, +} + + +def _theme_css(display: dict) -> str: + """Generate CSS variables for a variant display config.""" + theme_name = display.get("theme", "dark") + palette = THEMES.get(theme_name, THEMES["dark"]) + accent = display.get("accent", "#5b8cff") + vars_list = ", ".join(f"{k}:{v}" for k, v in palette.items()) + return f":root {{ {vars_list}; --acc:{accent}; }}" + + +def _fmt_time(first_seen): + if not first_seen: + return "" + try: + dt = datetime.datetime.strptime(first_seen, "%Y-%m-%dT%H:%M:%SZ") + return dt.strftime("%H:%M") + except Exception: + return "" + + +def _card(it, big=False): + title = html.escape(it["title"] or "(untitled)") + url = html.escape(it["url"] or "#") + src = html.escape(it["source"]) + t = _fmt_time(it.get("first_seen")) + summary_raw = _clean_summary(it.get("summary") or "") + summary = html.escape(summary_raw[:200]) + cls = "card big" if big else "card" + summary_html = ('

{0}

'.format(summary)) if (summary and big) else "" + return f""" +
+
{src} + {t} + sig {it.get('signal_score') or 0:.1f} + \U0001f525 {it['clickability_decayed']:.2f}
+

{title}

+ {summary_html} +
""" + + +def build_html(items): + """Build the full HTML page from ranked items.""" + now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + ranked = sorted(items, key=lambda x: x["clickability_decayed"], reverse=True) + + fresh = [it for it in ranked if it.get("fresh")] + top = fresh[:TOP_N] + stack = [it for it in ranked if it not in top] + + by_day = OrderedDict() + for it in stack: + day = (it.get("first_seen") or "")[:10] or "unknown" + by_day.setdefault(day, []).append(it) + + top_html = "".join(_card(it, big=True) for it in top) + + stack_html = "" + for day, rows in by_day.items(): + rows.sort(key=lambda x: x["clickability_decayed"], reverse=True) + cards = "".join(_card(it) for it in rows) + stack_html += f""" +

\U0001f4c5 {html.escape(day)}

+
{cards}
""" + + return f""" + + + + +Athena AI News — Ranked by Clickability + + + +
+

Athena AI News

+
Auto-ranked by Clickability Index · decays with age so the stack flows top → bottom · generated {now} · {len(items)} stories
+
+
+

\U0001f534 Top News

+
{top_html}
+

\U0001f4f0 The Stack

+ {stack_html} +
+ +""" + + +def render(dry_run=False, webroot=None): + """Run the full render pipeline. + + Returns (output_path, item_count). + """ + conn = sqlite3.connect(str(DB_PATH)) + items = fetch_items(conn) + conn.close() + + items = compute_index(items) + items = decay_index(items, HALF_LIFE_H) + page = build_html(items) + + if dry_run: + out = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "_preview.html") + with open(out, "w") as f: + f.write(page) + fresh = [it for it in items if it.get("fresh")] + top = sorted(fresh, key=lambda x: x["clickability_decayed"], reverse=True)[:TOP_N] + print(f"[dry-run] wrote {out} ({len(items)} items, {len(fresh)} fresh today)") + print(f"TOP {TOP_N} FRESH (today only) by decayed clickability:") + for i, it in enumerate(top, 1): + print(f" {i}. [{it['clickability_decayed']:.2f} | age {it['age_hours']:.0f}h] {it['source']:10} {it['title'][:55]}") + return out, len(items) + + target = webroot or (WEBROOT if os.path.isdir(WEBROOT) else str(FALLBACK_WEBROOT)) + os.makedirs(target, exist_ok=True) + + with open(os.path.join(target, "index.html"), "w") as f: + f.write(page) + + with open(os.path.join(target, "feed.json"), "w") as f: + json.dump([ + {"title": i["title"], "url": i["url"], "source": i["source"], + "clickability_decayed": i["clickability_decayed"], "age_hours": i["age_hours"], + "first_seen": i.get("first_seen")} + for i in sorted(items, key=lambda x: x["clickability_decayed"], reverse=True) + ], f, indent=2) + + where = "WEBROOT" if target == WEBROOT else "fallback(~oracle/site)" + print(f"[render] wrote {target}/index.html ({len(items)} items) -> {where}") + return os.path.join(target, "index.html"), len(items) + + +# ── Variant rendering ────────────────────────────────────────────────────── + +def _variant_card(it: dict, display: dict, big: bool = False) -> str: + """Build an HTML card for a variant edition.""" + title = html.escape(it.get("title") or "(untitled)") + url = html.escape(it.get("url") or "#") + src = html.escape(it.get("source", "")) + t = _fmt_time(it.get("first_seen")) + summary_raw = _clean_summary(it.get("summary") or "") + summary = html.escape(summary_raw[:200]) + + score = it.get("signal_score") + tier = it.get("source_tier") + verdict = it.get("verdict", "") + + cls = "card big" if big else "card" + summary_html = ('

{}

'.format(summary)) if (summary and display.get("show_summary") and big) else "" + + # Build meta badges + badges = f'{src}' + if t: + badges += f'{t}' + if display.get("show_score") and score is not None: + badges += f'sig {float(score):.1f}' + if display.get("show_tier") and tier is not None: + badges += f'T{tier}' + if display.get("show_verdict") and verdict: + badges += f'{verdict}' + + return f""" +
+
{badges}
+

{title}

+ {summary_html} +
""" + + +def render_variant_html(items: list[dict], variant: dict) -> str: + """Build the full HTML page for a variant edition.""" + display = variant.get("display", {}) + name = variant.get("name", "Athena") + desc = variant.get("description", "") + logo = display.get("logo", "🏛️") + top_n = display.get("top_n", 8) + half_life = variant.get("ranking", {}).get("half_life_h", 18) + + now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + + top = items[:top_n] + stack = items[top_n:] + + # Group stack by day + by_day = OrderedDict() + for it in stack: + day = (it.get("first_seen") or "")[:10] or "unknown" + by_day.setdefault(day, []).append(it) + + top_html = "\n".join(_variant_card(it, display, big=True) for it in top) + stack_html = "" + for day, rows in by_day.items(): + cards = "".join(_variant_card(it, display) for it in rows) + stack_html += f""" +

🗒️ {html.escape(day)}

+
{cards}
""" + + theme = _theme_css(display) + + return f""" + + + + +{html.escape(name)} + + + +
+

{logo} {html.escape(name)}

+
{html.escape(desc)} · generated {now} · {len(items)} stories
+
+
+

🔥 Top News

+
{top_html}
+ {('

🗄️ The Stack

' + stack_html) if stack else ''} +
+ + +""" + + +def render_variant_json(items: list[dict], variant: dict, path: str) -> None: + """Write a JSON feed for a variant edition.""" + display = variant.get("display", {}) + feed = [] + for i in items: + entry = { + "title": i.get("title", ""), + "url": i.get("url", ""), + "source": i.get("source", ""), + "signal_score": float(i.get("signal_score") or 0), + "first_seen": i.get("first_seen", ""), + } + if display.get("show_tier"): + entry["source_tier"] = i.get("source_tier") + if display.get("show_verdict"): + entry["verdict"] = i.get("verdict", "") + feed.append(entry) + + with open(path, "w") as f: + json.dump(feed, f, indent=2) diff --git a/oracle/scoring.py b/oracle/scoring.py new file mode 100644 index 0000000..a5edf14 --- /dev/null +++ b/oracle/scoring.py @@ -0,0 +1,332 @@ +"""Athena scoring engine — Sprint 1: Pure Rule-Based Bucket Classifier + Scorer. + +DESIGN CONSTRAINT (founder directive, 2026-07-15): + Pure rules only. No embeddings, no semantic similarity, no LLM classification. + +Pipeline position: + ingestion/dedup -> [attach_scoring] -> rendering +""" +import json +import re +import sqlite3 +from typing import Optional + +from oracle.config import DB_PATH + +# ── Score weights ────────────────────────────────────────────────────────── +WEIGHTS = { + "shipping": 0.20, + "utility": 0.20, + "replication": 0.25, + "enthusiast": 0.20, + "novelty": 0.15, +} +HYPE_CAP = 0.45 + +# ── Keyword sets ─────────────────────────────────────────────────────────── +KW_SHIPPING = [ + "released", "launch", "v1.0", "v2.0", "v3.0", "shipping", "now available", + "open source", "open-source", "open weights", "weights released", "live now", + "beta", "public beta", "ga release", "general availability", "ships", "deployed", + "production", "now in", "available today", "download", "gradio", "demo", "playground", +] +KW_LOCAL_AI = [ + "local llm", "local model", "local ai", "run locally", "run it locally", "on-device", + "on device", "ollama", "llama.cpp", "llamacpp", "gguf", "ggml", "lm studio", + "consumer hardware", "consumer gpu", "rtx", "your own gpu", "offline", "private ai", + "local-only", "self-host", "self-hosted", "home server", "edge device", "edge inference", + "quantized", "quantization", "q4", "q8", "int4", "fp16", "fine-tune at home", + "train at home", "local inference", "local deployment", "no api", "no cloud", +] +KW_PROBLEM_SOLVED = [ + "how to", "how i", "solved", "fix", "fixed", "workaround", "benchmark", "improves", + "improved", "speedup", "speed-up", "reduces", "reduce", "cut", "cuts", "boost", + "optimize", "optimized", "optimisation", "faster", "3x", "10x", "2x", "latency", + "throughput", "roi", "cost", "cheaper", "save", "saves", "eliminate", "eliminated", + "from 117s to 30s", "p95", "memory usage", "vram", "token cost", "bottleneck", + "case study", "results", "we measured", "we tested", "showdown", "comparison", +] +KW_MODEL_RELEASE = [ + "releases", "released", "unveils", "introduces", "new model", "new flagship", + "gpt-", "claude", "gemini", "llama", "mistral", "qwen", "deepseek", "grok", + "phi-", "command-r", "api access", "weights", "open model", "open-models", + "frontier", "checkpoint", "fine-tune", "finetune", "rl-trained", "rl train", + "trained", "post-training", "post training", "distilled", "distillation", +] +KW_RESEARCH = [ + "paper", "arxiv", "preprint", "study", "research", "we propose", "we present", + "we introduce", "we show", "method", "framework", "theorem", "analysis of", + "survey", "benchmark", "dataset", "neural", "transformer", "diffusion", + "gradient", "ablation", "we find", "our approach", "novel", "state-of-the-art", + "sota", "cs.lg", "cs.cl", "cs.cv", "cs.ai", +] +KW_BUSINESS = [ + "raises", "raised", "$", "valuation", "series a", "series b", "funding", "round", + "ipo", "acquisition", "acquires", "merger", "deal", "revenue", "layoff", "hiring", + "partnership", "invests", "investment", "market", "vc", "compute deal", + "billion", "million", "forecast", "miss", "earnings", "stock", +] +KW_INFRA = [ + "gpu", "tpu", "data center", "datacenter", "data centre", "cluster", "cuda", + "rocm", "vllm", "tensorrt", "inference server", "serving", "kubernetes", "docker", + "pipeline", "mlops", "ci/cd", "rag", "vector db", "vector database", "agent", + "agents", "orchestration", "observability", "evaluation", "eval", "guardrail", + "safety", "red team", "jailbreak", "prompt injection", "fine-tuning stack", +] +KW_CULTURE = [ + "says", "argues", "opinion", "essay", "think", "thinks", "the real", "why we", + "the future of", "dystopia", "utopia", "philosophy", "ethics", "regulation", + "policy", "ban", "lawsuit", "eu", "senate", "congress", "interview", "podcast", + "controversy", "controversial", "debate", "critic", "criticism", + "creepy", "creeping", "not sexy", "vibe", "hot take", "unpopular", +] + +BUCKETS = { + "SHIPPING": {"kw": KW_SHIPPING, "source_whitelist": None, "order": 0}, + "LOCAL AI": {"kw": KW_LOCAL_AI, "source_whitelist": None, "order": 1}, + "PROBLEM SOLVED": {"kw": KW_PROBLEM_SOLVED, "source_whitelist": None, "order": 2}, + "MODEL RELEASE": {"kw": KW_MODEL_RELEASE, "source_whitelist": None, "order": 3}, + "RESEARCH": {"kw": KW_RESEARCH, "source_whitelist": ["arxiv"], "order": 4}, + "BUSINESS": {"kw": KW_BUSINESS, "source_whitelist": None, "order": 5}, + "INFRASTRUCTURE": {"kw": KW_INFRA, "source_whitelist": None, "order": 6}, + "CULTURE": {"kw": KW_CULTURE, "source_whitelist": None, "order": 7}, +} +BUCKET_ORDER = sorted(BUCKETS.keys(), key=lambda b: BUCKETS[b]["order"]) + +HYPE_TERMS = [ + "revolutionary", "game-changing", "game changer", "breakthrough", "mind-blowing", + "insane", "crazy", "unbelievable", "shocking", "the future is here", "omg", + "you won't believe", "secret", "they don't want you to know", "leaked", "viral", + "hype", "buzzword", "disrupt", "disrupting everything", "ai will replace", + "will change everything", "paradigm shift", "godlike", "magic", "miracle", +] + +ENTHUSIAST_SIGNALS = [ + "github", "repo", "repository", "self-host", "local", "ollama", "llamacpp", + "hugging face", "huggingface", "colab", "notebook", "pip install", "docker", + "cli", "open source", "open-source", "diy", "build your own", "tutorial", + "how to", "implementation", "agent", "agents", "fine-tune", "finetune", + "quantiz", "vllm", "rtx", "gpu", "consumer", "homelab", "self-hosted", + "machine-learning", "machine learning", "deep learning", "python", "rust", + "benchmark", "reproduc", "weights", "gguf", +] +SOURCE_ENTHUSIAST_BONUS = { + "github": 0.20, "huggingface": 0.20, "arxiv": 0.10, + "hackernews": 0.10, "reddit": 0.05, "rss": 0.0, +} + +NEW_COLUMNS = [ + "bucket TEXT", + "shipping_score REAL DEFAULT 0", + "utility_score REAL DEFAULT 0", + "replication_score REAL DEFAULT 0", + "enthusiast_score REAL DEFAULT 0", + "novelty_score REAL DEFAULT 0", + "hype_penalty REAL DEFAULT 0", + "final_score REAL DEFAULT 0", + "actionability_score REAL DEFAULT 0", + "narrative_id TEXT", + "topic_id TEXT", + "relation_json TEXT", +] + + +# ── Helpers ──────────────────────────────────────────────────────────────── +def _norm(text): + if not text: + return "" + if isinstance(text, bytes): + text = text.decode("utf-8", "replace") + return " " + re.sub(r"\s+", " ", text.lower()) + " " + + +def _summary_text(raw): + if not raw: + return "" + try: + d = json.loads(raw) + if isinstance(d, dict): + return " ".join(str(v) for v in d.values() if isinstance(v, str)) + except Exception: + pass + return raw + + +# ── Classification ───────────────────────────────────────────────────────── +def classify(entry: dict) -> tuple[str, list[str]]: + """Pure rule classification. + + Returns (bucket, matched_list) where matched_list is human-readable proof. + """ + title = _norm(entry.get("title") or "") + summary = _norm(_summary_text(entry.get("summary"))) + tags_raw = entry.get("category_tags") or "" + try: + tags = " ".join(json.loads(tags_raw)) if tags_raw else "" + except Exception: + tags = tags_raw + tags = _norm(tags) + source = (entry.get("source") or "").lower() + haystack = title + " " + summary + " " + tags + + matched = [f"source={source}"] + best_bucket = "UNCATEGORIZED" + best_hits = 0 + + for bucket in BUCKET_ORDER: + spec = BUCKETS[bucket] + whitelist = spec["source_whitelist"] + if whitelist and source not in whitelist: + continue + hits = [] + for kw in spec["kw"]: + if f" {kw.lower()} " in haystack: + hits.append(kw) + if hits: + matched.extend(f"kw:{h}" for h in hits[:8]) + if len(hits) > best_hits: + best_hits = len(hits) + best_bucket = bucket + + if best_bucket == "UNCATEGORIZED": + matched.append("(no rule fired)") + + return best_bucket, matched + + +# ── Scoring ──────────────────────────────────────────────────────────────── +def score_entry(bucket: str, matched: list, entry: dict) -> dict: + """Return dict of component scores (0..1) + final (0..1).""" + source = (entry.get("source") or "").lower() + haystack = _norm(entry.get("title") or "") + " " + _norm(_summary_text(entry.get("summary"))) + + tags_raw = entry.get("category_tags") or "" + try: + tags = " ".join(json.loads(tags_raw)) if tags_raw else "" + except Exception: + tags = tags_raw + haystack += _norm(tags) + + # Enthusiast score + ent_hits = sum(1 for s in ENTHUSIAST_SIGNALS if f" {s} " in haystack) + enthusiast = min(ent_hits / 5.0 + SOURCE_ENTHUSIAST_BONUS.get(source, 0.0), 1.0) + + # Shipping score + ship_kw = [k for k in KW_SHIPPING if f" {k} " in haystack] + shipping = 0.0 + if bucket == "SHIPPING": + shipping = 0.9 + elif ship_kw: + shipping = min(0.4 + 0.1 * len(ship_kw), 0.8) + if source in ("github", "huggingface"): + shipping = max(shipping, 0.7) + + # Utility score + util_kw = [k for k in KW_PROBLEM_SOLVED if f" {k} " in haystack] + utility = 0.0 + if bucket == "PROBLEM SOLVED": + utility = 0.85 + elif util_kw: + utility = min(0.4 + 0.1 * len(util_kw), 0.8) + if any(s in haystack for s in [" github ", " huggingface ", " demo "]): + utility = max(utility, 0.6) + + # Replication score + repl_kw = [k for k in KW_LOCAL_AI if f" {k} " in haystack] + replication = 0.0 + if bucket == "LOCAL AI": + replication = 1.0 + elif repl_kw: + replication = min(0.5 + 0.1 * len(repl_kw), 0.9) + if source in ("github", "huggingface"): + replication = max(replication, 0.7) + if any(s in haystack for s in [" open source ", " open-source ", " weights "]): + replication = max(replication, 0.6) + + # Novelty score + novelty = 0.0 + if bucket in ("RESEARCH", "MODEL RELEASE"): + novelty = 0.6 + nov_kw = ["new", "novel", "first", "breakthrough-method", "we propose", + "we introduce", "we present", "state-of-the-art", "sota", "unveils"] + if any(f" {k} " in haystack for k in nov_kw): + novelty = min(novelty + 0.2, 0.9) + if bucket == "CULTURE": + novelty = min(novelty, 0.3) + + # Hype penalty + hype_penalty = min(0.1 * sum(1 for t in HYPE_TERMS if f" {t} " in haystack), 0.6) + + # Final score + raw = ( + WEIGHTS["shipping"] * shipping + + WEIGHTS["utility"] * utility + + WEIGHTS["replication"] * replication + + WEIGHTS["enthusiast"] * enthusiast + + WEIGHTS["novelty"] * novelty + ) + final = min(max(raw - hype_penalty, 0.0), 1.0) + + return { + "shipping_score": round(shipping, 3), + "utility_score": round(utility, 3), + "replication_score": round(replication, 3), + "enthusiast_score": round(enthusiast, 3), + "novelty_score": round(novelty, 3), + "hype_penalty": round(hype_penalty, 3), + "final_score": round(final, 4), + } + + +# ── DB Operations ────────────────────────────────────────────────────────── +def migrate(db_path: Optional[str] = None) -> list[str]: + """Idempotent schema migration — only adds missing columns.""" + conn = sqlite3.connect(db_path or str(DB_PATH)) + cur = conn.cursor() + cur.execute("PRAGMA table_info(entries)") + existing = {row[1] for row in cur.fetchall()} + added = [] + for col in NEW_COLUMNS: + name = col.split(" ")[0] + if name not in existing: + cur.execute(f"ALTER TABLE entries ADD COLUMN {col}") + added.append(name) + conn.commit() + conn.close() + print(f"[migrate] added columns: {', '.join(added) if added else 'none (already present)'}") + return added + + +def fetch_unscored(conn: sqlite3.Connection) -> list[dict]: + cur = conn.cursor() + cur.execute(""" + SELECT id, source, source_id, url, title, summary, category_tags, raw_metadata + FROM entries WHERE bucket IS NULL OR bucket = '' + """) + cols = ["id", "source", "source_id", "url", "title", "summary", "category_tags", "raw_metadata"] + return [dict(zip(cols, row)) for row in cur.fetchall()] + + +def attach_scoring(db_path: Optional[str] = None, dry_run: bool = False) -> None: + """Score every unscored entry. Call after ingestion/dedup, before render.""" + conn = sqlite3.connect(db_path or str(DB_PATH)) + rows = fetch_unscored(conn) + print(f"[attach] scoring {len(rows)} unscored entries") + for e in rows: + bucket, matched = classify(e) + scores = score_entry(bucket, matched, e) + if not dry_run: + conn.execute( + """UPDATE entries SET bucket=?, shipping_score=?, utility_score=?, + replication_score=?, enthusiast_score=?, novelty_score=?, + hype_penalty=?, final_score=?, actionability_score=?, + narrative_id=?, topic_id=?, relation_json=? WHERE id=?""", + (bucket, scores["shipping_score"], scores["utility_score"], + scores["replication_score"], scores["enthusiast_score"], + scores["novelty_score"], scores["hype_penalty"], scores["final_score"], + 0.0, None, None, json.dumps({"matched_rules": matched}), e["id"]), + ) + if not dry_run: + conn.commit() + conn.close() + print("[attach] done.") diff --git a/oracle/summarize.py b/oracle/summarize.py new file mode 100644 index 0000000..4287a6f --- /dev/null +++ b/oracle/summarize.py @@ -0,0 +1,301 @@ +"""Summarization engine for the AI Research Oracle. + +Generates structured summaries for entries where summary IS NULL. +Uses source-specific extraction logic (no LLM required). + +Output schema: {one_liner, key_technical_point, potential_use_case, confidence} +""" +import json +import re +import sqlite3 +from typing import Optional + +from oracle.config import DB_PATH + + +def extract_github_summary(title: str, content: str) -> dict: + """Extract summary from GitHub README content.""" + text = re.sub(r']*>', '\n', content) + text = re.sub(r'

', '\n', content) + text = re.sub(r']*>', '\n## ', text) + text = re.sub(r'', '\n', text) + text = re.sub(r'<[^>]+>', '', text) + text = re.sub(r'&', '&', text) + text = re.sub(r'—', '—', text) + text = re.sub(r''', "'", text) + text = re.sub(r'·', '·', text) + text = re.sub(r'```[\s\S]*?```', '', text) + text = re.sub(r'\n\s*\n+', '\n\n', text) + text = text.strip() + + source_confidence = "low" + if len(text) > 2000: + source_confidence = "high" + elif len(text) > 500: + source_confidence = "medium" + + one_liner = _find_project_description(text, title) or title[:200] + key_tech = _extract_technical_point(text, source_confidence) + use_case = _extract_use_case(text, title) + confidence = _assess_extraction_quality(one_liner, key_tech, use_case, source_confidence) + + if _is_security_tooling(title, one_liner, key_tech): + use_case = use_case + " [security:dual-use]" + + return { + "one_liner": one_liner[:200], + "key_technical_point": key_tech[:200], + "potential_use_case": use_case[:200], + "confidence": confidence, + } + + +def extract_arxiv_summary(title: str, content: str) -> dict: + """Extract summary from arXiv abstract.""" + text = re.sub(r'<[^>]+>', ' ', content) + text = re.sub(r'\s+', ' ', text).strip() + + confidence = "high" if len(text) > 300 else "medium" + one_liner = _find_contribution(text) or f"This paper presents {title.lower()}" + key_tech = _extract_method(text) + use_case = _extract_application(text) + + return { + "one_liner": one_liner[:200], + "key_technical_point": key_tech[:200], + "potential_use_case": use_case[:200], + "confidence": confidence, + } + + +def extract_reddit_summary(title: str, content: str) -> dict: + """Extract summary from Reddit post.""" + text = re.sub(r'<[^>]+>', ' ', content) + text = re.sub(r'\s+', ' ', text).strip() + + if len(text) > 500: + confidence = "high" + elif len(text) > 100: + confidence = "medium" + else: + confidence = "low" + + return { + "one_liner": (title or text[:150])[:200], + "key_technical_point": (text or "No additional content in post")[:200], + "potential_use_case": "AI community discussion", + "confidence": confidence, + } + + +# ── Extraction helpers ───────────────────────────────────────────────────── +def _assess_extraction_quality(one_liner, key_tech, use_case, source_confidence) -> str: + score = 0 + penalties = 0 + ol = one_liner.strip() + ol_len = len(ol) + + if 40 <= ol_len <= 200: + score += 2 + elif 20 <= ol_len < 40: + score += 1 + elif ol_len > 200: + penalties += 1 + + if ol.endswith(('.', '!', '?', '…')): + score += 1 + else: + penalties += 1 + + if re.search(r'\b(?:is|are|provides|enables|implements|makes|allows|builds|creates|runs|uses)\b', ol, re.I): + score += 1 + elif re.match(r'^[A-Z]\w+', ol) and ol_len > 30: + score += 0.5 + + open_brackets = ol.count('[') + ol.count('(') + close_brackets = ol.count(']') + ol.count(')') + if abs(open_brackets - close_brackets) > 0: + penalties += 1 + if open_brackets > 2: + penalties += 1 + + kt = key_tech.strip() + if kt and len(kt) > 20 and not kt.startswith('See '): + score += 1 + else: + penalties += 0.5 + + uc = use_case.strip() + if uc and len(uc) > 10 and not uc.startswith('Relevant for'): + score += 1 + else: + penalties += 0.5 + + net = score - penalties + if net >= 3: + return source_confidence + elif net >= 1: + return "medium" + return "low" + + +def _is_security_tooling(title: str, one_liner: str, key_tech: str) -> bool: + combined = f"{title} {one_liner} {key_tech}".lower() + return any(sig in combined for sig in [ + "offensive", "pentest", "red team", "exploit", "kill chain", + "attack surface", "vulnerability scan", "zero-day", + "reverse engineer", "c2", "command and control", + ]) + + +def _find_project_description(text: str, title: str) -> Optional[str]: + proj_name = title.split(':')[0].split('/')[0].strip().lower() + for para in text.split('\n\n'): + para = para.strip() + if not para or para.startswith('##') or len(para) < 20: + continue + if 'img' in para.lower() or 'badge' in para.lower() or 'shields' in para.lower(): + continue + if re.match(r'^[~$#€£¥*»\d]', para): + continue + special_chars = sum(1 for c in para if not c.isalnum() and not c.isspace() and c not in ',.!?;:\'\"-()[]') + if special_chars / max(len(para), 1) > 0.4: + continue + sentence = re.split(r'[.!?]', para)[0].strip() + if len(sentence) > 30: + return sentence + '.' + + for pattern in [ + rf'{re.escape(proj_name[:20])}\s+(?:is|enables|provides|implements)\s+[^.]+\.?', + r'(?:This\s+)?(?:project|library|framework|tool|package)\s+(?:is|enables|provides)\s+[^.]+\.?', + ]: + match = re.search(pattern, text, re.I) + if match: + return match.group(0)[:200] + return None + + +def _find_contribution(text: str) -> Optional[str]: + for pattern in [ + r'(?:we|this\s+paper)\s+(?:propose|introduce|present|propose and evaluate)\s+[^.]{10,150}\.', + r'(?:we\s+(?:show|demonstrate|find|discover|observe))\s+[^.]{10,150}\.', + ]: + match = re.search(pattern, text, re.I) + if match: + return match.group(0)[:200] + first = re.split(r'[.!?]', text)[0].strip() + return first if first else None + + +def _extract_technical_point(text: str, confidence: str) -> str: + for pattern in [ + r'architecture(?:\s+designed)?\s+(?:for|to|that)\s+[^.]+\.?', + r'(?:using|via|based\s+on|through)\s+[a-z][^.]{10,100}\.', + ]: + match = re.search(pattern, text, re.I) + if match: + return match.group(0)[:200] + if confidence == "low": + return "Technical details not available in extracted content" + return "See README for technical details" + + +def _extract_method(text: str) -> str: + for pattern in [ + r'(?:method|approach|framework|technique|model|system)\s+(?:based|using|via|through|with)\s+[a-z][^.]{10,120}\.', + r'(?:combining|leveraging|exploiting)\s+[a-z][^.]{10,120}\.', + ]: + match = re.search(pattern, text, re.I) + if match: + return match.group(0)[:200] + return "See full paper for methodology" + + +def _extract_use_case(text: str, title: str) -> str: + for pattern in [ + r'(?:for|to)\s+(?:developers|engineers|researchers|teams)\s+who?\s+[^.]{5,80}\.', + r'(?:enables|allows|helps)\s+[^\s]+\s+to\s+[^.]{10,80}\.', + ]: + match = re.search(pattern, text, re.I) + if match: + return match.group(0)[:200] + return f"Relevant for {title.lower()[:50]} developers and users" + + +def _extract_application(text: str) -> str: + title_lower = text[:200].lower() + if any(k in title_lower for k in ["agent", "agentic"]): + return "Building AI agent systems" + if any(k in title_lower for k in ["verification", "verify"]): + return "LLM output verification and reliability" + if any(k in title_lower for k in ["embodied", "robot"]): + return "Embodied AI and robotics applications" + if any(k in title_lower for k in ["distill"]): + return "Model distillation and knowledge transfer" + return "See paper for specific applications" + + +# ── Pipeline functions ───────────────────────────────────────────────────── +def summarize_entry(entry: dict, conn: sqlite3.Connection) -> bool: + """Summarize a single entry using rule-based extraction.""" + source = entry["source"] + title = entry["title"] + content = entry.get("extracted_text", "") + eid = entry["id"] + + if not content or len(content) < 50: + return False + + if source == "github": + summary = extract_github_summary(title, content) + elif source == "arxiv": + summary = extract_arxiv_summary(title, content) + elif source == "reddit": + summary = extract_reddit_summary(title, content) + else: + summary = extract_reddit_summary(title, content) + + conn.execute("UPDATE entries SET summary = ? WHERE id = ?", + (json.dumps(summary), eid)) + conn.commit() + return True + + +def run_summarization(source: Optional[str] = None, limit: int = 0) -> None: + """Summarize all pending entries.""" + conn = sqlite3.connect(str(DB_PATH)) + cur = conn.cursor() + + where = "summary IS NULL" + params = [] + if source: + where += " AND source = ?" + params.append(source) + + cur.execute(f"SELECT COUNT(*) FROM entries WHERE {where}", params) + total_pending = cur.fetchone()[0] + print(f"[summarize] {total_pending} pending entries") + + if limit: + limit_clause = f"LIMIT {limit}" + else: + limit_clause = "" + + cur.execute(f""" + SELECT id, source, title, extracted_text, summary + FROM entries WHERE {where} + ORDER BY first_seen DESC + {limit_clause} + """, params) + + summarized = 0 + for row in cur.fetchall(): + entry = { + "id": row[0], "source": row[1], "title": row[2], + "extracted_text": row[3], "summary": row[4], + } + if summarize_entry(entry, conn): + summarized += 1 + + conn.close() + print(f"[summarize] done — {summarized} entries summarized") diff --git a/oracle/themes.py b/oracle/themes.py new file mode 100644 index 0000000..6caf7e7 --- /dev/null +++ b/oracle/themes.py @@ -0,0 +1,93 @@ +"""Theme-based trend tracking for Athena. + +Tag by THEME, not by entry ID. Count NEW theme-tagged arrivals per cron cycle. +""" +import argparse +import os +import re +import sqlite3 +from collections import Counter + +from oracle.config import DB_PATH + +# Theme -> regex over title+summary+extracted text +THEME_PATTERNS = { + "tool-call": re.compile( + r"\b(tool[- ]?call|tool[- ]?use|competence gate|confidence gate|" + r"gate[d]? tool|action gate|tool reliability|function call gate)\b", + re.I), + "context": re.compile( + r"\b(context (compress|window|ceiling|summar)|semantic compress|" + r"token (compress|budget)|compress (context|session)|context (limit|overflow))\b", + re.I), + "compute": re.compile( + r"\b(small(er|est)? model|route to|inference cost|cpu (tts|infer)|" + r"cheap(er)? model|model routing|tiny model|on[- ]device (llm|model))\b", + re.I), + "trust": re.compile( + r"\b(trust(ed)? (adapter|lora)|vetted adapter|learn (only|what).*adapter|" + r"trust boundary|what a model (can|may) learn|auditable (adapter|skill))\b", + re.I), +} + + +def scan(conn=None, history=False) -> dict: + """Classify fresh entries and report new theme arrivals. + + Returns dict with counts and cumulative totals. + """ + if conn is None: + conn = sqlite3.connect(str(DB_PATH)) + conn.row_factory = sqlite3.Row + own_conn = True + else: + own_conn = False + + cur = conn.cursor() + cur.execute("""CREATE TABLE IF NOT EXISTS theme_tags ( + entry_id INTEGER NOT NULL, + theme TEXT NOT NULL, + first_seen_cycle TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')), + PRIMARY KEY (entry_id, theme))""") + + cur.execute(""" + SELECT e.id, e.source, e.title, + COALESCE(e.summary,'') AS summary, + COALESCE(e.extracted_text,'') AS extracted + FROM entries e + WHERE e.id NOT IN (SELECT entry_id FROM theme_tags) + """) + fresh = cur.fetchall() + + new_counts = Counter() + for row in fresh: + blob = f"{row['title']} {row['summary']} {row['extracted']}" + for theme, pat in THEME_PATTERNS.items(): + if pat.search(blob): + cur.execute( + "INSERT OR IGNORE INTO theme_tags (entry_id, theme) VALUES (?, ?)", + (row["id"], theme)) + new_counts[theme] += 1 + + conn.commit() + + cur.execute("SELECT theme, COUNT(*) AS c FROM theme_tags GROUP BY theme") + cum = {r["theme"]: r["c"] for r in cur.fetchall()} + + result = { + "fresh_count": len(fresh), + "new_arrivals": dict(new_counts), + "cumulative": cum, + } + + if history: + cur.execute(""" + SELECT substr(first_seen_cycle,1,10) AS day, theme, COUNT(*) AS c + FROM theme_tags GROUP BY day, theme ORDER BY day, theme + """) + result["history"] = [(r["day"], r["theme"], r["c"]) for r in cur.fetchall()] + + if own_conn: + conn.close() + + return result diff --git a/oracle/variants.py b/oracle/variants.py new file mode 100644 index 0000000..e769aeb --- /dev/null +++ b/oracle/variants.py @@ -0,0 +1,254 @@ +"""Multi-variant edition engine for Athena. + +One data pipeline → multiple audience-specific editions. +Each variant is a YAML config that defines filters, ranking, and display. + +World Monitor pattern: 6 variants from 1 codebase. +""" +import os +import sqlite3 +import yaml +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +from oracle.config import ROOT, DB_PATH, VERDICT_THRESHOLDS + + +# ── Variant config defaults ──────────────────────────────────────────────── + +DEFAULT_CONFIG = { + "name": "Athena AI News", + "description": "", + "output": "index.html", + "filters": { + "verdicts": ["PUBLISH", "WATCH", "ARCHIVE", "DROP"], + "sources": [], + "min_score": 0, + "max_age_h": 0, + "max_items": 0, + }, + "ranking": { + "by": "clickability", + "half_life_h": 18, + }, + "display": { + "top_n": 8, + "show_summary": True, + "show_score": True, + "show_tier": False, + "show_verdict": False, + "theme": "dark", + "accent": "#5b8cff", + "logo": "🏛️", + }, +} + + +def load_variant(name: str) -> dict: + """Load a variant config by name, merging with defaults. + + Looks in variants/.yaml relative to project root. + Returns merged config dict. + """ + variants_dir = ROOT / "variants" + variant_file = variants_dir / f"{name}.yaml" + + if not variant_file.exists(): + available = [p.stem for p in sorted(variants_dir.glob("*.yaml"))] + raise FileNotFoundError( + f"Variant '{name}' not found. Available: {', '.join(available)}" + ) + + with open(variant_file) as f: + raw = yaml.safe_load(f) or {} + + # Deep merge with defaults + config = _deep_merge(DEFAULT_CONFIG, raw) + config["_name"] = name + config["_file"] = str(variant_file) + return config + + +def list_variants() -> list[str]: + """List all available variant names.""" + variants_dir = ROOT / "variants" + if not variants_dir.exists(): + return [] + return sorted(p.stem for p in variants_dir.glob("*.yaml")) + + +def apply_filters(conn: sqlite3.Connection, variant: dict) -> list[dict]: + """Fetch entries from DB filtered by variant config. + + Returns list of entry dicts matching the variant's filter criteria. + """ + f = variant["filters"] + + # Build query + clauses = [] + params = [] + + # Verdict filter + if f.get("verdicts"): + verdicts = [v for v in f["verdicts"] if v] # strip empties + if verdicts: + placeholders = ", ".join("?" for _ in verdicts) + clauses.append(f"verdict IN ({placeholders})") + params.extend(verdicts) + + # Source filter (empty = all) + if f.get("sources"): + sources = [s for s in f["sources"] if s] + if sources: + placeholders = ", ".join("?" for _ in sources) + clauses.append(f"source IN ({placeholders})") + params.extend(sources) + + # Min signal score + if f.get("min_score", 0) > 0: + clauses.append("COALESCE(signal_score, 0) >= ?") + params.append(f["min_score"]) + + # Max age + if f.get("max_age_h", 0) > 0: + max_age = f["max_age_h"] + cutoff = datetime.now(timezone.utc).timestamp() - (max_age * 3600) + clauses.append("first_seen >= datetime(?, 'unixepoch')") + params.append(cutoff) + + # Base query + sql = "SELECT * FROM entries" + if clauses: + sql += " WHERE " + " AND ".join(clauses) + + sql += " ORDER BY first_seen DESC" + + # Max items + if f.get("max_items", 0) > 0: + sql += f" LIMIT {int(f['max_items'])}" + + conn.row_factory = sqlite3.Row + cur = conn.cursor() + cur.execute(sql, params) + rows = cur.fetchall() + + return [dict(r) for r in rows] + + +def rank_items(items: list[dict], variant: dict) -> list[dict]: + """Rank items according to variant ranking config. + + Supports: clickability, signal_score, verdict_priority, freshness + """ + ranking = variant["ranking"] + by = ranking.get("by", "clickability") + half_life = ranking.get("half_life_h", 18) + + if by == "clickability": + # Use clickability module for decayed scoring + from oracle.clickability import compute_index, decay_index + items = compute_index(items) + items = decay_index(items, half_life) + items.sort(key=lambda x: x.get("clickability_decayed", 0), reverse=True) + + elif by == "signal_score": + items.sort(key=lambda x: float(x.get("signal_score") or 0), reverse=True) + + elif by == "verdict_priority": + # PUBLISH > WATCH > ARCHIVE > DROP + priority = {"PUBLISH": 0, "WATCH": 1, "ARCHIVE": 2, "DROP": 3} + items.sort(key=lambda x: ( + priority.get(x.get("verdict", "DROP"), 4), + float(x.get("signal_score") or 0), + ), reverse=False) + + elif by == "freshness": + items.sort(key=lambda x: x.get("first_seen", ""), reverse=True) + + else: + # Default fallback: signal score + items.sort(key=lambda x: float(x.get("signal_score") or 0), reverse=True) + + return items + + +def render_variant(variant: dict, dry_run: bool = False, webroot: Optional[str] = None): + """Full render pipeline for one variant. + + Load config → filter → rank → render HTML. + Returns (output_path, item_count). + """ + from oracle.render import render_variant_html, render_variant_json + + conn = sqlite3.connect(str(DB_PATH)) + conn.row_factory = sqlite3.Row + + items = apply_filters(conn, variant) + items = rank_items(items, variant) + conn.close() + + if not items: + print(f"[variant:{variant['_name']}] No items matched filters") + return None, 0 + + page = render_variant_html(items, variant) + + # Determine output path + output_rel = variant.get("output", "index.html") + + if dry_run: + out_dir = ROOT / "_preview" / variant["_name"] + out_dir.mkdir(parents=True, exist_ok=True) + + # Use just the filename for dry-run (output_rel may have subdirs) + dry_filename = os.path.basename(output_rel) + out_path = str(out_dir / dry_filename) + + with open(out_path, "w") as f: + f.write(page) + + # Also write JSON feed + json_path = str(out_dir / "feed.json") + render_variant_json(items, variant, json_path) + + display = variant["display"] + top_n = display.get("top_n", 8) + top = items[:top_n] + print(f"[variant:{variant['_name']}] wrote {out_path} ({len(items)} items)") + print(f" TOP {top_n}:") + for i, it in enumerate(top, 1): + score = f" sig={it.get('signal_score', 0):.1f}" if display.get("show_score") else "" + tier = f" tier={it.get('source_tier', '?')}" if display.get("show_tier") else "" + verdict = f" [{it.get('verdict', '?')}]" if display.get("show_verdict") else "" + print(f" {i}. {it['title'][:60]}{score}{tier}{verdict}") + + return out_path, len(items) + + # Production render + target = webroot or str(ROOT / "site") + out_path = os.path.join(target, output_rel) + os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True) + + with open(out_path, "w") as f: + f.write(page) + + # JSON feed alongside + json_path = os.path.join(os.path.dirname(out_path), "feed.json") + render_variant_json(items, variant, json_path) + + print(f"[variant:{variant['_name']}] wrote {out_path} ({len(items)} items)") + return out_path, len(items) + + +# ── Helpers ──────────────────────────────────────────────────────────────── + +def _deep_merge(base: dict, override: dict) -> dict: + """Recursively merge override into base dict.""" + merged = base.copy() + for key, value in override.items(): + if key in merged and isinstance(merged[key], dict) and isinstance(value, dict): + merged[key] = _deep_merge(merged[key], value) + else: + merged[key] = value + return merged diff --git a/pipeline.py b/pipeline.py index a4ce5ba..f527bba 100644 --- a/pipeline.py +++ b/pipeline.py @@ -1,370 +1,27 @@ #!/usr/bin/env python3 +"""Thin wrapper — delegates to oracle.cli ingest subcommand. + +Kept for backward compatibility with existing cron/calls. """ -AI Research Oracle — Pipeline Orchestrator. - -Runs source adapters, deduplicates, stores to unified SQLite DB. -Designed so adding a new adapter is one line of registration. - -Usage: - python3 pipeline.py # run all enabled adapters - python3 pipeline.py --sources github,arxiv # specific sources - python3 pipeline.py --limit 15 # per-source limit - python3 pipeline.py --dry-run # fetch but don't store - python3 pipeline.py --verify # spot-check N entries -""" - -import argparse -import json -import os -import sqlite3 import sys -import time -from datetime import datetime, timezone +import os -# Allow running from project root sys.path.insert(0, os.path.dirname(__file__)) -from adapters import SourceAdapter -from adapters._store import upsert_entries +# Translate old args to new CLI format +args = sys.argv[1:] +cli_args = ["ingest"] +for i, arg in enumerate(args): + if arg in ("--sources", "--limit"): + cli_args.append(arg) + if i + 1 < len(args): + cli_args.append(args[i + 1]) + elif arg == "--dry-run": + cli_args.append("--dry-run") + elif arg == "--verify": + pass # verify handled internally -# Sprint 1 (2026-07-15): pure-rule bucket classifier + scorer. -# Attaches immediately after ingest/dedup and before any rendering step. -from athena import scoring as _scoring +from oracle.cli import main as cli_main -# Adapter registry — add new adapters here (one line each) -ADAPTERS = { - "github": lambda: __import__("adapters.github", fromlist=["GitHubAdapter"]).GitHubAdapter(), - "arxiv": lambda: __import__("adapters.arxiv", fromlist=["ArxivAdapter"]).ArxivAdapter(), - "reddit": lambda: __import__("adapters.reddit", fromlist=["RedditAdapter"]).RedditAdapter(), - "hackernews": lambda: __import__("adapters.hackernews", fromlist=["HackerNewsAdapter"]).HackerNewsAdapter(), - "huggingface": lambda: __import__("adapters.huggingface", fromlist=["HuggingFaceAdapter"]).HuggingFaceAdapter(), - "rss": lambda: __import__("adapters.rss_feeds", fromlist=["RSSFeedsAdapter"]).RSSFeedsAdapter(), -} - -# Default enabled sources -ENABLED_SOURCES = ["github", "arxiv", "reddit", "hackernews", "huggingface", "rss"] - - -def init_db(db_path: str, schema_path: str) -> sqlite3.Connection: - """Initialize or open the database.""" - conn = sqlite3.connect(db_path) - if os.path.exists(schema_path): - with open(schema_path) as f: - conn.executescript(f.read()) - conn.commit() - return conn - - -def store_entries(conn: sqlite3.Connection, entries: list[dict]) -> int: - """Store entries idempotently (dedup by url). - - FIX (2026-07-13, Tony): was INSERT OR REPLACE which OVERWROTE first_seen - with the harvest time on every re-harvest, turning stale stories into - "today". Now uses an UPSERT that preserves the ORIGINAL first_seen and only - bumps last_updated. Entries must already carry first_seen = true publish date. - """ - return upsert_entries(conn, entries) - - -def verify_entries(conn: sqlite3.Connection, source: str, sample_size: int = 3): - """Spot-check N entries from a source against live data. - - This is a standing quality gate: after each adapter run, verify a - random sample of high-signal entries match the live source. - Protects against bad batches propagating to the reasoning layer. - """ - import urllib.request - import random - - cur = conn.cursor() - cur.execute(""" - SELECT id, source, source_id, url, title, raw_metadata, signal_score - FROM entries WHERE source = ? - ORDER BY signal_score DESC - LIMIT ? - """, (source, sample_size)) - - rows = cur.fetchall() - if not rows: - print(f" No entries to verify for {source}") - return True - - passed = 0 - for row in rows: - eid, src, sid, url, title, meta_json, score = row - meta = json.loads(meta_json) if meta_json else {} - - # Source-specific verification - if src == "github": - repo = meta.get("full_name", "") - db_stars = meta.get("stars", 0) - if repo: - try: - api_url = f"https://api.github.com/repos/{repo}" - req = urllib.request.Request(api_url, headers={ - "Accept": "application/vnd.github.v3+json", - "User-Agent": "ai-oracle/0.1", - }) - with urllib.request.urlopen(req, timeout=10) as resp: - live = json.loads(resp.read().decode("utf-8")) - live_stars = live.get("stargazers_count", 0) - drift = abs(live_stars - db_stars) - # Allow up to 1% drift or 100 stars (whichever larger) - threshold = max(int(db_stars * 0.01), 100) - if drift <= threshold: - print(f" ✓ [{eid}] {title[:60]}... stars={db_stars} drift={drift}") - passed += 1 - else: - print(f" ✗ [{eid}] {title[:60]}... stars={db_stars} vs live={live_stars} DRIFT={drift}") - except Exception as e: - print(f" ? [{eid}] {title[:60]}... verify failed: {e}") - - elif src == "arxiv": - arxiv_id = meta.get("arxiv_id", "") - if arxiv_id: - # Clean version suffix for URL - clean_id = arxiv_id.split("v")[0] - try: - req = urllib.request.Request( - f"https://arxiv.org/abs/{clean_id}", - headers={"User-Agent": "ai-oracle/0.1"} - ) - with urllib.request.urlopen(req, timeout=10) as resp: - status = resp.status - if status == 200: - print(f" ✓ [{eid}] {title[:60]}... arxiv.org/abs/{clean_id} exists") - passed += 1 - else: - print(f" ? [{eid}] {title[:60]}... HTTP {status}") - except Exception as e: - print(f" ? [{eid}] {title[:60]}... verify failed: {e}") - - elif src == "hackernews": - hn_id = meta.get("id", "") - if hn_id: - try: - req = urllib.request.Request( - f"https://hacker-news.firebaseio.com/v0/item/{hn_id}.json", - headers={"User-Agent": "ai-oracle/0.1"}, - ) - with urllib.request.urlopen(req, timeout=10) as resp: - live = json.loads(resp.read().decode("utf-8")) - live_score = live.get("score", 0) - db_score = meta.get("score", 0) - drift = abs(live_score - db_score) - if drift <= 20: - print(f" ✓ [{eid}] {title[:60]}... hn_score={db_score} drift={drift}") - passed += 1 - else: - print(f" ? [{eid}] {title[:60]}... hn_score={db_score} vs live={live_score} DRIFT={drift}") - except Exception as e: - print(f" ? [{eid}] {title[:60]}... verify failed: {e}") - - elif src == "huggingface": - model_id = meta.get("model_id", "") - if model_id: - try: - req = urllib.request.Request( - f"https://huggingface.co/api/models/{model_id}", - headers={"User-Agent": "athena/0.1"}, - ) - with urllib.request.urlopen(req, timeout=10) as resp: - live = json.loads(resp.read().decode("utf-8")) - live_likes = live.get("likes", 0) - db_likes = meta.get("likes", 0) - drift = abs(live_likes - db_likes) - if drift <= 10: # HF likes update in real-time, allow small drift - print(f" ✓ [{eid}] {title[:60]}... hf_likes={db_likes} drift={drift}") - passed += 1 - else: - print(f" ? [{eid}] {title[:60]}... hf_likes={db_likes} vs live={live_likes} DRIFT={drift}") - except Exception as e: - print(f" ? [{eid}] {title[:60]}... verify failed: {e}") - - elif src == "reddit": - # For Reddit, just check the URL is reachable (no easy verification of scores via RSS) - if url: - try: - req = urllib.request.Request(url, headers={"User-Agent": "ai-oracle/0.1"}) - with urllib.request.urlopen(req, timeout=10) as resp: - status = resp.status - if status in (200, 302): - print(f" ✓ [{eid}] {title[:60]}... URL reachable") - passed += 1 - else: - print(f" ? [{eid}] {title[:60]}... HTTP {status}") - except Exception as e: - print(f" ? [{eid}] {title[:60]}... verify failed: {e}") - - elif src == "rss": - # For RSS, verify the article URL is reachable - if url: - try: - req = urllib.request.Request(url, headers={"User-Agent": "ai-oracle/0.1"}) - with urllib.request.urlopen(req, timeout=10) as resp: - status = resp.status - if status in (200, 301, 302): - print(f" ✓ [{eid}] {title[:60]}... URL reachable (HTTP {status})") - passed += 1 - else: - print(f" ? [{eid}] {title[:60]}... HTTP {status}") - except Exception as e: - print(f" ? [{eid}] {title[:60]}... verify failed: {e}") - - time.sleep(0.5) # polite spacing - - return passed > 0 - - -def run_pipeline(sources: list[str] | None = None, limit: int = 20, dry_run: bool = False, verify: bool = False): - """Run the ingestion pipeline.""" - sources = sources or ENABLED_SOURCES - now = datetime.now(timezone.utc) - print(f"=== AI Research Oracle Pipeline ===") - print(f" Sources: {', '.join(sources)}") - print(f" Limit: {limit}/source") - print(f" Dry run: {dry_run}") - print() - - db_path = os.path.join(os.path.dirname(__file__), "oracle.db") - schema_path = os.path.join(os.path.dirname(__file__), "schema.sql") - - all_entries = [] - source_stats = {} - - for source_name in sources: - if source_name not in ADAPTERS: - print(f" ⚠ Unknown source: {source_name} (available: {', '.join(ADAPTERS.keys())})") - continue - - print(f" [{source_name}]") - adapter = ADAPTERS[source_name]() - - try: - entries = adapter.fetch(limit=limit) - except Exception as e: - print(f" ✗ {source_name} failed: {e}") - source_stats[source_name] = {"fetched": 0, "stored": 0, - "error": str(e), - "failure_class": "error"} - continue - - # Capture classification from the adapter (set by http_get on failure) - fc = getattr(adapter, "last_failure_class", None) - - # Add adapter_version to metadata - for entry in entries: - meta = json.loads(entry["raw_metadata"]) if isinstance(entry["raw_metadata"], str) else entry["raw_metadata"] - meta["adapter_version"] = "0.1" - entry["raw_metadata"] = json.dumps(meta) - - all_entries.extend(entries) - source_stats[source_name] = {"fetched": len(entries), "stored": 0, - "failure_class": fc or "ok"} - print(f" Fetched: {len(entries)} entries") - - # Small spacing between sources - time.sleep(1) - - # Store - if not dry_run and all_entries: - conn = init_db(db_path, schema_path) - stored = store_entries(conn, all_entries) - - # Update per-source stored counts - for entry in all_entries: - src = entry["source"] - if src in source_stats: - source_stats[src]["stored"] += 1 - - # --- Sprint 1 attach point: score after ingest/dedup, before render --- - try: - _scoring.attach_scoring(db_path) - except Exception as e: - print(f" ⚠ scoring attach failed: {e}") - - # Verification - if verify: - print(f"\n [Verification]") - for src in sources: - if source_stats.get(src, {}).get("stored", 0) > 0: - print(f" Checking {src}...") - verify_entries(conn, src, sample_size=3) - print() - - # Record run log (failure visibility + growth control) — before conn.close() - ok = [s for s, st in source_stats.items() if not st.get("error")] - failed = [s for s, st in source_stats.items() if st.get("error")] - # Zero-fetch (e.g. Reddit fully rate-limited) raises no exception but - # is still a degraded run — record it so run_log can tell - # "intermittent vs consistently-broken" apart over time. - zero = [s for s, st in source_stats.items() - if st.get("fetched", 0) == 0 and not st.get("error")] - notes_parts = [f"{s}: {st['error']}" for s, st in source_stats.items() if st.get("error")] - if zero: - notes_parts.append(f"no-fetch (degraded): {', '.join(zero)}") - notes = "; ".join(notes_parts) or "all sources ok" - - # Rollup failure_class (issue #2): most severe across sources. - # Priority: 5xx > 4xx > 429 > error > zero_fetch > ok - rank = {"5xx": 5, "4xx": 4, "429": 3, "error": 2, "zero_fetch": 1, "ok": 0} - classes = [st.get("failure_class", "ok") for st in source_stats.values()] - if any(c in ("5xx", "4xx", "429", "error") for c in classes): - run_fc = max((c for c in classes if c in rank), - key=lambda c: rank[c]) - elif zero: - run_fc = "zero_fetch" - else: - run_fc = "ok" - - try: - conn.execute(""" - INSERT INTO run_log (total_fetched, total_stored, sources_ok, - sources_failed, failure_class, notes) - VALUES (?, ?, ?, ?, ?, ?) - """, (len(all_entries), stored, json.dumps(ok), json.dumps(failed), - run_fc, notes)) - conn.commit() - except Exception as e: - print(f" ⚠ run_log write failed: {e}") - - conn.close() - print(f" Total stored: {stored} entries") - else: - print(f" Total fetched: {len(all_entries)} entries (dry run, not stored)") - - # Summary - print(f"\n Source summary:") - for src, stats in source_stats.items(): - error = stats.get("error", "") - error_str = f" ✗ {error}" if error else "" - print(f" {src}: {stats['fetched']} fetched, {stats.get('stored', '—')} stored{error_str}") - - # Top entries across all sources - if all_entries: - print(f"\n Recent entries by signal score (per-source ranking, NOT cross-source comparable):") - print(f" (Note: GitHub scores use actual star counts; arXiv/reddit use estimated heuristics)") - sorted_entries = sorted(all_entries, key=lambda e: e["signal_score"], reverse=True) - for i, entry in enumerate(sorted_entries[:5]): - meta = json.loads(entry["raw_metadata"]) if isinstance(entry["raw_metadata"], str) else entry["raw_metadata"] - score_type = meta.get("score_type", "?") - print(f" [{i+1}] {entry['source'].upper():6} ({score_type:8}) score={entry['signal_score']:.2f} {entry['title'][:75]}") - - print(f"\n Done.") - return all_entries - - -def main(): - parser = argparse.ArgumentParser(description="AI Research Oracle Pipeline") - parser.add_argument("--sources", default=None, help="Comma-separated sources (default: github,arxiv)") - parser.add_argument("--limit", type=int, default=20, help="Entries per source") - parser.add_argument("--dry-run", action="store_true", help="Fetch but don't store") - parser.add_argument("--verify", action="store_true", help="Spot-check entries against live sources") - args = parser.parse_args() - - sources = args.sources.split(",") if args.sources else None - run_pipeline(sources=sources, limit=args.limit, dry_run=args.dry_run, verify=args.verify) - - -if __name__ == "__main__": - main() +sys.argv = ["oracle"] + cli_args +cli_main() diff --git a/propagate_stack_now.py b/propagate_stack_now.py index 189cf11..98882a8 100644 --- a/propagate_stack_now.py +++ b/propagate_stack_now.py @@ -1,277 +1,8 @@ #!/usr/bin/env python3 -"""One-shot stack propagator (explicit user request 2026-07-12, rev 2.2). +"""Thin wrapper — delegates to oracle.render (full publish, not dry-run).""" +import sys, os +sys.path.insert(0, os.path.dirname(__file__)) -Editorial rules applied (reuses BUILT-IN pipeline functions, no pipeline edits): - - clickability.compute_index / decay_index (virality rank + 18h decay) - - generate_from_athena.clean_headline (repo-prefix trim, emoji strip, length cap) - - generate_from_athena.add_prefix (Breaking | text prefix, BREAKING GATE) - - render_site._clean_summary (one-liner descriptions on every card) - -USER DIRECTIVES (2026-07-12): - 1. GitHub source EXCLUDED entirely (until further notice). - 2. 'update' green tier REMOVED. Only 'breaking' (rare real events) or 'normal'. - 3. Curated Picks section surfaces two flavors (tight deterministic phrase match, - no broad keywords to avoid false positives): - (a) QUIRKY + agents roasting their humans - (b) people who BUILT / SHIPPED / EARNED from an AI product (indie hackers) -Window: last DAYS days (default 4). Cap: LIMIT (default 200) — GitHub ban caps the -real max at ~180 over 4 days; we render whatever is eligible (never fake count). -""" -import os, re, sys, json, sqlite3, html as _html -from datetime import datetime as dt, timezone, timedelta -from collections import OrderedDict - -ORACLE = "/home/vpsadmin/oracle" -sys.path.insert(0, ORACLE) -sys.path.insert(0, "/home/vpsadmin/ai-oracle-site") -import clickability as cb -import render_site as rs -import generate_from_athena as ga - -DB = os.path.join(ORACLE, "oracle.db") -WEBROOT = "/var/www/preprod3" -FALLBACK = os.path.join(ORACLE, "site") -NOW = dt.now(timezone.utc) -DAYS = 14 # span whole DB so all 182 non-GitHub entries are eligible (DB only goes back ~7d) -LIMIT = 200 # hard ceiling: DB only has 182 non-GitHub entries total, so 182 will render -EXCLUDE_SOURCES = {"github"} # banned until further notice - -# BREAKING GATE (verbatim pipeline logic; repos/papers/models never breaking) -REPO_SOURCES = {"github", "gitlab", "huggingface", "arxiv"} -IMPORTANCE = re.compile( - r"\b(sues?|sue|lawsuit|launches?|launch|releases?|release|" - r"bans?|ban|war|strikes?|attack|acquires?|acquisition|trillion|billions?|" - r"layoffs?|declares?|emergency|outage|breach|stolen|steals?|theft|antitrust|" - r"monopoly|reveals?|exposed|breakthrough|first|warns?|crackdown|shutdown|" - r"GPT-?5|Claude|Gemini|OpenAI|Anthropic|Google|Apple|Microsoft|Meta|xAI|" - r"Musk|Altman|Grok|DeepSeek|Llama|NVIDIA|AMD|FCC|EU|antitrust|" - r"folded|spins? off|partners?|raises?|ipo|funding)\\b", re.I) -BREAKING_PCT = 0.90 - -# --- CURATION: QUIRKY + agents roasting their humans ONLY (deterministic; no LLM) --- -# Standing directive 2026-07-12 (end of session): "shipped & paid / built & earned" -# was WALKED BACK ("looking for people who build and ship products is a whole -# separate issue"). Do NOT bake it in. Curation = quirky + agents-roasting-humans. -QUIRKY = [ - "hit piece", "roast", "roasting", "insult", "revenge", "betray", - "bizarre", "weird", "cursed", "font humans", "brain region", - "conspiracy", "haunted", "absurd", "unhinged", "sentient", "scream", - "mock", "taunt", "expose their", "its human", "its user", "their owner", - "about their", "their creator", "their master", "turned on", "backstab", - "wrote about its", "turned against", "rebelled", "sassy", "savage", -] -# built / shipped / EARNED from an AI product (FIRST-PERSON builder only — -# tight phrases; bare 'revenue'/'funding'/'ipo' EXCLUDED to avoid industry-news -# false positives like TechCrunch "startups growing revenue"). -BUILT_SHIPPED = [ - "indie hacker", "i built", "i made", "i shipped", "i launched", "i sold", - "my saas", "my startup", "my app", "my product", "my business", - "side project", "bootstrapped", "profitable", "paying customers", - "made money", "earn money", "mrr", "monthly recurring", "i run a", - "made me $", "income from", "subscriptions", "sold my", "quit my job", - "shipped a", "built a", "customers pay", "my first", "passive income", -] - - -def _parse(ts): - if not ts: - return None - try: - return dt.fromisoformat(ts.replace("Z", "+00:00")) - except Exception: - return None - - -def _curation(it): - blob = f"{(it.get('title') or '')} {(rs._clean_summary(it.get('summary') or ''))}".lower() - if any(k in blob for k in BUILT_SHIPPED): - return ("built", 1.22) - if any(k in blob for k in QUIRKY): - return ("quirky", 1.16) - return (None, 1.0) - - -def main(): - conn = sqlite3.connect(f"file:{DB}?mode=ro", uri=True) - items = cb.fetch_items(conn) - conn.close() - items = cb.compute_index(items) - items = cb.decay_index(items, rs.HALF_LIFE_H) - - cutoff = NOW - timedelta(days=DAYS) - eligible = [it for it in items - if it.get("title") and it.get("url") - and it.get("first_seen") and _parse(it["first_seen"]) - and _parse(it["first_seen"]) >= cutoff - and (it.get("source") or "").lower() not in EXCLUDE_SOURCES] - # --- RECENCY GUARD (2026-07-13, Tony's correction): NEVER re-post old news. - # Age is the dominant measure: today's items ALWAYS lead (week-open fresh - # news); older items keep only if NEVER posted before (md-stack / seen). - # This kills the Apple-vs-OpenAI / GPT-5.6 re-post problem at the source. --- - from recency_guard import filter_fresh as _rg_filter - _today, _older_new, _dropped = _rg_filter(eligible) - if _dropped: - print(f"[recency_guard] dropped {len(_dropped)} already-posted older items") - eligible = _today + _older_new - eligible.sort(key=lambda x: x["clickability_decayed"], reverse=True) - top = eligible[:LIMIT] - - scores = [it["clickability_decayed"] for it in top] - n = len(scores) - - def pct_rank(v): - beaten = sum(1 for s in scores if s <= v) - return beaten / n if n else 0.0 - - for it in top: - src = (it.get("source") or "").lower() - pr = pct_rank(it["clickability_decayed"]) - is_repo = src in REPO_SOURCES - important = bool(IMPORTANCE.search(it.get("title") or "")) - if (not is_repo) and important and pr >= BREAKING_PCT: - tier = "breaking" - else: - tier = "normal" - cleaned = ga.clean_headline(it["title"], it.get("source", "")) - it["title"] = ga.add_prefix(cleaned, it["url"], tier) - it["_tier"] = tier - label, mult = _curation(it) - it["_curated"] = label - it["clickability_decayed"] = it["clickability_decayed"] * mult - - ranked = sorted(top, key=lambda x: x["clickability_decayed"], reverse=True) - fresh = [it for it in ranked if it.get("fresh")] - top_cards = fresh[:rs.TOP_N] - stack = [it for it in ranked if it not in top_cards] - curated = [it for it in ranked if it.get("_curated")] - curated.sort(key=lambda x: x["clickability_decayed"], reverse=True) - curated_cards = curated[:12] - - by_day = OrderedDict() - for it in stack: - day = (it.get("first_seen") or "")[:10] or "unknown" - by_day.setdefault(day, []).append(it) - - def card(it): - title = _html.escape(it["title"] or "(untitled)") - url = _html.escape(it["url"] or "#") - src = _html.escape(it["source"]) - sig = it.get("signal_score") or 0 - t = rs._fmt_time(it.get("first_seen")) - summary = _html.escape(rs._clean_summary(it.get("summary") or "")[:200]) - cls = "card" - if it.get("_tier") == "breaking": - cls += " breaking" - if it.get("_curated"): - cls += " curated" - badge = "" - if it.get("_curated") == "built": - badge = '\U0001f4b0 Built & Earned' - elif it.get("_curated") == "quirky": - badge = '\U0001f300 Quirky' - sum_html = f'

{summary}

' if summary else "" - return f""" -
-
{src} - {t} - sig {sig:.1f} - {badge} - \U0001f525 {it['clickability_decayed']:.2f}
-

{title}

- {sum_html} -
""" - - top_html = "".join(card(it) for it in top_cards) - curated_html = "".join(card(it) for it in curated_cards) - stack_html = "" - for day, rows in by_day.items(): - rows.sort(key=lambda x: x["clickability_decayed"], reverse=True) - cards = "".join(card(it) for it in rows) - stack_html += f""" -

\U0001f4c5 {day}

-
{cards}
""" - - now_str = NOW.strftime("%Y-%m-%d %H:%M UTC") - page = f""" - - - - -Athena AI News — Ranked by Clickability - - - -
-

Athena AI News

-
Auto-ranked by Clickability Index · {len(top)} stories (4-day window, GitHub excluded) · curated: quirky + agents roasting their humans · generated {now_str}
-
-
-

\U0001f4b0\U0001f300 Curated Picks — Built & Earned · Quirky · Agents Roasting Their Humans

-
{curated_html}
-

\U0001f534 Top News

-
{top_html}
-

\U0001f4f0 The Stack

- {stack_html} -
- -""" - - target = WEBROOT if os.path.isdir(WEBROOT) else FALLBACK - os.makedirs(target, exist_ok=True) - with open(os.path.join(target, "index.html"), "w") as f: - f.write(page) - with open(os.path.join(target, "feed.json"), "w") as f: - json.dump([ - {"title": i["title"], "url": i["url"], "source": i["source"], - "tier": i.get("_tier"), "curated": i.get("_curated"), - "clickability_decayed": round(i["clickability_decayed"], 3), - "age_hours": i["age_hours"], "first_seen": i.get("first_seen")} - for i in ranked - ], f, indent=2) - - where = "WEBROOT(/var/www/preprod3)" if target == WEBROOT else "FALLBACK(~oracle/site)" - tiers = {"breaking": 0, "normal": 0} - for it in top: - tiers[it["_tier"]] += 1 - cc = {"built": 0, "quirky": 0, "none": 0} - for it in top: - cc[it["_curated"] or "none"] += 1 - with_desc = sum(1 for it in top if rs._clean_summary(it.get("summary") or "")) - print(f"[propagate v2.2] wrote {target}/index.html + feed.json") - print(f" target : {where}") - print(f" window : last {DAYS} days, GitHub EXCLUDED") - print(f" eligible : {len(eligible)} (cap {LIMIT} -> rendered {len(top)})") - print(f" tiers : {tiers['breaking']} breaking / {tiers['normal']} normal (update tier REMOVED)") - print(f" curated flags : {cc['built']} built&earned | {cc['quirky']} quirky | {cc['none']} none") - print(f" curated shown : top {len(curated_cards)} in Curated Picks section") - print(f" with desc : {with_desc}/{len(top)} cards have a one-liner description") - - -if __name__ == "__main__": - main() +from oracle.cli import main as cli_main +sys.argv = ["oracle", "render"] +cli_main() diff --git a/query.py b/query.py index fb18230..6b8fc9b 100644 --- a/query.py +++ b/query.py @@ -1,459 +1,17 @@ #!/usr/bin/env python3 -""" -AI Research Oracle — Query & Snapshot CLI. - -Query the unified database and produce Claude-ready snapshots. -Supports filtering by source, score, confidence, tag, and date range. - -Usage: - python3 query.py top 10 # top 10 across all sources - python3 query.py by-source github 5 # top 5 from GitHub - python3 query.py by-tag "agent" # entries tagged with "agent" - python3 query.py snapshot # full snapshot for Claude relay - python3 query.py search "world model" # keyword search in titles/summaries - python3 query.py recent --hours 24 # entries from last 24h - python3 query.py stats # database statistics -""" - -import argparse -import json -import os -import sqlite3 -import sys -from datetime import datetime, timezone, timedelta -from urllib.parse import quote_plus - +"""Thin wrapper — delegates to oracle.cli query subcommand.""" +import sys, os sys.path.insert(0, os.path.dirname(__file__)) -DB_PATH = os.path.join(os.path.dirname(__file__), "oracle.db") - - -def get_db(): - """Open database connection.""" - conn = sqlite3.connect(DB_PATH) - conn.row_factory = sqlite3.Row - return conn - - -def format_entry(row: sqlite3.Row, rank: int = 0) -> dict: - """Format a DB row into a clean dict for output.""" - summary = json.loads(row["summary"]) if row["summary"] else {} - meta = json.loads(row["raw_metadata"]) if row["raw_metadata"] else {} - tags = json.loads(row["category_tags"]) if row["category_tags"] else [] - - score_type = meta.get("score_type", "?") - source_detail = "" - if row["source"] == "github": - source_detail = f"⭐ {meta.get('stars', '?')} stars" - elif row["source"] == "arxiv": - source_detail = f"arXiv:{meta.get('arxiv_id', '?')}" - elif row["source"] == "reddit": - source_detail = f"r/{meta.get('subreddit', '?')}" - - return { - "rank": rank, - "source": row["source"], - "title": row["title"], - "url": row["url"], - "score": row["signal_score"], - "score_type": score_type, - "confidence": summary.get("confidence", "?"), - "source_detail": source_detail, - "tags": tags, - "one_liner": summary.get("one_liner", ""), - "key_technical_point": summary.get("key_technical_point", ""), - "potential_use_case": summary.get("potential_use_case", ""), - "first_seen": row["first_seen"], - } - - -def cmd_top(args): - """Top N entries across all sources (per-source ranking).""" - conn = get_db() - cur = conn.cursor() - - # Score filter - min_score = getattr(args, 'min_score', 0) or 0 - # Confidence filter - min_confidence = getattr(args, 'min_confidence', None) or None - # Source filter - source_filter = getattr(args, 'source', None) or None - - where = [] - params = [] - if min_score > 0: - where.append("signal_score >= ?") - params.append(min_score) - if min_confidence: - where.append("json_extract(summary,'$.confidence') = ?") - params.append(min_confidence) - if source_filter: - where.append("source = ?") - params.append(source_filter) - - where_str = (" AND " if where else "") + " AND ".join(where) if where else "" - limit = args.n or 10 - - cur.execute(f""" - SELECT * FROM entries {where_str} - ORDER BY signal_score DESC - LIMIT ? - """, params + [limit]) - - rows = cur.fetchall() - entries = [format_entry(r, i+1) for i, r in enumerate(rows)] - - print(f"Top {len(entries)} entries{' by score' if not source_filter else f' from {source_filter}'}:\n") - _print_entries(entries) - conn.close() - - -def cmd_by_source(args): - """Top N from a specific source.""" - conn = get_db() - cur = conn.cursor() - cur.execute(""" - SELECT * FROM entries WHERE source = ? - ORDER BY signal_score DESC - LIMIT ? - """, (args.source, args.n or 10)) - - rows = cur.fetchall() - entries = [format_entry(r, i+1) for i, r in enumerate(rows)] - - print(f"Top {len(entries)} from {args.source}:\n") - _print_entries(entries) - conn.close() - - -def cmd_by_tag(args): - """Entries matching a tag.""" - conn = get_db() - cur = conn.cursor() - cur.execute(""" - SELECT * FROM entries - WHERE json_extract(category_tags,'$') LIKE ? - ORDER BY signal_score DESC - LIMIT 20 - """, (f'%"{args.tag}"%',)) - - rows = cur.fetchall() - entries = [format_entry(r, i+1) for i, r in enumerate(rows)] - - print(f"Entries tagged '{args.tag}':\n") - _print_entries(entries) - conn.close() - - -def cmd_search(args): - """Keyword search in titles and summaries.""" - conn = get_db() - cur = conn.cursor() - q = f"%{args.query}%" - cur.execute(""" - SELECT * FROM entries - WHERE title LIKE ? - OR json_extract(summary,'$.one_liner') LIKE ? - OR json_extract(summary,'$.key_technical_point') LIKE ? - ORDER BY signal_score DESC - LIMIT 20 - """, (q, q, q)) - - rows = cur.fetchall() - entries = [format_entry(r, i+1) for i, r in enumerate(rows)] - - print(f"Search results for '{args.query}':\n") - _print_entries(entries) - conn.close() - - -def cmd_recent(args): - """Entries from the last N hours.""" - hours = args.hours or 24 - cutoff = (datetime.now(timezone.utc) - timedelta(hours=hours)).strftime("%Y-%m-%dT%H:%M:%SZ") - conn = get_db() - cur = conn.cursor() - cur.execute(""" - SELECT * FROM entries WHERE first_seen >= ? - ORDER BY first_seen DESC - """, (cutoff,)) - - rows = cur.fetchall() - entries = [format_entry(r, i+1) for i, r in enumerate(rows)] - - print(f"Entries from last {hours}h:\n") - _print_entries(entries) - conn.close() - - -def cmd_snapshot(args): - """Full snapshot for Claude relay. - - Produces a structured summary of the current DB state, - formatted for Claude to reason over. - """ - conn = get_db() - cur = conn.cursor() - - # DB stats - cur.execute("SELECT COUNT(*) FROM entries") - total = cur.fetchone()[0] - - cur.execute("SELECT source, COUNT(*) as cnt, AVG(signal_score) as avg_score, MIN(first_seen) as oldest, MAX(last_updated) as newest FROM entries GROUP BY source") - source_stats = {r["source"]: dict(r) for r in cur.fetchall()} - - # Security-tagged entries - cur.execute(""" - SELECT COUNT(*) FROM entries - WHERE summary IS NOT NULL AND summary != '' AND json_extract(summary,'$.potential_use_case') LIKE '%security%' - """) - security_count = cur.fetchone()[0] - - # Top 10 overall - cur.execute("SELECT * FROM entries WHERE summary IS NOT NULL AND summary != '' ORDER BY signal_score DESC LIMIT 10") - top_entries = [format_entry(r, i+1) for i, r in enumerate(cur.fetchall())] - - # Confidence distribution - cur.execute(""" - SELECT json_extract(summary,'$.confidence') as conf, COUNT(*) as cnt - FROM entries WHERE summary IS NOT NULL AND summary != '' - GROUP BY conf - """) - conf_dist = {r["conf"]: r["cnt"] for r in cur.fetchall()} - - now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - snapshot = { - "snapshot_time": now, - "total_entries": total, - "sources": source_stats, - "security_flagged": security_count, - "confidence_distribution": conf_dist, - "top_10": top_entries, - } - - # Output as formatted text for relay - print("=" * 70) - print("AI RESEARCH ORACLE — SNAPSHOT") - print("=" * 70) - print(f"Time: {now}") - print(f"Total entries: {total}") - print() - - print("Source breakdown:") - for src, stats in source_stats.items(): - print(f" {src}: {stats['cnt']} entries, avg score {stats['avg_score']:.2f}") - print() - - if conf_dist: - print(f"Confidence distribution: {conf_dist}") - print() - - if security_count: - print(f"⚠ {security_count} entries flagged as security:dual-use") - print() - - print("Top 10 by signal score (per-source ranking, NOT cross-source comparable):") - print("-" * 70) - for e in top_entries: - score_label = f"{e['score']:.2f} ({e['score_type']})" - conf = e["confidence"] - print(f"\n [{e['rank']}] {e['source'].upper()} | {score_label} | confidence={conf}") - print(f" {e['title']}") - print(f" {e['source_detail']}") - if e["one_liner"]: - print(f" → {e['one_liner'][:120]}") - print(f" Tags: {', '.join(e['tags'][:4])}") - - print("\n" + "=" * 70) - print("NOTE: Scores are NOT comparable across sources. GitHub uses") - print("actual star counts (log-scaled), arXiv/reddit use estimated") - print("heuristics. Rank within-source, not cross-source.") - print("=" * 70) - - conn.close() - - -def cmd_explain(args): - """Explain why an entry scored the way it did.""" - conn = get_db() - cur = conn.cursor() - - if args.entry_id.isdigit(): - cur.execute("SELECT * FROM entries WHERE id = ?", (args.entry_id,)) - else: - q = f"%{args.entry_id}%" - cur.execute("SELECT * FROM entries WHERE title LIKE ?", (q,)) - - row = cur.fetchone() - if not row: - print(f"Entry not found: {args.entry_id}") - conn.close() - return - - meta = json.loads(row["raw_metadata"]) if row["raw_metadata"] else {} - summary = json.loads(row["summary"]) if row["summary"] else {} - tags = json.loads(row["category_tags"]) if row["category_tags"] else [] - - print(f"=== Score Explanation ===\n") - print(f"Title: {row['title']}") - print(f"Source: {row['source']}") - print(f"Score: {row['signal_score']:.2f} ({meta.get('score_type', '?')})") - print(f"Confidence: {summary.get('confidence', '?')}") - print() - - if row["source"] == "arxiv": - print(f"Authors: {meta.get('author_count', '?')}") - print(f"Categories: {', '.join(meta.get('categories', []))}") - print(f"Published: {meta.get('published', '?')}") - print(f"Abstract length: {meta.get('abstract_length', '?')} chars") - print(f"Applied domain: {meta.get('applied_domain', 'none (core AI)')}") - print() - print("Scoring (arXiv — relevance-weighted, structural capped):") - print(f" - Structural (recency+authors+diversity, capped ~3.5): weak signals") - print(f" - AI keyword density in abstract (capped 2.0)") - print(f" - AI methodology claim (propose/novel = up to 2.5)") - print(f" - Applied-domain is TAGGED, not penalized") - if meta.get('applied_domain'): - print(f" Note: tagged '{meta['applied_domain']}' for filtering — no score penalty") - elif row["source"] == "github": - print(f"Stars: {meta.get('stars', '?')}") - print(f"Language: {meta.get('language', '?')}") - print() - print("Scoring (GitHub actual star count, log-scaled)") - - print() - print(f"Tags: {', '.join(tags)}") - if summary.get('one_liner'): - print(f"One-liner: {summary['one_liner'][:120]}") - - conn.close() - - -def cmd_stats(args): - """Database statistics.""" - conn = get_db() - cur = conn.cursor() - - cur.execute("SELECT COUNT(*) FROM entries") - total = cur.fetchone()[0] - - cur.execute("SELECT source, COUNT(*) as cnt, ROUND(AVG(signal_score),2) as avg_score, MIN(signal_score) as min_score, MAX(signal_score) as max_score FROM entries GROUP BY source") - rows = cur.fetchall() - - print(f"Database: {DB_PATH}") - print(f"Total entries: {total}\n") - - print(f"{'Source':<12} {'Count':<8} {'Avg':<8} {'Min':<8} {'Max':<8}") - print("-" * 44) - for r in rows: - print(f"{r['source']:<12} {r['cnt']:<8} {r['avg_score']:<8} {r['min_score']:<8} {r['max_score']:<8}") - - # Summarization status - cur.execute("SELECT COUNT(*) FROM entries WHERE summary IS NOT NULL") - summarized = cur.fetchone()[0] - cur.execute("SELECT COUNT(*) FROM entries WHERE summary IS NULL") - pending = cur.fetchone()[0] - print(f"\nSummarization: {summarized} done, {pending} pending") - - # Confidence distribution - cur.execute("SELECT json_extract(summary,'$.confidence') as c, COUNT(*) as n FROM entries WHERE summary IS NOT NULL AND summary != '' GROUP BY c") - if cur.fetchall(): - print(f"Confidence: {' | '.join(f'{r[0]}={r[1]}' for r in cur.fetchall())}") - - # Recent run history (failure visibility) - cur.execute("SELECT run_time, total_fetched, total_stored, sources_ok, sources_failed FROM run_log ORDER BY id DESC LIMIT 5") - runs = cur.fetchall() - if runs: - print(f"\nRecent runs (last {len(runs)}):") - for r in runs: - failed = json.loads(r["sources_failed"]) if r["sources_failed"] else [] - status = "✓ all ok" if not failed else f"⚠ partial: {', '.join(failed)}" - print(f" {r['run_time']} fetched={r['total_fetched']} stored={r['total_stored']} {status}") - - conn.close() - - -def _print_entries(entries: list[dict]): - """Pretty-print a list of entries.""" - if not entries: - print(" (no results)") - return - - for e in entries: - score_label = f"{e['score']:.2f} ({e['score_type']})" - conf = e["confidence"] - print(f" [{e['rank']}] {e['source'].upper():6} | {score_label} | confidence={conf}") - print(f" {e['title']}") - if e["source_detail"]: - print(f" {e['source_detail']}") - if e["one_liner"]: - print(f" → {e['one_liner'][:120]}") - # Show applied-domain and security tags prominently - shown_tags = [t for t in e["tags"] if t.startswith("applied:") or t.startswith("security:")] - other_tags = [t for t in e["tags"] if not t.startswith("applied:") and not t.startswith("security:")] - display_tags = shown_tags + other_tags[:4] - if display_tags: - print(f" Tags: {', '.join(display_tags)}") - print() - - -def main(): - parser = argparse.ArgumentParser(description="AI Research Oracle — Query") - sub = parser.add_subparsers(dest="command") - - # top - p_top = sub.add_parser("top", help="Top N entries") - p_top.add_argument("n", type=int, nargs="?", default=10) - p_top.add_argument("--source", default=None) - p_top.add_argument("--min-score", type=float, default=0) - - # by-source - p_src = sub.add_parser("by-source", help="Top N from a source") - p_src.add_argument("source") - p_src.add_argument("n", type=int, nargs="?", default=10) - - # by-tag - p_tag = sub.add_parser("by-tag", help="Entries by tag") - p_tag.add_argument("tag") - - # search - p_search = sub.add_parser("search", help="Keyword search") - p_search.add_argument("query") - - # recent - p_recent = sub.add_parser("recent", help="Recent entries") - p_recent.add_argument("--hours", type=int, default=24) - - # snapshot - sub.add_parser("snapshot", help="Full snapshot for Claude") - - # explain - p_explain = sub.add_parser("explain", help="Explain why an entry scored high") - p_explain.add_argument("entry_id", help="Entry ID or partial title") - - # stats - sub.add_parser("stats", help="Database statistics") - - args = parser.parse_args() - - commands = { - "top": cmd_top, - "by-source": cmd_by_source, - "by-tag": cmd_by_tag, - "search": cmd_search, - "recent": cmd_recent, - "snapshot": cmd_snapshot, - "explain": cmd_explain, - "stats": cmd_stats, - } - - cmd = commands.get(args.command) - if cmd: - cmd(args) - else: - parser.print_help() - - -if __name__ == "__main__": - main() +args = sys.argv[1:] +# Map old query commands to new CLI format +if args and args[0] in ("top", "search", "recent", "by-tag", "stats", "snapshot"): + cli_args = ["query"] + args +elif args and args[0].startswith("-"): + cli_args = ["query", "top"] + args +else: + cli_args = ["query", "top"] + args + +from oracle.cli import main as cli_main +sys.argv = ["oracle"] + cli_args +cli_main() diff --git a/recency_guard.py b/recency_guard.py index 0eaa973..861293f 100644 --- a/recency_guard.py +++ b/recency_guard.py @@ -1,201 +1,29 @@ #!/usr/bin/env python3 -""" -recency_guard.py — Athena "how old is this news?" gate (Tony's correction, 2026-07-13). +"""Thin wrapper — delegates to oracle.recency filter_fresh + analysis.""" +import sys, os +sys.path.insert(0, os.path.dirname(__file__)) -THE PROBLEM IT SOLVES: - The old pipeline ranked by virality + editorial fit ONLY. That surfaced the - SAME stories week after week (Apple-vs-OpenAI, GPT-5.6, etc.) because the - curation had NO memory of what was already posted. Tony's rule: - - "when you receive the news stories, morality [editorial fit] is only ONE - measure. The OTHER measure -- maybe MORE important -- is how OLD is the - news. Today is Monday; the week's news is just beginning, so we begin the - week with stories dated for TODAY." - -So AGE is the dominant gate. The algorithm (editorial, not render-time): - - * TODAY's items -> ALWAYS eligible. They are this week's fresh news; a - Monday stack leads with them even if rendered earlier today. - * OLDER items -> eligible ONLY if NEVER posted before (not in the - markdown Top-N stack history AND not in seen_urls.json). - This kills the re-post problem at the source. - -WHAT IS "ALREADY POSTED": - Two signals, differing in authority: - - athena_top*.md = the CURATED/PUBLIC stack history (authoritative) - - seen_urls.json = the live-site render dedup (secondary; ALSO flags items - rendered in prior runs TODAY, which we must NOT drop) - Because seen_urls.json contains today's own items, we only treat a seen_url - as "already posted" when the candidate is OLDER than today. Today's items are - exempt from the seen_urls gate entirely (fresh by definition). - -EXPORTS: - load_posted() -> (md_urls, md_titles, seen_urls) sets - is_today(first_seen, now) - age_days(first_seen, now) - day_bucket(first_seen, now) - already_posted_fs(url,title,fs,now) -> bool (age-aware dedup) - filter_fresh(items, now) -> (today_items, older_new_items, dropped_items) - recency_weight(first_seen, now) -> float (1.0 today -> ~0 over 7d) - blend_score(item, now) -> clickability_decayed * recency_weight -Read-only against markdown + json + passed-in items. No DB writes. -""" -import os, re, json from datetime import datetime, timezone +from oracle.clickability import fetch_items, compute_index, decay_index +from oracle.recency import filter_fresh, blend_score +import sqlite3 +from oracle.config import DB_PATH, HALF_LIFE_H -ORACLE = os.path.dirname(os.path.abspath(__file__)) -SEEN_JSON = "/home/vpsadmin/ai-oracle-site/seen_urls.json" +conn = sqlite3.connect(str(DB_PATH)) +items = fetch_items(conn) +conn.close() +items = compute_index(items) +items = decay_index(items, HALF_LIFE_H) -def _parse(ts): - if not ts: - return None - try: - return datetime.fromisoformat(ts.replace("Z", "+00:00")) - except Exception: - return None +today, older_new, dropped = filter_fresh(items) +print(f"=== Recency Guard ===") +print(f" Total items: {len(items)}") +print(f" TODAY (always eligible): {len(today)}") +print(f" OLDER but never posted: {len(older_new)}") +print(f" DROPPED (posted + old): {len(dropped)}") +print() -def _norm_url(u): - if not u: - return "" - return u.split("?")[0].split("#")[0].rstrip("/").lower() - - -def _norm_title(t): - if not t: - return "" - t = t.lower() - t = re.sub(r"[^a-z0-9 ]", " ", t) - t = re.sub(r"\s+", " ", t).strip() - return t[:60] - - -def load_posted(md_dir=ORACLE, seen_json=SEEN_JSON): - """Return (md_urls:set, md_titles:set, seen_urls:set).""" - md_urls, md_titles, seen_urls = set(), set(), set() - - for fn in sorted(os.listdir(md_dir)): - if re.match(r"athena_top.*\.md$", fn): - try: - txt = open(os.path.join(md_dir, fn), encoding="utf-8", errors="replace").read() - except OSError: - continue - for m in re.findall(r"\]\((https?://[^)\s]+)\)", txt): - nu = _norm_url(m) - if nu: - md_urls.add(nu) - for t in re.findall(r"^\|\s*\d+\s*\|\s*(.+?)\s*\|", txt, re.M): - nt = _norm_title(t) - if nt: - md_titles.add(nt) - - if os.path.exists(seen_json): - try: - with open(seen_json, encoding="utf-8") as f: - for u in json.load(f): - nu = _norm_url(u) - if nu: - seen_urls.add(nu) - except (json.JSONDecodeError, OSError): - pass - - return md_urls, md_titles, seen_urls - - -def is_today(first_seen, now=None): - now = now or datetime.now(timezone.utc) - d = _parse(first_seen) - return bool(d) and d.strftime("%Y-%m-%d") == now.strftime("%Y-%m-%d") - - -def age_days(first_seen, now=None): - now = now or datetime.now(timezone.utc) - d = _parse(first_seen) - if not d: - return 9999.0 - return max((now - d).total_seconds() / 86400.0, 0.0) - - -def day_bucket(first_seen, now=None): - """'today' | 'yesterday' | 'this-week' (<=6d) | 'older'.""" - days = age_days(first_seen, now) - if days < 1: - return "today" - if days < 2: - return "yesterday" - if days <= 6: - return "this-week" - return "older" - - -def already_posted_fs(url, title, first_seen, now=None, - md_urls=None, md_titles=None, seen_urls=None): - """Age-aware dedup. A candidate is 'already posted' iff: - (a) it matches the curated md-stack history, OR - (b) it is OLDER than today AND its URL is in seen_urls.json. - Today's items are NEVER flagged -- they are fresh by definition. - """ - if md_urls is None or md_titles is None or seen_urls is None: - md_urls, md_titles, seen_urls = load_posted() - if _norm_url(url) in md_urls: - return True - nt = _norm_title(title) - if nt and nt in md_titles: - return True - if is_today(first_seen, now): - return False - if _norm_url(url) in seen_urls: - return True - return False - - -def recency_weight(first_seen, now=None, half_life_days=2.0): - """1.0 for today, decays ~halving every 2 days. The 'age' measure.""" - return 0.5 ** (age_days(first_seen, now) / half_life_days) - - -def blend_score(item, now=None): - """clickability_decayed * recency_weight. Today's items dominate; old sink.""" - base = item.get("clickability_decayed", 0) or 0 - return base * recency_weight(item.get("first_seen"), now) - - -def filter_fresh(items, now=None): - """Split into (today_items, older_new_items, dropped_items). - - today_items = first_seen == today (always eligible; the week-open lead) - older_new_items= older, but never before posted (md/seen) - dropped_items = older AND already posted (the re-posts we are killing) - """ - now = now or datetime.now(timezone.utc) - md_urls, md_titles, seen_urls = load_posted() - today_items, older_new, dropped = [], [], [] - for it in items: - fs = it.get("first_seen") - if is_today(fs, now): - today_items.append(it) - continue - if already_posted_fs(it.get("url"), it.get("title"), fs, now, - md_urls, md_titles, seen_urls): - dropped.append(it) - else: - older_new.append(it) - return today_items, older_new, dropped - - -if __name__ == "__main__": - import sys - sys.path.insert(0, ORACLE) - import clickability as cb - DB = os.path.join(ORACLE, "oracle.db") - conn = sqlite3.connect(f"file:{DB}?mode=ro", uri=True) - items = cb.fetch_items(conn); conn.close() - items = cb.compute_index(items); items = cb.decay_index(items) - today_items, older_new, dropped = filter_fresh(items) - print(f"TODAY-new (week-open lead): {len(today_items)}") - print(f"OLDER-but-never-posted: {len(older_new)}") - print(f"DROPPED (already posted): {len(dropped)}") - print("\nSample dropped (the re-posts that caused the problem):") - for it in sorted(dropped, key=lambda x: -x["clickability_decayed"])[:6]: - print(f" - [{it['clickability_decayed']:.3f}] {it['title'][:66]}") +for it in sorted(today, key=lambda x: x["clickability_decayed"], reverse=True)[:5]: + print(f" [TODAY {it['clickability_decayed']:.2f}] {it['source']:10} {it['title'][:60]}") diff --git a/render_site.py b/render_site.py index 728f6d7..7af5cb9 100644 --- a/render_site.py +++ b/render_site.py @@ -1,185 +1,11 @@ #!/usr/bin/env python3 -"""Render Athena entries into a static news site (two-layer: Top News + aging Stack). +"""Thin wrapper — delegates to oracle.cli render subcommand.""" +import sys, os +sys.path.insert(0, os.path.dirname(__file__)) -Read-only against oracle.db. Writes static HTML to the preprod3 webroot. -Designed for a 20-min no_agent cron. +args = sys.argv[1:] +cli_args = ["render"] + args -Usage: - python3 render_site.py # write to WEBROOT - python3 render_site.py --dry-run # print stats, write to ./_preview.html -""" -import argparse, html, os, json, sqlite3, datetime, re -from collections import OrderedDict - -import clickability as cb - -HERE = os.path.dirname(os.path.abspath(__file__)) -WEBROOT = "/var/www/preprod2" -DB_PATH = os.path.join(HERE, "oracle.db") -TOP_N = 8 -HALF_LIFE_H = 18.0 - - -def _clean_summary(raw): - """summary is stored as JSON {one_liner, key_technical_point, potential_use_case}. - Pull the most readable field; fall back to the raw text if it isn't JSON. - Strips markdown/latex noise so the card reads clean on the page.""" - if not raw: - return "" - try: - d = json.loads(raw) - if isinstance(d, dict): - for k in ("one_liner", "key_technical_point", "potential_use_case"): - v = d.get(k) - if isinstance(v, str) and v.strip(): - return re.sub(r"\\+|_|`", "", v).strip() - except Exception: - pass - return re.sub(r"\\+|_|`", "", raw).strip() - - -def _fmt_time(first_seen): - if not first_seen: - return "" - try: - dt = datetime.datetime.strptime(first_seen, "%Y-%m-%dT%H:%M:%SZ") - return dt.strftime("%H:%M") - except Exception: - return "" - - -def _card(it, big=False): - title = html.escape(it["title"] or "(untitled)") - url = html.escape(it["url"] or "#") - src = html.escape(it["source"]) - sig = it.get("signal_score") or 0 - t = _fmt_time(it.get("first_seen")) - summary_raw = _clean_summary(it.get("summary") or "") - summary = html.escape(summary_raw[:200]) - cls = "card big" if big else "card" - summary_html = ('

{0}

'.format(summary)) if (summary and big) else "" - return f""" -
-
{src} - {t} - sig {sig:.1f} - \U0001f525 {it['clickability_decayed']:.2f}
-

{title}

- {summary_html} -
""" - - -def build_html(items): - now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC") - ranked = sorted(items, key=lambda x: x["clickability_decayed"], reverse=True) - # Top News = fresh items only (ingested today, UTC). Yesterday's viral - # leftovers sink into the Stack instead of dominating the front page. - fresh = [it for it in ranked if it.get("fresh")] - top = fresh[:TOP_N] - stack = [it for it in ranked if it not in top] - - # group stack by day (first_seen date) - by_day = OrderedDict() - for it in stack: - day = (it.get("first_seen") or "")[:10] or "unknown" - by_day.setdefault(day, []).append(it) - - top_html = "".join(_card(it, big=True) for it in top) - - stack_html = "" - for day, rows in by_day.items(): - rows.sort(key=lambda x: x["clickability_decayed"], reverse=True) - cards = "".join(_card(it) for it in rows) - stack_html += f""" -

\U0001f4c5 {html.escape(day)}

-
{cards}
""" - - return f""" - - - - -Athena AI News — Ranked by Clickability - - - -
-

Athena AI News

-
Auto-ranked by Clickability Index · decays with age so the stack flows top → bottom · generated {now} · {len(items)} stories
-
-
-

\U0001f534 Top News

-
{top_html}
-

\U0001f4f0 The Stack

- {stack_html} -
- -""" - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--dry-run", action="store_true") - args = ap.parse_args() - - conn = sqlite3.connect(DB_PATH) - items = cb.fetch_items(conn) - conn.close() - items = cb.compute_index(items) - items = cb.decay_index(items, HALF_LIFE_H) - page = build_html(items) - - if args.dry_run: - out = os.path.join(HERE, "_preview.html") - with open(out, "w") as f: - f.write(page) - fresh = [it for it in items if it.get("fresh")] - top = sorted(fresh, key=lambda x: x["clickability_decayed"], reverse=True)[:TOP_N] - print(f"[dry-run] wrote {out} ({len(items)} items, {len(fresh)} fresh today)") - print(f"TOP {TOP_N} FRESH (today only) by decayed clickability:") - for i, it in enumerate(top, 1): - print(f" {i}. [{it['clickability_decayed']:.2f} | age {it['age_hours']:.0f}h] {it['source']:10} {it['title'][:55]}") - return - - # Write to webroot if it exists (deployed); otherwise fall back to a - # user-owned dir so the no_agent cron never errors pre-deploy. - fallback = os.path.join(HERE, "site") - target = WEBROOT if os.path.isdir(WEBROOT) else fallback - os.makedirs(target, exist_ok=True) - with open(os.path.join(target, "index.html"), "w") as f: - f.write(page) - with open(os.path.join(target, "feed.json"), "w") as f: - json.dump([ - {"title": i["title"], "url": i["url"], "source": i["source"], - "clickability_decayed": i["clickability_decayed"], "age_hours": i["age_hours"], - "first_seen": i.get("first_seen")} - for i in sorted(items, key=lambda x: x["clickability_decayed"], reverse=True) - ], f, indent=2) - where = "WEBROOT" if target == WEBROOT else "fallback(~oracle/site)" - print(f"[render] wrote {target}/index.html ({len(items)} items) -> {where}") - -if __name__ == "__main__": - main() +from oracle.cli import main as cli_main +sys.argv = ["oracle"] + cli_args +cli_main() diff --git a/summarize.py b/summarize.py index af9a6df..ba61769 100644 --- a/summarize.py +++ b/summarize.py @@ -1,552 +1,11 @@ #!/usr/bin/env python3 -""" -AI Research Oracle — Summarization Engine (v1). - -Generates structured summaries for entries where summary IS NULL. -Uses source-specific extraction logic (no LLM required — eliminates hallucination). - -Output schema: {one_liner, key_technical_point, potential_use_case, confidence} - -Architecture note: This v1 uses deterministic extraction rules to avoid -hallucination. When a local LLM becomes available (Ollama GPU, Hermes API), -swap in LLM mode via --llm flag. The DB schema is identical. - -Usage: - python3 summarize.py # summarize all pending - python3 summarize.py --source github # specific source - python3 summarize.py --limit 10 # max entries - python3 summarize.py --verify # spot-check 2-3 summaries -""" - -import argparse -import json -import os -import re -import sqlite3 -import sys -import time -from datetime import datetime, timezone - +"""Thin wrapper — delegates to oracle.cli summarize subcommand.""" +import sys, os sys.path.insert(0, os.path.dirname(__file__)) +args = sys.argv[1:] +cli_args = ["summarize"] + args -def extract_github_summary(title: str, content: str) -> dict: - """Extract summary from GitHub README content. - - Strategy: Clean HTML, find the first substantive paragraph that - describes the project (usually below the badges), extract the - "what it does" sentence. - """ - # Aggressive HTML cleaning - text = re.sub(r']*>', '\n', content) - text = re.sub(r'

', '\n', content) - text = re.sub(r']*>', '\n## ', text) - text = re.sub(r'', '\n', text) - text = re.sub(r'<[^>]+>', '', text) - text = re.sub(r'&', '&', text) - text = re.sub(r'—', '—', text) - text = re.sub(r''', "'", text) - text = re.sub(r'·', '·', text) - # Remove code blocks (``` ... ```) — often ASCII art - text = re.sub(r'```[\s\S]*?```', '', text) - text = re.sub(r'\n\s*\n+', '\n\n', text) - text = text.strip() - - # Confidence starts from source content quality - source_confidence = "low" - if len(text) > 2000: - source_confidence = "high" - elif len(text) > 500: - source_confidence = "medium" - - # Find the one-liner: look for project description paragraph - one_liner = _find_project_description(text, title) - if not one_liner: - one_liner = title[:200] - - # Key technical point - key_tech = _extract_technical_point(text, source_confidence) - - # Use case - use_case = _extract_use_case(text, title) - - # Quality-gate confidence on extraction signals, not raw length - confidence = _assess_extraction_quality(one_liner, key_tech, use_case, source_confidence) - - # Tag security tooling if detected - if _is_security_tooling(title, one_liner, key_tech): - use_case = use_case + " [security:dual-use]" - - return { - "one_liner": one_liner[:200], - "key_technical_point": key_tech[:200], - "potential_use_case": use_case[:200], - "confidence": confidence, - } - - -def extract_arxiv_summary(title: str, content: str) -> dict: - """Extract summary from arXiv abstract. - - Strategy: arXiv abstracts have a predictable structure: - 1. Background/motivation - 2. "In this paper we propose..." - 3. Results - 4. Implications - - We extract the contribution statement and key finding. - """ - text = re.sub(r'<[^>]+>', ' ', content) - text = re.sub(r'\s+', ' ', text).strip() - - # Confidence based on abstract clarity - confidence = "high" if len(text) > 300 else "medium" - - # One-liner: find the contribution statement - one_liner = _find_contribution(text) - if not one_liner: - # Fallback: use title as base - one_liner = f"This paper presents {title.lower()}" - - # Key technical point: look for method description - key_tech = _extract_method(text) - - # Use case: look for application statements - use_case = _extract_application(text) - - return { - "one_liner": one_liner[:200], - "key_technical_point": key_tech[:200], - "potential_use_case": use_case[:200], - "confidence": confidence, - } - - -def extract_reddit_summary(title: str, content: str) -> dict: - """Extract summary from Reddit post. - - Strategy: Reddit posts vary wildly in quality. Extract the core - question or claim, note if it's discussion vs announcement. - """ - text = re.sub(r'<[^>]+>', ' ', content) - text = re.sub(r'\s+', ' ', text).strip() - - # Confidence based on content length - if len(text) > 500: - confidence = "high" - elif len(text) > 100: - confidence = "medium" - else: - confidence = "low" - - # One-liner from title (Reddit titles are usually the summary) - one_liner = title[:200] if title else text[:150] - - # Key technical point from content - key_tech = text[:200] if text else "No additional content in post" - - # Use case: community relevance - use_case = "AI community discussion" - - return { - "one_liner": one_liner, - "key_technical_point": key_tech, - "potential_use_case": use_case, - "confidence": confidence, - } - - -# --- Extraction helpers --- - -def _assess_extraction_quality(one_liner: str, key_tech: str, use_case: str, source_confidence: str) -> str: - """Assess extraction quality based on output signals, not source length. - - A short-but-complete Reddit title should score higher confidence - than a long README that yielded a fragment. - """ - score = 0 - penalties = 0 - - # One-liner quality - ol = one_liner.strip() - ol_len = len(ol) - - # Length window: 40-200 chars is a reasonable sentence - if 40 <= ol_len <= 200: - score += 2 - elif 20 <= ol_len < 40: - score += 1 - elif ol_len > 200: - penalties += 1 # too long, likely grabbed too much - - # Ends with terminal punctuation - if ol.endswith(('.', '!', '?', '…')): - score += 1 - else: - penalties += 1 - - # Contains subject-verb pattern (basic heuristic) - if re.search(r'\b(?:is|are|provides|enables|implements|makes|allows|builds|creates|runs|uses)\b', ol, re.I): - score += 1 - # Or starts with a proper noun/capitalized phrase - elif re.match(r'^[A-Z]\w+', ol) and ol_len > 30: - score += 0.5 - - # No unmatched brackets (artifact from markdown/HTML) - open_brackets = ol.count('[') + ol.count('(') - close_brackets = ol.count(']') + ol.count(')') - if abs(open_brackets - close_brackets) > 0: - penalties += 1 - if open_brackets > 2: - penalties += 1 # likely grabbed markdown link syntax - - # Key technical point quality - kt = key_tech.strip() - if kt and len(kt) > 20 and not kt.startswith('See '): - score += 1 - else: - penalties += 0.5 - - # Use case quality - uc = use_case.strip() - if uc and len(uc) > 10 and not uc.startswith('Relevant for'): - score += 1 - else: - penalties += 0.5 - - # Final confidence based on score - penalties - net = score - penalties - if net >= 3: - return source_confidence # extraction is good, trust source quality - elif net >= 1: - return "medium" - else: - return "low" - - -def _is_security_tooling(title: str, one_liner: str, key_tech: str) -> bool: - """Detect if a project is security/offensive tooling.""" - combined = f"{title} {one_liner} {key_tech}".lower() - security_signals = [ - "offensive", "pentest", "red team", "exploit", "kill chain", - "attack surface", "vulnerability scan", "zero-day", - "reverse engineer", "c2", "command and control", - ] - return any(sig in combined for sig in security_signals) - - -def _find_project_description(text: str, title: str) -> str | None: - """Find the project description paragraph in a README.""" - paras = text.split('\n\n') - proj_name = title.split(':')[0].split('/')[0].strip().lower() - - for para in paras: - para = para.strip() - if not para or para.startswith('##') or len(para) < 20: - continue - # Skip badges, stats lines, separator lines - if 'img' in para.lower() or 'badge' in para.lower() or 'shields' in para.lower(): - continue - # Skip lines that start with stats (~54%, etc.) - if re.match(r'^[~$#€£¥*»\d]', para): - continue - # Skip ASCII art (high ratio of special chars) - special_chars = sum(1 for c in para if not c.isalnum() and not c.isspace() and c not in ',.!?;:\'"-()[]') - if special_chars / max(len(para), 1) > 0.4: - continue - if len(para) < 40: - continue - # Good paragraph — extract first sentence - sentence = re.split(r'[.!?]', para)[0].strip() - if len(sentence) > 30: - return sentence + '.' - - # Fallback: look for "is a" pattern anywhere - patterns = [ - rf'{re.escape(proj_name[:20])}\s+(?:is|enables|provides|implements)\s+[^.]+\.?', - r'(?:This\s+)?(?:project|library|framework|tool|package)\s+(?:is|enables|provides)\s+[^.]+\.?', - ] - for pattern in patterns: - match = re.search(pattern, text, re.I) - if match: - return match.group(0)[:200] - - return None - - -def _find_what_sentence(text: str, title: str) -> str | None: - """Find the 'X is a...' sentence that describes what the project does.""" - patterns = [ - rf'{re.escape(title[:30])}\s+(?:is|enables|provides|implements)\s+[^.]+\.?', - r'(?:This\s+)?(?:project|library|framework|tool|package)\s+(?:is|enables|provides|implements)\s+[^.]+\.?', - r'(?:makes|allows)\s+[^\s]+\s+(?:to|can)\s+[^.]+\.?', - r'(?:\w+\s+(?:is|provides|enables|implements|delivers))\s+[a-z].{10,100}\.', - ] - - for pattern in patterns: - match = re.search(pattern, text, re.I) - if match: - return match.group(0)[:200] - - # Fallback: first meaningful paragraph - for para in text.split('\n\n'): - para = para.strip() - if len(para) > 30 and not para.startswith('#'): - return para[:200] - return None - - -def _find_contribution(text: str) -> str | None: - """Find the 'we propose/introduce/present' statement in an abstract.""" - patterns = [ - r'(?:we|this\s+paper)\s+(?:propose|introduce|present|propose and evaluate)\s+[^.]{10,150}\.', - r'(?:we\s+(?:show|demonstrate|find|discover|observe))\s+[^.]{10,150}\.', - r'(?:we\s+(?:introduce|present|propose))\s+(?:a|an|our)\s+\w+\s+[^.]{5,150}\.', - ] - - for pattern in patterns: - match = re.search(pattern, text, re.I) - if match: - return match.group(0)[:200] - - # Fallback: first sentence - first = re.split(r'[.!?]', text)[0].strip() - return first if first else None - - -def _extract_technical_point(text: str, confidence: str) -> str: - """Extract the main technical approach or innovation.""" - patterns = [ - r'architecture(?:\s+designed)?\s+(?:for|to|that)\s+[^.]+\.?', - r'(?:using|via|based\s+on|through)\s+[a-z][^.]{10,100}\.', - r'(?:novel|new|unique|innovative)\s+\w+\s+[^.]{5,80}\.', - ] - - for pattern in patterns: - match = re.search(pattern, text, re.I) - if match: - return match.group(0)[:200] - - # Fallback: confidence-based - if confidence == "low": - return "Technical details not available in extracted content" - return "See README for technical details" - - -def _extract_method(text: str) -> str: - """Extract the method/approach from an arXiv abstract.""" - patterns = [ - r'(?:method|approach|framework|technique|model|system)\s+(?:based|using|via|through|with)\s+[a-z][^.]{10,120}\.', - r'(?:combining|leveraging|exploiting)\s+[a-z][^.]{10,120}\.', - r'(?:learn|train|optimize|generate)\s+[a-z][^.]{10,120}\.', - ] - - for pattern in patterns: - match = re.search(pattern, text, re.I) - if match: - return match.group(0)[:200] - - # Fallback: core contribution - for pattern in [ - r'(?:propose|introduce)\s+(?:a|an)\s+[^.]{10,100}\.', - ]: - match = re.search(pattern, text, re.I) - if match: - return match.group(0)[:200] - - return "See full paper for methodology" - - -def _extract_use_case(text: str, title: str) -> str: - """Extract potential use case from README content.""" - patterns = [ - r'(?:for|to)\s+(?:developers|engineers|researchers|teams)\s+who?\s+[^.]{5,80}\.', - r'(?:enables|allows|helps)\s+[^\s]+\s+to\s+[^.]{10,80}\.', - r'(?:use\s+case|application|target\s+user)\s*:\s*[^.]{10,80}\.', - ] - - for pattern in patterns: - match = re.search(pattern, text, re.I) - if match: - return match.group(0)[:200] - - return f"Relevant for {title.lower()[:50]} developers and users" - - -def _extract_application(text: str) -> str: - """Extract application/use case from arXiv abstract.""" - patterns = [ - r'(?:application|use\s+case|can\s+be\s+used|could\s+be\s+applied)\s+(?:for|in|to)\s+[a-z][^.]{10,80}\.', - r'(?:improve|enhance|advance)\s+[a-z][^.]{10,80}\.', - ] - - for pattern in patterns: - match = re.search(pattern, text, re.I) - if match: - return match.group(0)[:200] - - # Generic fallback based on title keywords - title_lower = text[:200].lower() - if any(k in title_lower for k in ["agent", "agentic"]): - return "Building AI agent systems" - elif any(k in title_lower for k in ["verification", "verify"]): - return "LLM output verification and reliability" - elif any(k in title_lower for k in ["embodied", "robot"]): - return "Embodied AI and robotics applications" - elif any(k in title_lower for k in ["distill"]): - return "Model distillation and knowledge transfer" - return "See paper for specific applications" - - -def summarize_entry(entry: dict, conn: sqlite3.Connection) -> bool: - """Summarize a single entry using rule-based extraction.""" - source = entry["source"] - title = entry["title"] - content = entry.get("extracted_text", "") - eid = entry["id"] - - if not content or len(content) < 50: - return False - - # Source-specific extraction - if source == "github": - summary = extract_github_summary(title, content) - elif source == "arxiv": - summary = extract_arxiv_summary(title, content) - elif source == "reddit": - summary = extract_reddit_summary(title, content) - else: - summary = extract_reddit_summary(title, content) # fallback - - # Store - cur = conn.cursor() - cur.execute("UPDATE entries SET summary = ? WHERE id = ?", - (json.dumps(summary), eid)) - conn.commit() - return True - - -def verify_summaries(conn: sqlite3.Connection, source: str, sample_size: int = 3): - """Spot-check summaries against source text. - - Look for hallucinated specifics: numbers, claims, features not - present in the original extracted_text. - - NOTE: Rule-based extraction v1 is inherently lower-risk for - hallucination since it extracts actual text, not generates new claims. - But we still verify the extraction logic is working correctly. - """ - cur = conn.cursor() - cur.execute(""" - SELECT id, title, extracted_text, summary - FROM entries WHERE source = ? AND summary IS NOT NULL - ORDER BY RANDOM() - LIMIT ? - """, (source, sample_size)) - - rows = cur.fetchall() - if not rows: - print(f" No summaries to verify for {source}") - return - - for eid, title, source_text, summary_json in rows: - summary = json.loads(summary_json) - one_liner = summary.get("one_liner", "") - confidence = summary.get("confidence", "?") - - issues = [] - - # Check: does the one-liner contain text actually present in source? - # (For rule-based extraction, this should always be true) - words = one_liner.split()[:5] - found = sum(1 for w in words if w.lower() in source_text.lower()) - if found < 3: - issues.append(f"Low overlap: {found}/5 words from source") - - # Check: confidence matches content length - if confidence == "high" and len(source_text) < 500: - issues.append("High confidence on short source") - elif confidence == "low" and len(source_text) > 2000: - issues.append("Low confidence on long source") - - if issues: - print(f" ⚠ [{eid}] {title[:50]}... issues: {'; '.join(issues)}") - print(f" Summary: {one_liner[:80]}...") - else: - print(f" ✓ [{eid}] {title[:50]}... confidence={confidence}") - - time.sleep(0.3) - - -def main(): - parser = argparse.ArgumentParser(description="AI Research Oracle — Summarization") - parser.add_argument("--source", default=None, help="Filter by source (github/arxiv/reddit)") - parser.add_argument("--limit", type=int, default=0, help="Max entries (0=all)") - parser.add_argument("--verify", action="store_true", help="Spot-check summaries") - args = parser.parse_args() - - db_path = os.path.join(os.path.dirname(__file__), "oracle.db") - conn = sqlite3.connect(db_path) - cur = conn.cursor() - - # Find pending entries - where = "summary IS NULL" - params = [] - if args.source: - where += " AND source = ?" - params.append(args.source) - - cur.execute(f"SELECT COUNT(*) FROM entries WHERE {where}", params) - total_pending = cur.fetchone()[0] - print(f"=== Summarization Engine (Rule-based v1) ===") - print(f" Pending entries: {total_pending}") - - if total_pending == 0: - print(" Nothing to summarize.") - conn.close() - return - - # Fetch entries - limit_clause = " LIMIT ?" if args.limit > 0 else "" - limit_params = params + [args.limit] if args.limit > 0 else params - - cur.execute(f""" - SELECT id, source, title, extracted_text - FROM entries WHERE {where} - ORDER BY signal_score DESC - {limit_clause} - """, limit_params) - - entries = [{"id": r[0], "source": r[1], "title": r[2], "extracted_text": r[3]} for r in cur.fetchall()] - print(f" Processing: {len(entries)} entries") - print() - - success = 0 - failed = 0 - for entry in entries: - try: - if summarize_entry(entry, conn): - success += 1 - print(f" ✓ [{entry['id']}] {entry['title'][:60]}... ({entry['source']})") - else: - failed += 1 - print(f" ⚠ [{entry['id']}] Skipped: {entry['title'][:40]}... (too short)") - except Exception as e: - print(f" ✗ [{entry['id']}] Error: {e}") - failed += 1 - - if args.verify: - print(f"\n [Verification]") - sources = [args.source] if args.source else ["github", "arxiv", "reddit"] - for src in sources: - print(f" Checking {src}...") - verify_summaries(conn, src) - print() - - print(f" Results: {success} summarized, {failed} failed") - conn.close() - print(f"\n Done.") - - -if __name__ == "__main__": - main() +from oracle.cli import main as cli_main +sys.argv = ["oracle"] + cli_args +cli_main() diff --git a/theme_scan.py b/theme_scan.py index 6249387..ae6f13a 100644 --- a/theme_scan.py +++ b/theme_scan.py @@ -1,128 +1,11 @@ #!/usr/bin/env python3 -"""Phase 6 trend-tracking: theme-based arrival counter. +"""Thin wrapper — delegates to oracle.cli themes subcommand.""" +import sys, os +sys.path.insert(0, os.path.dirname(__file__)) -The question this answers: is the 2026-07-08 practitioner cluster a real TREND -or a one-day COINCIDENCE? +args = sys.argv[1:] +cli_args = ["themes"] + args -Design (per review): - - Tag by THEME, not by entry ID. - - Count NEW theme-tagged ARRIVALS per cron cycle (only classify rows that - have no theme_tags yet -> fresh entries each run). - - If new theme arrivals stay ~0 all week => coincidence. - - If 2-4+ new relevant items/cycle across sources => trend. - Only then does the ponytail-generalization idea graduate from a one-day - read to something worth further investment. - -Themes: - tool-call : tool-call gating on confidence / reliability of tool use - context : context / token compression before window ceiling - compute : routing to smallest sufficient model / inference cost - trust : trust boundaries on what a model may learn / vetted adapters - -Run: python3 theme_scan.py # classify + report new arrivals - python3 theme_scan.py --history # also print per-cycle history -""" - -import argparse -import os -import re -import sqlite3 -import sys -from collections import Counter - -DB = os.path.join(os.path.dirname(__file__), "oracle.db") - -# Theme -> regex over title+summary+extracted text (case-insensitive). -# Deliberately keyword-anchored, not semantic — cheap, auditable, reproducible. -THEME_PATTERNS = { - "tool-call": re.compile( - r"\b(tool[- ]?call|tool[- ]?use|competence gate|confidence gate|" - r"gate[d]? tool|action gate|tool reliability|function call gate)\b", - re.I), - "context": re.compile( - r"\b(context (compress|window|ceiling|summar)|semantic compress|" - r"token (compress|budget)|compress (context|session)|context (limit|overflow))\b", - re.I), - "compute": re.compile( - r"\b(small(er|est)? model|route to|inference cost|cpu (tts|infer)|" - r"cheap(er)? model|model routing|tiny model|on[- ]device (llm|model))\b", - re.I), - "trust": re.compile( - r"\b(trust(ed)? (adapter|lora)|vetted adapter|learn (only|what).*adapter|" - r"trust boundary|what a model (can|may) learn|auditable (adapter|skill))\b", - re.I), -} - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--history", action="store_true", - help="Print per-cycle new-arrival history after the run") - args = ap.parse_args() - - if not os.path.exists(DB): - print("No oracle.db — nothing to scan") - return - - conn = sqlite3.connect(DB) - conn.row_factory = sqlite3.Row - cur = conn.cursor() - - # Ensure theme_tags table exists (defensive; schema.sql creates it). - cur.execute("""CREATE TABLE IF NOT EXISTS theme_tags ( - entry_id INTEGER NOT NULL, - theme TEXT NOT NULL, - first_seen_cycle TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ','now')), - PRIMARY KEY (entry_id, theme))""") - - # Only rows not yet classified -> fresh arrivals this cycle. - cur.execute(""" - SELECT e.id, e.source, e.title, - COALESCE(e.summary,'') AS summary, - COALESCE(e.extracted_text,'') AS extracted - FROM entries e - WHERE e.id NOT IN (SELECT entry_id FROM theme_tags) - """) - fresh = cur.fetchall() - - new_counts = Counter() - for row in fresh: - blob = f"{row['title']} {row['summary']} {row['extracted']}" - for theme, pat in THEME_PATTERNS.items(): - if pat.search(blob): - cur.execute( - "INSERT OR IGNORE INTO theme_tags (entry_id, theme) VALUES (?, ?)", - (row["id"], theme)) - new_counts[theme] += 1 - - conn.commit() - - print("=== Theme trend scan ===") - print(f" Fresh (unclassified) entries this cycle: {len(fresh)}") - if new_counts: - print(" NEW theme arrivals this cycle:") - for theme in ("tool-call", "context", "compute", "trust"): - if new_counts.get(theme): - print(f" {theme}: +{new_counts[theme]}") - else: - print(" NEW theme arrivals this cycle: 0") - - # Cumulative context for the trend question. - cur.execute("SELECT theme, COUNT(*) AS c FROM theme_tags GROUP BY theme") - cum = {r["theme"]: r["c"] for r in cur.fetchall()} - print(f" Cumulative theme_tags totals: {cum}") - - if args.history: - print("\n Per-cycle new arrivals (by first_seen_cycle):") - cur.execute(""" - SELECT substr(first_seen_cycle,1,10) AS day, theme, COUNT(*) AS c - FROM theme_tags GROUP BY day, theme ORDER BY day, theme - """) - for r in cur.fetchall(): - print(f" {r['day']} {r['theme']}: {r['c']}") - - conn.close() - - -if __name__ == "__main__": - main() +from oracle.cli import main as cli_main +sys.argv = ["oracle"] + cli_args +cli_main() diff --git a/variants/brief.yaml b/variants/brief.yaml new file mode 100644 index 0000000..884c122 --- /dev/null +++ b/variants/brief.yaml @@ -0,0 +1,25 @@ +# Brief edition — executive summary, top 8 PUBLISH only +name: "Athena Brief" +description: "Top 8 AI stories that matter right now" +output: "brief/index.html" + +filters: + verdicts: ["PUBLISH"] + sources: [] + min_score: 0 + max_age_h: 48 + max_items: 8 + +ranking: + by: "signal_score" + half_life_h: 6 + +display: + top_n: 8 + show_summary: true + show_score: false + show_tier: false + show_verdict: false + theme: "light" + accent: "#f59e0b" + logo: "⚡" diff --git a/variants/default.yaml b/variants/default.yaml new file mode 100644 index 0000000..4e78d4b --- /dev/null +++ b/variants/default.yaml @@ -0,0 +1,31 @@ +# Default Athena edition — full feed, clickability-ranked +name: "Athena AI News" +description: "Full AI research feed ranked by clickability index" +output: "index.html" + +filters: + # Include all verdicts + verdicts: ["PUBLISH", "WATCH", "ARCHIVE", "DROP"] + # No source restriction + sources: [] + # Minimum signal score + min_score: 0 + # Maximum age in hours (0 = no limit) + max_age_h: 0 + # Maximum items (0 = no limit) + max_items: 0 + +ranking: + # Sort by: clickability | signal_score | verdict_priority | freshness + by: "clickability" + half_life_h: 18 + +display: + top_n: 8 + show_summary: true + show_score: true + show_tier: false + show_verdict: false + theme: "dark" + accent: "#5b8cff" + logo: "🏛️" diff --git a/variants/devops.yaml b/variants/devops.yaml new file mode 100644 index 0000000..a4f7a4f --- /dev/null +++ b/variants/devops.yaml @@ -0,0 +1,25 @@ +# DevOps edition — shipping tools, frameworks, infrastructure +name: "Athena DevOps" +description: "Shipping AI tools, frameworks, and infrastructure releases" +output: "devops/index.html" + +filters: + verdicts: ["PUBLISH", "WATCH"] + sources: ["github", "hackernews", "rss"] + min_score: 2.5 + max_age_h: 168 + max_items: 40 + +ranking: + by: "clickability" + half_life_h: 12 + +display: + top_n: 10 + show_summary: true + show_score: true + show_tier: true + show_verdict: true + theme: "dark" + accent: "#34d399" + logo: "🔧" diff --git a/variants/research.yaml b/variants/research.yaml new file mode 100644 index 0000000..0e2d15d --- /dev/null +++ b/variants/research.yaml @@ -0,0 +1,25 @@ +# Research edition — arXiv + HF papers, high-signal only +name: "Athena Research" +description: "Peer-reviewed AI research papers and model releases" +output: "research/index.html" + +filters: + verdicts: ["PUBLISH", "WATCH", "ARCHIVE"] + sources: ["arxiv", "huggingface"] + min_score: 3.0 + max_age_h: 720 + max_items: 50 + +ranking: + by: "signal_score" + half_life_h: 72 + +display: + top_n: 12 + show_summary: true + show_score: true + show_tier: true + show_verdict: true + theme: "dark" + accent: "#a78bfa" + logo: "🔬" diff --git a/write_summaries.py b/write_summaries.py index 8f3fb95..ba61769 100644 --- a/write_summaries.py +++ b/write_summaries.py @@ -1,81 +1,11 @@ #!/usr/bin/env python3 -""" -Write batch summaries back to oracle.db. +"""Thin wrapper — delegates to oracle.cli summarize subcommand.""" +import sys, os +sys.path.insert(0, os.path.dirname(__file__)) -Reads /tmp/athena_summarize_batch.json (array of entry dicts with 'summary' key added by sub-agent), -writes each summary JSON to entries.summary column. +args = sys.argv[1:] +cli_args = ["summarize"] + args -Usage: - python3 write_summaries.py /tmp/athena_summarize_batch_result.json -""" -import json -import sqlite3 -import sys - -def main(): - if len(sys.argv) < 2: - print("Usage: python3 write_summaries.py ", file=sys.stderr) - sys.exit(1) - - result_path = sys.argv[1] - try: - with open(result_path) as f: - results = json.load(f) - except (FileNotFoundError, json.JSONDecodeError) as e: - print(f"Error reading {result_path}: {e}", file=sys.stderr) - sys.exit(1) - - db_path = '/home/vpsadmin/oracle/oracle.db' - conn = sqlite3.connect(db_path) - cur = conn.cursor() - - written = 0 - skipped = 0 - errors = 0 - - for item in results: - eid = item.get('id') - summary = item.get('summary') - if not eid or not summary: - skipped += 1 - continue - - # Validate summary has expected keys - if not all(k in summary for k in ('one_liner', 'key_technical_point', 'potential_use_case', 'confidence')): - print(f" ⚠ ID {eid}: missing required keys, skipping", file=sys.stderr) - skipped += 1 - continue - - # Quality gate: reject low-confidence or generic summaries - ol = summary.get('one_liner', '') - if len(ol) < 20: - print(f" ⚠ ID {eid}: one_liner too short ({len(ol)} chars), skipping", file=sys.stderr) - skipped += 1 - continue - if any(generic in ol.lower() for generic in ('this article discusses', 'this paper presents', 'see full')): - print(f" ⚠ ID {eid}: generic one_liner, skipping", file=sys.stderr) - skipped += 1 - continue - - try: - cur.execute("UPDATE entries SET summary = ? WHERE id = ?", - (json.dumps(summary), eid)) - written += 1 - print(f" ✓ ID {eid}: {ol[:70]}...") - except Exception as e: - print(f" ✗ ID {eid}: {e}", file=sys.stderr) - errors += 1 - - conn.commit() - - # Verify - cur.execute("SELECT COUNT(*) FROM entries WHERE summary IS NOT NULL") - total = cur.fetchone()[0] - conn.close() - - print(f"\nResults: {written} written, {skipped} skipped, {errors} errors") - print(f"Total entries with summary: {total}") - return 0 if errors == 0 else 1 - -if __name__ == "__main__": - sys.exit(main()) +from oracle.cli import main as cli_main +sys.argv = ["oracle"] + cli_args +cli_main()