Sprint 0+1: Package restructure, source tiers, verdicts, multi-variant editions

- New oracle/ package (11 modules) with unified CLI (python -m oracle)
- Source tiers: Tier 1 (arxiv/github/hf), Tier 2 (rss/hn), Tier 3 (reddit)
- Composite verdicts: PUBLISH/WATCH/ARCHIVE/DROP based on signal score + age
- Content-hash dedup: SHA-256[:16] normalized, atomic at insert time
- Multi-variant editions: 4 YAML configs (default/research/devops/brief)
- Variant engine: filter → rank → render (HTML + JSON, themed)
- Per-adapter timeout (10s) + threading fallback
- Consolidated 12 root scripts → thin wrappers + oracle/ package
- Archived stale scripts (_engagement, _live_compare, reddit_proof)
- Updated .gitignore, README.md, schema.sql
This commit is contained in:
Epictetus
2026-07-22 13:32:15 +00:00
parent 9f72ff4d6a
commit 07c5f9a5c2
38 changed files with 3195 additions and 3234 deletions
+8 -91
View File
@@ -1,94 +1,11 @@
#!/usr/bin/env python3
"""Oracle soft-cap archival.
"""Thin wrapper — delegates to oracle.cli archive subcommand."""
import sys, os
sys.path.insert(0, os.path.dirname(__file__))
Bounds live `entries` growth by moving old / excess rows into
`entries_archive` (preserving data — soft cap, not hard delete).
Two triggers:
--days N : move entries not updated in N days (default 30)
--cap N : if live entries exceed N, archive oldest beyond the cap (default 5000)
Default is a real run (data moves). Use --dry-run to report only.
"""
args = sys.argv[1:]
cli_args = ["archive"] + args
import argparse
import os
import sqlite3
import time
DB = os.path.join(os.path.dirname(__file__), "oracle.db")
ARCHIVE_SCHEMA = """
CREATE TABLE IF NOT EXISTS entries_archive (
id INTEGER PRIMARY KEY,
source TEXT, source_id TEXT, url TEXT, title TEXT,
extracted_text TEXT, summary TEXT, category_tags TEXT,
signal_score REAL, raw_metadata TEXT,
first_seen TEXT, last_updated TEXT,
archived_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now'))
);
"""
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--days", type=int, default=30, help="Archive entries not updated in N days")
ap.add_argument("--cap", type=int, default=5000, help="Soft cap on live entries; archive oldest beyond this")
ap.add_argument("--dry-run", action="store_true", help="Report only, make no changes")
args = ap.parse_args()
if not os.path.exists(DB):
print("No oracle.db — nothing to archive")
return
conn = sqlite3.connect(DB)
conn.execute(ARCHIVE_SCHEMA)
cutoff = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - args.days * 86400))
cur = conn.cursor()
cur.execute("SELECT id, last_updated FROM entries")
rows = cur.fetchall()
n_total = len(rows)
old_ids = [r[0] for r in rows if (r[1] or "") < cutoff]
beyond = max(0, n_total - args.cap)
if beyond > 0:
ordered = sorted(rows, key=lambda r: r[1] or "")[:beyond]
cap_ids = [r[0] for r in ordered]
else:
cap_ids = []
move_ids = sorted(set(old_ids) | set(cap_ids))
if not move_ids:
print(f"Archive check: {n_total} live entries, none older than {args.days}d "
f"or beyond cap {args.cap}. Nothing to archive.")
conn.close()
return
print(f"Archive check: {n_total} live entries -> would archive {len(move_ids)} "
f"(old={len(old_ids)}, cap={len(cap_ids)}).")
if args.dry_run:
print("DRY RUN — no changes made.")
conn.close()
return
q = ",".join("?" * len(move_ids))
conn.execute(
f"""INSERT OR REPLACE INTO entries_archive
(id, source, source_id, url, title, extracted_text, summary,
category_tags, signal_score, raw_metadata, first_seen, last_updated)
SELECT id, source, source_id, url, title, extracted_text, summary,
category_tags, signal_score, raw_metadata, first_seen, last_updated
FROM entries WHERE id IN ({q})""",
move_ids,
)
conn.execute(f"DELETE FROM entries WHERE id IN ({q})", move_ids)
conn.commit()
conn.close()
print(f"Archived {len(move_ids)} entries (live now {n_total - len(move_ids)}). "
f"Preserved in entries_archive.")
if __name__ == "__main__":
main()
from oracle.cli import main as cli_main
sys.argv = ["oracle"] + cli_args
cli_main()