Epictetus
07c5f9a5c2
- New oracle/ package (11 modules) with unified CLI (python -m oracle) - Source tiers: Tier 1 (arxiv/github/hf), Tier 2 (rss/hn), Tier 3 (reddit) - Composite verdicts: PUBLISH/WATCH/ARCHIVE/DROP based on signal score + age - Content-hash dedup: SHA-256[:16] normalized, atomic at insert time - Multi-variant editions: 4 YAML configs (default/research/devops/brief) - Variant engine: filter → rank → render (HTML + JSON, themed) - Per-adapter timeout (10s) + threading fallback - Consolidated 12 root scripts → thin wrappers + oracle/ package - Archived stale scripts (_engagement, _live_compare, reddit_proof) - Updated .gitignore, README.md, schema.sql
84 lines
2.7 KiB
Python
84 lines
2.7 KiB
Python
"""Oracle soft-cap archival.
|
|
|
|
Bounds live `entries` growth by moving old / excess rows into
|
|
`entries_archive` (preserving data — soft cap, not hard delete).
|
|
"""
|
|
import os
|
|
import sqlite3
|
|
import time
|
|
|
|
from oracle.config import DB_PATH
|
|
|
|
ARCHIVE_SCHEMA = """
|
|
CREATE TABLE IF NOT EXISTS entries_archive (
|
|
id INTEGER PRIMARY KEY,
|
|
source TEXT, source_id TEXT, url TEXT, title TEXT,
|
|
extracted_text TEXT, summary TEXT, category_tags TEXT,
|
|
signal_score REAL, raw_metadata TEXT,
|
|
first_seen TEXT, last_updated TEXT,
|
|
archived_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now'))
|
|
);
|
|
"""
|
|
|
|
|
|
def archive(days=30, cap=5000, dry_run=False, db_path=None):
|
|
"""Archive entries older than N days or beyond the cap.
|
|
|
|
Returns (total_live, archived_count).
|
|
"""
|
|
path = db_path or str(DB_PATH)
|
|
if not os.path.exists(path):
|
|
print("No oracle.db — nothing to archive")
|
|
return 0, 0
|
|
|
|
conn = sqlite3.connect(path)
|
|
conn.execute(ARCHIVE_SCHEMA)
|
|
|
|
cutoff = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - days * 86400))
|
|
|
|
cur = conn.cursor()
|
|
cur.execute("SELECT id, last_updated FROM entries")
|
|
rows = cur.fetchall()
|
|
n_total = len(rows)
|
|
|
|
old_ids = [r[0] for r in rows if (r[1] or "") < cutoff]
|
|
beyond = max(0, n_total - cap)
|
|
if beyond > 0:
|
|
ordered = sorted(rows, key=lambda r: r[1] or "")[:beyond]
|
|
cap_ids = [r[0] for r in ordered]
|
|
else:
|
|
cap_ids = []
|
|
|
|
move_ids = sorted(set(old_ids) | set(cap_ids))
|
|
|
|
if not move_ids:
|
|
print(f"Archive check: {n_total} live entries, none older than {days}d "
|
|
f"or beyond cap {cap}. Nothing to archive.")
|
|
conn.close()
|
|
return n_total, 0
|
|
|
|
print(f"Archive check: {n_total} live entries -> would archive {len(move_ids)} "
|
|
f"(old={len(old_ids)}, cap={len(cap_ids)}).")
|
|
|
|
if dry_run:
|
|
print("DRY RUN — no changes made.")
|
|
conn.close()
|
|
return n_total, 0
|
|
|
|
q = ",".join("?" * len(move_ids))
|
|
conn.execute(
|
|
f"""INSERT OR REPLACE INTO entries_archive
|
|
(id, source, source_id, url, title, extracted_text, summary,
|
|
category_tags, signal_score, raw_metadata, first_seen, last_updated)
|
|
SELECT id, source, source_id, url, title, extracted_text, summary,
|
|
category_tags, signal_score, raw_metadata, first_seen, last_updated
|
|
FROM entries WHERE id IN ({q})""",
|
|
move_ids,
|
|
)
|
|
conn.execute(f"DELETE FROM entries WHERE id IN ({q})", move_ids)
|
|
conn.commit()
|
|
conn.close()
|
|
print(f"Archived {len(move_ids)} entries (live now {n_total - len(move_ids)}). "
|
|
f"Preserved in entries_archive.")
|
|
return n_total, len(move_ids)
|