Phase 5: cron entry script, soft-cap archive, run_log zero-fetch degradation
- oracle-pipeline.sh: single cron entry point (pipeline -> summarize -> archive) - archive.py: soft-cap archival to entries_archive (preserve, not delete) - pipeline.py: record zero-fetch (rate-limited) runs as degraded in run_log.notes Verified: full script runs exit 0, run_log captures per-source status.
This commit is contained in:
+94
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Oracle soft-cap archival.
|
||||
|
||||
Bounds live `entries` growth by moving old / excess rows into
|
||||
`entries_archive` (preserving data — soft cap, not hard delete).
|
||||
Two triggers:
|
||||
--days N : move entries not updated in N days (default 30)
|
||||
--cap N : if live entries exceed N, archive oldest beyond the cap (default 5000)
|
||||
Default is a real run (data moves). Use --dry-run to report only.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
DB = os.path.join(os.path.dirname(__file__), "oracle.db")
|
||||
|
||||
ARCHIVE_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS entries_archive (
|
||||
id INTEGER PRIMARY KEY,
|
||||
source TEXT, source_id TEXT, url TEXT, title TEXT,
|
||||
extracted_text TEXT, summary TEXT, category_tags TEXT,
|
||||
signal_score REAL, raw_metadata TEXT,
|
||||
first_seen TEXT, last_updated TEXT,
|
||||
archived_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now'))
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--days", type=int, default=30, help="Archive entries not updated in N days")
|
||||
ap.add_argument("--cap", type=int, default=5000, help="Soft cap on live entries; archive oldest beyond this")
|
||||
ap.add_argument("--dry-run", action="store_true", help="Report only, make no changes")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not os.path.exists(DB):
|
||||
print("No oracle.db — nothing to archive")
|
||||
return
|
||||
|
||||
conn = sqlite3.connect(DB)
|
||||
conn.execute(ARCHIVE_SCHEMA)
|
||||
|
||||
cutoff = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - args.days * 86400))
|
||||
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT id, last_updated FROM entries")
|
||||
rows = cur.fetchall()
|
||||
n_total = len(rows)
|
||||
|
||||
old_ids = [r[0] for r in rows if (r[1] or "") < cutoff]
|
||||
beyond = max(0, n_total - args.cap)
|
||||
if beyond > 0:
|
||||
ordered = sorted(rows, key=lambda r: r[1] or "")[:beyond]
|
||||
cap_ids = [r[0] for r in ordered]
|
||||
else:
|
||||
cap_ids = []
|
||||
|
||||
move_ids = sorted(set(old_ids) | set(cap_ids))
|
||||
|
||||
if not move_ids:
|
||||
print(f"Archive check: {n_total} live entries, none older than {args.days}d "
|
||||
f"or beyond cap {args.cap}. Nothing to archive.")
|
||||
conn.close()
|
||||
return
|
||||
|
||||
print(f"Archive check: {n_total} live entries -> would archive {len(move_ids)} "
|
||||
f"(old={len(old_ids)}, cap={len(cap_ids)}).")
|
||||
|
||||
if args.dry_run:
|
||||
print("DRY RUN — no changes made.")
|
||||
conn.close()
|
||||
return
|
||||
|
||||
q = ",".join("?" * len(move_ids))
|
||||
conn.execute(
|
||||
f"""INSERT OR REPLACE INTO entries_archive
|
||||
(id, source, source_id, url, title, extracted_text, summary,
|
||||
category_tags, signal_score, raw_metadata, first_seen, last_updated)
|
||||
SELECT id, source, source_id, url, title, extracted_text, summary,
|
||||
category_tags, signal_score, raw_metadata, first_seen, last_updated
|
||||
FROM entries WHERE id IN ({q})""",
|
||||
move_ids,
|
||||
)
|
||||
conn.execute(f"DELETE FROM entries WHERE id IN ({q})", move_ids)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"Archived {len(move_ids)} entries (live now {n_total - len(move_ids)}). "
|
||||
f"Preserved in entries_archive.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user