Phase 5: cron entry script, soft-cap archive, run_log zero-fetch degradation
- oracle-pipeline.sh: single cron entry point (pipeline -> summarize -> archive) - archive.py: soft-cap archival to entries_archive (preserve, not delete) - pipeline.py: record zero-fetch (rate-limited) runs as degraded in run_log.notes Verified: full script runs exit 0, run_log captures per-source status.
This commit is contained in:
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Oracle daily pipeline — cron entry point (no_agent; stdout is delivered verbatim).
|
||||
# Runs: fetch+store (pipeline.py) -> summarize (summarize.py) -> soft-cap archive (archive.py).
|
||||
# run_log captures per-source failure + zero-fetch degradation for visibility.
|
||||
#
|
||||
set -u
|
||||
|
||||
ORACLE_DIR="/home/vpsadmin/oracle"
|
||||
LOG_DIR="$ORACLE_DIR/logs"
|
||||
TS="$(date -u +%Y%m%d-%H%M%S)"
|
||||
LOG="$LOG_DIR/cron_run_${TS}.log"
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
cd "$ORACLE_DIR" || { echo "FATAL: cannot cd $ORACLE_DIR"; exit 1; }
|
||||
|
||||
{
|
||||
echo "=== Oracle pipeline run: $(date -u) ==="
|
||||
python3 pipeline.py --limit 20
|
||||
echo
|
||||
echo "=== Summarization Engine ==="
|
||||
python3 summarize.py
|
||||
echo
|
||||
echo "=== Soft-cap archive (dry-safe default: 30d / 5000 cap) ==="
|
||||
python3 archive.py --days 30 --cap 5000
|
||||
echo
|
||||
echo "=== run_log tail (failure visibility) ==="
|
||||
python3 -c "
|
||||
import sqlite3
|
||||
c = sqlite3.connect('oracle.db')
|
||||
for r in c.execute('SELECT id,run_time,total_fetched,total_stored,sources_failed,notes FROM run_log ORDER BY id DESC LIMIT 1'):
|
||||
print(' run', r[0], '|', r[1], '| fetched', r[2], '| stored', r[3], '| failed', r[4], '| notes:', r[5])
|
||||
print(' live entries:', c.execute('SELECT COUNT(*) FROM entries').fetchone()[0])
|
||||
print(' archived entries:', c.execute('SELECT COUNT(*) FROM entries_archive').fetchone()[0])
|
||||
"
|
||||
echo "=== Done ==="
|
||||
} 2>&1 | tee "$LOG"
|
||||
Reference in New Issue
Block a user