Epictetus
b51be6ed48
- oracle-pipeline.sh: single cron entry point (pipeline -> summarize -> archive) - archive.py: soft-cap archival to entries_archive (preserve, not delete) - pipeline.py: record zero-fetch (rate-limited) runs as degraded in run_log.notes Verified: full script runs exit 0, run_log captures per-source status.
38 lines
1.3 KiB
Bash
Executable File
38 lines
1.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Oracle daily pipeline — cron entry point (no_agent; stdout is delivered verbatim).
|
|
# Runs: fetch+store (pipeline.py) -> summarize (summarize.py) -> soft-cap archive (archive.py).
|
|
# run_log captures per-source failure + zero-fetch degradation for visibility.
|
|
#
|
|
set -u
|
|
|
|
ORACLE_DIR="/home/vpsadmin/oracle"
|
|
LOG_DIR="$ORACLE_DIR/logs"
|
|
TS="$(date -u +%Y%m%d-%H%M%S)"
|
|
LOG="$LOG_DIR/cron_run_${TS}.log"
|
|
|
|
mkdir -p "$LOG_DIR"
|
|
cd "$ORACLE_DIR" || { echo "FATAL: cannot cd $ORACLE_DIR"; exit 1; }
|
|
|
|
{
|
|
echo "=== Oracle pipeline run: $(date -u) ==="
|
|
python3 pipeline.py --limit 20
|
|
echo
|
|
echo "=== Summarization Engine ==="
|
|
python3 summarize.py
|
|
echo
|
|
echo "=== Soft-cap archive (dry-safe default: 30d / 5000 cap) ==="
|
|
python3 archive.py --days 30 --cap 5000
|
|
echo
|
|
echo "=== run_log tail (failure visibility) ==="
|
|
python3 -c "
|
|
import sqlite3
|
|
c = sqlite3.connect('oracle.db')
|
|
for r in c.execute('SELECT id,run_time,total_fetched,total_stored,sources_failed,notes FROM run_log ORDER BY id DESC LIMIT 1'):
|
|
print(' run', r[0], '|', r[1], '| fetched', r[2], '| stored', r[3], '| failed', r[4], '| notes:', r[5])
|
|
print(' live entries:', c.execute('SELECT COUNT(*) FROM entries').fetchone()[0])
|
|
print(' archived entries:', c.execute('SELECT COUNT(*) FROM entries_archive').fetchone()[0])
|
|
"
|
|
echo "=== Done ==="
|
|
} 2>&1 | tee "$LOG"
|