Leonard
48 lines
1.6 KiB
Bash
Executable File
48 lines
1.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Oracle daily pipeline — cron entry point (no_agent; stdout is delivered verbatim).
|
|
# Runs: fetch+store (pipeline.py) -> summarize (summarize.py) -> soft-cap archive (archive.py).
|
|
# run_log captures per-source failure + zero-fetch degradation for visibility.
|
|
#
|
|
set -u
|
|
|
|
ORACLE_DIR="/home/vpsadmin/oracle"
|
|
LOG_DIR="$ORACLE_DIR/logs"
|
|
TS="$(date -u +%Y%m%d-%H%M%S)"
|
|
LOG="$LOG_DIR/cron_run_${TS}.log"
|
|
|
|
mkdir -p "$LOG_DIR"
|
|
cd "$ORACLE_DIR" || { echo "FATAL: cannot cd $ORACLE_DIR"; exit 1; }
|
|
|
|
# Source tokens from .env (GITHUB_TOKEN, HUGGINGFACE_TOKEN)
|
|
if [ -f "$ORACLE_DIR/.env" ]; then
|
|
set -a
|
|
source "$ORACLE_DIR/.env"
|
|
set +a
|
|
fi
|
|
|
|
{
|
|
echo "=== Oracle pipeline run: $(date -u) ==="
|
|
python3 pipeline.py --limit 20
|
|
echo
|
|
echo "=== Summarization Engine ==="
|
|
python3 summarize.py
|
|
echo
|
|
echo "=== Phase 6 theme trend scan (new arrivals this cycle) ==="
|
|
python3 theme_scan.py
|
|
echo
|
|
echo "=== Soft-cap archive (dry-safe default: 30d / 5000 cap) ==="
|
|
python3 archive.py --days 30 --cap 5000
|
|
echo
|
|
echo "=== run_log tail (failure visibility) ==="
|
|
python3 -c "
|
|
import sqlite3
|
|
c = sqlite3.connect('oracle.db')
|
|
for r in c.execute('SELECT id,run_time,total_fetched,total_stored,sources_failed,notes FROM run_log ORDER BY id DESC LIMIT 1'):
|
|
print(' run', r[0], '|', r[1], '| fetched', r[2], '| stored', r[3], '| failed', r[4], '| notes:', r[5])
|
|
print(' live entries:', c.execute('SELECT COUNT(*) FROM entries').fetchone()[0])
|
|
print(' archived entries:', c.execute('SELECT COUNT(*) FROM entries_archive').fetchone()[0])
|
|
"
|
|
echo "=== Done ==="
|
|
} 2>&1 | tee "$LOG"
|