Phase 5: cron entry script, soft-cap archive, run_log zero-fetch degradation
- oracle-pipeline.sh: single cron entry point (pipeline -> summarize -> archive) - archive.py: soft-cap archival to entries_archive (preserve, not delete) - pipeline.py: record zero-fetch (rate-limited) runs as degraded in run_log.notes Verified: full script runs exit 0, run_log captures per-source status.
This commit is contained in:
+8
-1
@@ -231,7 +231,14 @@ def run_pipeline(sources: list[str] | None = None, limit: int = 20, dry_run: boo
|
||||
# Record run log (failure visibility + growth control) — before conn.close()
|
||||
ok = [s for s, st in source_stats.items() if not st.get("error")]
|
||||
failed = [s for s, st in source_stats.items() if st.get("error")]
|
||||
notes = "; ".join(f"{s}: {st['error']}" for s, st in source_stats.items() if st.get("error")) or "all sources ok"
|
||||
# Zero-fetch (e.g. Reddit fully rate-limited) raises no exception but
|
||||
# is still a degraded run — record it so run_log can tell
|
||||
# "intermittent vs consistently-broken" apart over time.
|
||||
zero = [s for s, st in source_stats.items() if st.get("fetched", 0) == 0 and not st.get("error")]
|
||||
notes_parts = [f"{s}: {st['error']}" for s, st in source_stats.items() if st.get("error")]
|
||||
if zero:
|
||||
notes_parts.append(f"no-fetch (degraded): {', '.join(zero)}")
|
||||
notes = "; ".join(notes_parts) or "all sources ok"
|
||||
try:
|
||||
conn.execute("""
|
||||
INSERT INTO run_log (total_fetched, total_stored, sources_ok, sources_failed, notes)
|
||||
|
||||
Reference in New Issue
Block a user