Fix arxiv/reddit timeouts: batch queries, retry logic, reduced backoff

This commit is contained in:
Epictetus
2026-07-22 14:51:51 +00:00
parent 641d531d88
commit ce440149f0
3 changed files with 38 additions and 29 deletions
+4 -4
View File
@@ -91,7 +91,7 @@ def cmd_ingest(args):
entries = []
error = None
try:
entries = adapter.fetch(limit=args.limit, timeout=10)
entries = adapter.fetch(limit=args.limit, timeout=30)
except TypeError:
# Old adapter signature without timeout param — use thread-based fallback
import threading
@@ -103,9 +103,9 @@ def cmd_ingest(args):
result["error"] = str(e)
t = threading.Thread(target=_fetch, daemon=True)
t.start()
t.join(timeout=10)
t.join(timeout=30)
if t.is_alive():
error = f"timeout after 10s"
error = f"timeout after 30s"
else:
entries = result["entries"]
error = result["error"]
@@ -527,7 +527,7 @@ def main():
p_weekly.add_argument("--json", action="store_true", help="Output as JSON")
# metrics
p_metrics = subparsers.add_parser("metrics", help="Pipeline metrics and adapter health")
p_metrics = sub.add_parser("metrics", help="Pipeline metrics and adapter health")
p_metrics.add_argument("--db", default="oracle.db")
p_metrics.add_argument("--days", type=int, default=7)
p_metrics.add_argument("--adapters", action="store_true", help="Show adapter health table")