Fix arxiv/reddit timeouts: batch queries, retry logic, reduced backoff
This commit is contained in:
+4
-4
@@ -91,7 +91,7 @@ def cmd_ingest(args):
|
||||
entries = []
|
||||
error = None
|
||||
try:
|
||||
entries = adapter.fetch(limit=args.limit, timeout=10)
|
||||
entries = adapter.fetch(limit=args.limit, timeout=30)
|
||||
except TypeError:
|
||||
# Old adapter signature without timeout param — use thread-based fallback
|
||||
import threading
|
||||
@@ -103,9 +103,9 @@ def cmd_ingest(args):
|
||||
result["error"] = str(e)
|
||||
t = threading.Thread(target=_fetch, daemon=True)
|
||||
t.start()
|
||||
t.join(timeout=10)
|
||||
t.join(timeout=30)
|
||||
if t.is_alive():
|
||||
error = f"timeout after 10s"
|
||||
error = f"timeout after 30s"
|
||||
else:
|
||||
entries = result["entries"]
|
||||
error = result["error"]
|
||||
@@ -527,7 +527,7 @@ def main():
|
||||
p_weekly.add_argument("--json", action="store_true", help="Output as JSON")
|
||||
|
||||
# metrics
|
||||
p_metrics = subparsers.add_parser("metrics", help="Pipeline metrics and adapter health")
|
||||
p_metrics = sub.add_parser("metrics", help="Pipeline metrics and adapter health")
|
||||
p_metrics.add_argument("--db", default="oracle.db")
|
||||
p_metrics.add_argument("--days", type=int, default=7)
|
||||
p_metrics.add_argument("--adapters", action="store_true", help="Show adapter health table")
|
||||
|
||||
Reference in New Issue
Block a user