Fix arxiv/reddit timeouts: batch queries, retry logic, reduced backoff

adapters/arxiv.py

@@ -143,8 +143,8 @@ class ArxivAdapter(SourceAdapter):
 
         return papers
 
-    def _request(self, query: str, max_results: int = 20, sort_by="submittedDate") -> list[dict]:
-        """Make an arXiv API request."""
+    def _request(self, query: str, max_results: int = 20, sort_by="submittedDate", retries: int = 3) -> list[dict]:
+        """Make an arXiv API request with retry on 429."""
         url = (
             f"{ARXIV_API}"
             f"?search_query={urllib.parse.quote(query)}"
@@ -153,18 +153,29 @@ class ArxivAdapter(SourceAdapter):
             f"&max_results={max_results}"
         )
 
-        req = urllib.request.Request(url, headers=browser_headers())
+        for attempt in range(retries):
+            req = urllib.request.Request(url, headers=browser_headers())
+            try:
+                with urllib.request.urlopen(req, timeout=15) as resp:
+                    xml_data = resp.read().decode("utf-8")
+                return self._parse_atom(xml_data)
+            except urllib.error.HTTPError as e:
+                if e.code == 429 and attempt < retries - 1:
+                    wait = (attempt + 1) * self.rate_limit
+                    print(f"    HTTP 429 for arXiv query, retrying in {wait}s...")
+                    jitter_sleep(wait)
+                    continue
+                print(f"    HTTP {e.code} for arXiv query")
+                return []
+            except Exception as e:
+                if attempt < retries - 1:
+                    print(f"    arXiv request error: {e}, retrying...")
+                    jitter_sleep(2)
+                    continue
+                print(f"    arXiv request error: {e}")
+                return []
 
-        try:
-            with urllib.request.urlopen(req, timeout=30) as resp:
-                xml_data = resp.read().decode("utf-8")
-            return self._parse_atom(xml_data)
-        except urllib.error.HTTPError as e:
-            print(f"    HTTP {e.code} for arXiv query")
-            return []
-        except Exception as e:
-            print(f"    arXiv request error: {e}")
-            return []
+        return []
 
     def _score(self, paper: dict, age_days: float) -> float:
         """Score based on AI-methodology relevance, not structural metadata.
@@ -385,12 +396,10 @@ class ArxivAdapter(SourceAdapter):
             papers = self._request(query, max_results=limit, sort_by="submittedDate")
             all_papers.extend(papers)
         else:
-            # Fetch from each configured category
-            for cat in self.categories:
-                q = f"cat:{cat}"
-                cat_papers = self._request(q, max_results=limit, sort_by="submittedDate")
-                all_papers.extend(cat_papers)
-                jitter_sleep(self.rate_limit)
+            # Batch all categories into ONE request to stay within timeout budget
+            batch_query = " OR ".join(f"cat:{cat}" for cat in self.categories)
+            cat_papers = self._request(batch_query, max_results=limit * len(self.categories), sort_by="submittedDate")
+            all_papers.extend(cat_papers)
 
             # Deduplicate by arxiv_id
             seen = set()