- adapters/__init__.py: add http_get() unified retry (429/5xx only, max 2 attempts, capped exp backoff) + AdapterHTTPError carrying failure_class; SourceAdapter.last_failure_class set on failure for pipeline capture. - arxiv/github/huggingface/hackernews/reddit: route HTTP through http_get. Preserves GitHub 403 rate-limit retry and Reddit 403/429 fast-bail. - schema.sql + pipeline.py: add run_log.failure_class column; rollup most- severe class across sources (5xx>4xx>429>error>zero_fetch>ok). - pipeline.py: ENABLE RSS in ENABLED_SOURCES (was registered, disabled). - RSS smoke test surfaced 3 broken feeds (anthropic 404, googleai 404, metaai 301) — left as-is, captured in feed_failures; URL fix is separate discovery task, not guessed. Verified: full dry-run fetches all 6 sources; github live fetch OK; Reddit 429 fast-bail preserved; no import/syntax errors.
This commit is contained in:
+6
-8
@@ -32,7 +32,7 @@ import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from html import unescape
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters import SourceAdapter, http_get, AdapterHTTPError
|
||||
|
||||
# arXiv API
|
||||
ARXIV_API = "http://export.arxiv.org/api/query"
|
||||
@@ -151,14 +151,12 @@ class ArxivAdapter(SourceAdapter):
|
||||
f"&max_results={max_results}"
|
||||
)
|
||||
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "ai-oracle/0.1"})
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
xml_data = resp.read().decode("utf-8")
|
||||
return self._parse_atom(xml_data)
|
||||
except urllib.error.HTTPError as e:
|
||||
print(f" HTTP {e.code} for arXiv query")
|
||||
raw = http_get(url, headers={"User-Agent": "ai-oracle/0.1"},
|
||||
timeout=30, max_retries=2, owner=self)
|
||||
return self._parse_atom(raw.decode("utf-8"))
|
||||
except AdapterHTTPError as e:
|
||||
print(f" {e.failure_class}: arXiv query ({e})")
|
||||
return []
|
||||
except Exception as e:
|
||||
print(f" arXiv request error: {e}")
|
||||
|
||||
Reference in New Issue
Block a user