- adapters/__init__.py: add http_get() unified retry (429/5xx only, max 2 attempts, capped exp backoff) + AdapterHTTPError carrying failure_class; SourceAdapter.last_failure_class set on failure for pipeline capture. - arxiv/github/huggingface/hackernews/reddit: route HTTP through http_get. Preserves GitHub 403 rate-limit retry and Reddit 403/429 fast-bail. - schema.sql + pipeline.py: add run_log.failure_class column; rollup most- severe class across sources (5xx>4xx>429>error>zero_fetch>ok). - pipeline.py: ENABLE RSS in ENABLED_SOURCES (was registered, disabled). - RSS smoke test surfaced 3 broken feeds (anthropic 404, googleai 404, metaai 301) — left as-is, captured in feed_failures; URL fix is separate discovery task, not guessed. Verified: full dry-run fetches all 6 sources; github live fetch OK; Reddit 429 fast-bail preserved; no import/syntax errors.
This commit is contained in:
+22
-35
@@ -17,7 +17,7 @@ import urllib.error
|
||||
import urllib.parse
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters import SourceAdapter, http_get, AdapterHTTPError
|
||||
|
||||
|
||||
class GitHubAdapter(SourceAdapter):
|
||||
@@ -43,40 +43,27 @@ class GitHubAdapter(SourceAdapter):
|
||||
return headers
|
||||
|
||||
def _request(self, url: str, max_retries: int = 2) -> dict | list | None:
|
||||
"""Make a GET request with retry on 403 (rate limit)."""
|
||||
req = urllib.request.Request(url, headers=self._headers())
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
# Check rate limit headers
|
||||
remaining = int(resp.headers.get("X-RateLimit-Remaining", 0))
|
||||
if remaining <= 5:
|
||||
print(f" ⚠ Rate limit low ({remaining} remaining), stopping")
|
||||
break
|
||||
|
||||
return data
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 403:
|
||||
# Rate limited — reset time is in headers
|
||||
reset = int(e.headers.get("X-RateLimit-Reset", 0))
|
||||
if reset:
|
||||
wait = max(reset - int(time.time()), 0) + 1
|
||||
print(f" ⚠ Rate limited, wait {wait}s")
|
||||
else:
|
||||
wait = 30 * (attempt + 1)
|
||||
print(f" 403 on attempt {attempt + 1}, retry in {wait}s")
|
||||
time.sleep(min(wait, 300)) # cap at 5 min
|
||||
continue
|
||||
print(f" HTTP {e.code} for {url}")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f" Request error: {e}")
|
||||
return None
|
||||
|
||||
return None
|
||||
"""GET via shared retry helper; 403 rate-limit handled as transient."""
|
||||
try:
|
||||
raw, headers = http_get(
|
||||
url, headers=self._headers(), timeout=15,
|
||||
max_retries=max_retries, retry_403_ratelimit=True,
|
||||
return_headers=True, owner=self)
|
||||
except AdapterHTTPError as e:
|
||||
print(f" {e.failure_class}: GitHub {url}")
|
||||
return None
|
||||
# Informational: flag if we're close to the unauth rate ceiling
|
||||
try:
|
||||
remaining = int(headers.get("X-RateLimit-Remaining", 0))
|
||||
if remaining <= 5:
|
||||
print(f" ⚠ Rate limit low ({remaining} remaining)")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
except Exception as e:
|
||||
print(f" GitHub decode error: {e}")
|
||||
return None
|
||||
|
||||
def _search_repos(self, query: str, sort: str = "stars", order: str = "desc", per_page: int = 30) -> list:
|
||||
"""Search repositories via GitHub API."""
|
||||
|
||||
Reference in New Issue
Block a user