- adapters/__init__.py: add http_get() unified retry (429/5xx only, max 2 attempts, capped exp backoff) + AdapterHTTPError carrying failure_class; SourceAdapter.last_failure_class set on failure for pipeline capture. - arxiv/github/huggingface/hackernews/reddit: route HTTP through http_get. Preserves GitHub 403 rate-limit retry and Reddit 403/429 fast-bail. - schema.sql + pipeline.py: add run_log.failure_class column; rollup most- severe class across sources (5xx>4xx>429>error>zero_fetch>ok). - pipeline.py: ENABLE RSS in ENABLED_SOURCES (was registered, disabled). - RSS smoke test surfaced 3 broken feeds (anthropic 404, googleai 404, metaai 301) — left as-is, captured in feed_failures; URL fix is separate discovery task, not guessed. Verified: full dry-run fetches all 6 sources; github live fetch OK; Reddit 429 fast-bail preserved; no import/syntax errors.
This commit is contained in:
+28
-7
@@ -37,7 +37,7 @@ ADAPTERS = {
|
||||
}
|
||||
|
||||
# Default enabled sources
|
||||
ENABLED_SOURCES = ["github", "arxiv", "reddit", "hackernews", "huggingface"]
|
||||
ENABLED_SOURCES = ["github", "arxiv", "reddit", "hackernews", "huggingface", "rss"]
|
||||
|
||||
|
||||
def init_db(db_path: str, schema_path: str) -> sqlite3.Connection:
|
||||
@@ -252,9 +252,14 @@ def run_pipeline(sources: list[str] | None = None, limit: int = 20, dry_run: boo
|
||||
entries = adapter.fetch(limit=limit)
|
||||
except Exception as e:
|
||||
print(f" ✗ {source_name} failed: {e}")
|
||||
source_stats[source_name] = {"fetched": 0, "stored": 0, "error": str(e)}
|
||||
source_stats[source_name] = {"fetched": 0, "stored": 0,
|
||||
"error": str(e),
|
||||
"failure_class": "error"}
|
||||
continue
|
||||
|
||||
# Capture classification from the adapter (set by http_get on failure)
|
||||
fc = getattr(adapter, "last_failure_class", None)
|
||||
|
||||
# Add adapter_version to metadata
|
||||
for entry in entries:
|
||||
meta = json.loads(entry["raw_metadata"]) if isinstance(entry["raw_metadata"], str) else entry["raw_metadata"]
|
||||
@@ -262,7 +267,8 @@ def run_pipeline(sources: list[str] | None = None, limit: int = 20, dry_run: boo
|
||||
entry["raw_metadata"] = json.dumps(meta)
|
||||
|
||||
all_entries.extend(entries)
|
||||
source_stats[source_name] = {"fetched": len(entries), "stored": 0}
|
||||
source_stats[source_name] = {"fetched": len(entries), "stored": 0,
|
||||
"failure_class": fc or "ok"}
|
||||
print(f" Fetched: {len(entries)} entries")
|
||||
|
||||
# Small spacing between sources
|
||||
@@ -294,16 +300,31 @@ def run_pipeline(sources: list[str] | None = None, limit: int = 20, dry_run: boo
|
||||
# Zero-fetch (e.g. Reddit fully rate-limited) raises no exception but
|
||||
# is still a degraded run — record it so run_log can tell
|
||||
# "intermittent vs consistently-broken" apart over time.
|
||||
zero = [s for s, st in source_stats.items() if st.get("fetched", 0) == 0 and not st.get("error")]
|
||||
zero = [s for s, st in source_stats.items()
|
||||
if st.get("fetched", 0) == 0 and not st.get("error")]
|
||||
notes_parts = [f"{s}: {st['error']}" for s, st in source_stats.items() if st.get("error")]
|
||||
if zero:
|
||||
notes_parts.append(f"no-fetch (degraded): {', '.join(zero)}")
|
||||
notes = "; ".join(notes_parts) or "all sources ok"
|
||||
|
||||
# Rollup failure_class (issue #2): most severe across sources.
|
||||
# Priority: 5xx > 4xx > 429 > error > zero_fetch > ok
|
||||
rank = {"5xx": 5, "4xx": 4, "429": 3, "error": 2, "zero_fetch": 1, "ok": 0}
|
||||
classes = [st.get("failure_class", "ok") for st in source_stats.values()]
|
||||
if any(c in ("5xx", "4xx", "429", "error") for c in classes):
|
||||
run_fc = max((c for c in classes if c in rank),
|
||||
key=lambda c: rank[c])
|
||||
elif zero:
|
||||
run_fc = "zero_fetch"
|
||||
else:
|
||||
run_fc = "ok"
|
||||
try:
|
||||
conn.execute("""
|
||||
INSERT INTO run_log (total_fetched, total_stored, sources_ok, sources_failed, notes)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""", (len(all_entries), stored, json.dumps(ok), json.dumps(failed), notes))
|
||||
INSERT INTO run_log (total_fetched, total_stored, sources_ok,
|
||||
sources_failed, failure_class, notes)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""", (len(all_entries), stored, json.dumps(ok), json.dumps(failed),
|
||||
run_fc, notes))
|
||||
conn.commit()
|
||||
except Exception as e:
|
||||
print(f" ⚠ run_log write failed: {e}")
|
||||
|
||||
Reference in New Issue
Block a user