fix(adapters): shared retry helper + run_log failure_class + enable RSS (issues #1 #2 #9)

- adapters/__init__.py: add http_get() unified retry (429/5xx only, max 2
  attempts, capped exp backoff) + AdapterHTTPError carrying failure_class;
  SourceAdapter.last_failure_class set on failure for pipeline capture.
- arxiv/github/huggingface/hackernews/reddit: route HTTP through http_get.
  Preserves GitHub 403 rate-limit retry and Reddit 403/429 fast-bail.
- schema.sql + pipeline.py: add run_log.failure_class column; rollup most-
  severe class across sources (5xx>4xx>429>error>zero_fetch>ok).
- pipeline.py: ENABLE RSS in ENABLED_SOURCES (was registered, disabled).
- RSS smoke test surfaced 3 broken feeds (anthropic 404, googleai 404,
  metaai 301) — left as-is, captured in feed_failures; URL fix is separate
  discovery task, not guessed.

Verified: full dry-run fetches all 6 sources; github live fetch OK;
Reddit 429 fast-bail preserved; no import/syntax errors.

adapters/__init__.py

@@ -1,11 +1,18 @@
 """Source adapters for AI Research Oracle."""
 
+import urllib.request
+import urllib.error
+import time
 from abc import ABC, abstractmethod
 
 
 class SourceAdapter(ABC):
     """Base class for all ingestion adapters."""
 
+    def __init__(self):
+        # Set by http_get when a request fails (issue #2 classification)
+        self.last_failure_class = None
+
     @abstractmethod
     def name(self) -> str:
         """Source name: 'github', 'arxiv', 'reddit'."""
@@ -15,3 +22,81 @@ class SourceAdapter(ABC):
     def fetch(self, query: str = "", limit: int = 20) -> list[dict]:
         """Return entries matching DB schema fields."""
         pass
+
+
+class AdapterHTTPError(Exception):
+    """Raised by http_get on non-retryable or exhausted HTTP failures.
+
+    failure_class is one of: '429', '5xx', '4xx', 'error'.
+    Consumed by pipeline.py to populate run_log.failure_class (issue #2).
+    """
+
+    def __init__(self, status, failure_class, message=""):
+        super().__init__(message)
+        self.status = status
+        self.failure_class = failure_class
+
+
+def classify_http_status(code):
+    """Map an HTTP status code to a run_log failure_class."""
+    if code == 429:
+        return "429"
+    if 500 <= code < 600:
+        return "5xx"
+    return "4xx"  # 401/403/404 etc = client/config problem, not retried
+
+
+def http_get(url, headers=None, timeout=15, max_retries=2,
+             backoff_base=2, retry_403_ratelimit=False, return_headers=False,
+             owner=None):
+    """GET with a unified retry policy shared by all adapters (issue #1).
+
+    Retries ONLY on 429 and 5xx (transient). 4xx other than 429 are NOT
+    retried (config/client problems). Optional GitHub-style 403 rate-limit
+    handling via retry_403_ratelimit (waits for X-RateLimit-Reset header).
+
+    Returns response body bytes (or (body, headers) tuple if return_headers).
+    Raises AdapterHTTPError on non-retryable or exhausted failures, carrying
+    .failure_class for run_log classification. If owner is provided, sets
+    owner.last_failure_class so the pipeline can record it (issue #2).
+    """
+    def _fail(code, fclass, msg):
+        if owner is not None:
+            owner.last_failure_class = fclass
+        raise AdapterHTTPError(code, fclass, msg)
+
+    req = urllib.request.Request(url, headers=headers or {})
+    last_err = None
+    for attempt in range(max_retries + 1):
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as resp:
+                body = resp.read()
+                if return_headers:
+                    return body, resp.headers
+                return body
+        except urllib.error.HTTPError as e:
+            last_err = e
+            code = e.code
+            if code == 429 or (code == 403 and retry_403_ratelimit):
+                if attempt < max_retries:
+                    if code == 403:
+                        reset = int(e.headers.get("X-RateLimit-Reset", 0))
+                        wait = (max(reset - int(time.time()), 0) + 1
+                                if reset else backoff_base * (attempt + 1) * 15)
+                        time.sleep(min(wait, 300))
+                    else:
+                        time.sleep(min(backoff_base * (attempt + 1), 30))
+                    continue
+                fclass = "429" if code == 429 else "4xx"
+                _fail(code, fclass, f"HTTP {code} for {url} (exhausted)")
+            if 500 <= code < 600:
+                if attempt < max_retries:
+                    time.sleep(min(backoff_base * (attempt + 1), 30))
+                    continue
+                _fail(code, "5xx", f"HTTP {code} for {url} (exhausted)")
+            # other 4xx (401/403 w/o flag/404) — not retried
+            _fail(code, classify_http_status(code), f"HTTP {code} for {url}")
+        except Exception as e:  # non-HTTP (timeout, DNS, conn reset)
+            _fail(None, "error", f"Request error: {e}")
+    # Defensive: loop should always raise or return
+    _fail(getattr(last_err, "code", None), "error", str(last_err))

adapters/arxiv.py

@@ -32,7 +32,7 @@ import xml.etree.ElementTree as ET
 from datetime import datetime, timedelta, timezone
 from html import unescape
 
-from adapters import SourceAdapter
+from adapters import SourceAdapter, http_get, AdapterHTTPError
 
 # arXiv API
 ARXIV_API = "http://export.arxiv.org/api/query"
@@ -151,14 +151,12 @@ class ArxivAdapter(SourceAdapter):
             f"&max_results={max_results}"
         )
 
-        req = urllib.request.Request(url, headers={"User-Agent": "ai-oracle/0.1"})
-
         try:
-            with urllib.request.urlopen(req, timeout=30) as resp:
-                xml_data = resp.read().decode("utf-8")
-            return self._parse_atom(xml_data)
-        except urllib.error.HTTPError as e:
-            print(f"    HTTP {e.code} for arXiv query")
+            raw = http_get(url, headers={"User-Agent": "ai-oracle/0.1"},
+                           timeout=30, max_retries=2, owner=self)
+            return self._parse_atom(raw.decode("utf-8"))
+        except AdapterHTTPError as e:
+            print(f"    {e.failure_class}: arXiv query ({e})")
             return []
         except Exception as e:
             print(f"    arXiv request error: {e}")

adapters/github.py

@@ -17,7 +17,7 @@ import urllib.error
 import urllib.parse
 from datetime import datetime, timedelta, timezone
 
-from adapters import SourceAdapter
+from adapters import SourceAdapter, http_get, AdapterHTTPError
 
 
 class GitHubAdapter(SourceAdapter):
@@ -43,40 +43,27 @@ class GitHubAdapter(SourceAdapter):
         return headers
 
     def _request(self, url: str, max_retries: int = 2) -> dict | list | None:
-        """Make a GET request with retry on 403 (rate limit)."""
-        req = urllib.request.Request(url, headers=self._headers())
-
-        for attempt in range(max_retries + 1):
-            try:
-                with urllib.request.urlopen(req, timeout=15) as resp:
-                    data = json.loads(resp.read().decode("utf-8"))
-
-                # Check rate limit headers
-                remaining = int(resp.headers.get("X-RateLimit-Remaining", 0))
-                if remaining <= 5:
-                    print(f"    ⚠ Rate limit low ({remaining} remaining), stopping")
-                    break
-
-                return data
-            except urllib.error.HTTPError as e:
-                if e.code == 403:
-                    # Rate limited — reset time is in headers
-                    reset = int(e.headers.get("X-RateLimit-Reset", 0))
-                    if reset:
-                        wait = max(reset - int(time.time()), 0) + 1
-                        print(f"    ⚠ Rate limited, wait {wait}s")
-                    else:
-                        wait = 30 * (attempt + 1)
-                        print(f"    403 on attempt {attempt + 1}, retry in {wait}s")
-                    time.sleep(min(wait, 300))  # cap at 5 min
-                    continue
-                print(f"    HTTP {e.code} for {url}")
-                return None
-            except Exception as e:
-                print(f"    Request error: {e}")
-                return None
-
-        return None
+        """GET via shared retry helper; 403 rate-limit handled as transient."""
+        try:
+            raw, headers = http_get(
+                url, headers=self._headers(), timeout=15,
+                max_retries=max_retries, retry_403_ratelimit=True,
+                return_headers=True, owner=self)
+        except AdapterHTTPError as e:
+            print(f"    {e.failure_class}: GitHub {url}")
+            return None
+        # Informational: flag if we're close to the unauth rate ceiling
+        try:
+            remaining = int(headers.get("X-RateLimit-Remaining", 0))
+            if remaining <= 5:
+                print(f"    ⚠ Rate limit low ({remaining} remaining)")
+        except Exception:
+            pass
+        try:
+            return json.loads(raw.decode("utf-8"))
+        except Exception as e:
+            print(f"    GitHub decode error: {e}")
+            return None
 
     def _search_repos(self, query: str, sort: str = "stars", order: str = "desc", per_page: int = 30) -> list:
         """Search repositories via GitHub API."""

adapters/hackernews.py

@@ -20,7 +20,7 @@ import urllib.request
 import urllib.error
 from datetime import datetime, timezone
 
-from adapters import SourceAdapter
+from adapters import SourceAdapter, http_get, AdapterHTTPError
 
 
 class HackerNewsAdapter(SourceAdapter):
@@ -54,24 +54,19 @@ class HackerNewsAdapter(SourceAdapter):
         return "hackernews"
 
     def _request(self, path: str, max_retries: int = 2) -> dict | list | None:
-        """Make a GET request to the HN Firebase API."""
+        """GET via shared retry helper (retries 429/5xx)."""
         url = f"{self.BASE}{path}"
-        req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
-
-        for attempt in range(max_retries + 1):
-            try:
-                with urllib.request.urlopen(req, timeout=15) as resp:
-                    return json.loads(resp.read().decode("utf-8"))
-            except (urllib.error.HTTPError, urllib.error.URLError) as e:
-                if attempt < max_retries:
-                    time.sleep(3 * (attempt + 1))
-                    continue
-                print(f"    HTTP error: {e}")
-                return None
-            except Exception as e:
-                print(f"    Request error: {e}")
-                return None
-        return None
+        try:
+            raw = http_get(url, headers={"User-Agent": self.user_agent},
+                           timeout=15, max_retries=max_retries, owner=self)
+        except AdapterHTTPError as e:
+            print(f"    {e.failure_class}: HN {path}")
+            return None
+        try:
+            return json.loads(raw.decode("utf-8"))
+        except Exception as e:
+            print(f"    HN decode error: {e}")
+            return None
 
     def _is_ai_relevant(self, title: str) -> bool:
         """Check if a story title is AI/ML relevant.

adapters/huggingface.py

@@ -34,7 +34,7 @@ import urllib.request
 import urllib.error
 from datetime import datetime, timedelta, timezone
 
-from adapters import SourceAdapter
+from adapters import SourceAdapter, http_get, AdapterHTTPError
 
 
 class HuggingFaceAdapter(SourceAdapter):
@@ -84,24 +84,19 @@ class HuggingFaceAdapter(SourceAdapter):
         return headers
 
     def _request(self, path: str, max_retries: int = 2) -> list | dict | None:
-        """Make a GET request to the HF API."""
+        """GET via shared retry helper (retries 429/5xx)."""
         url = f"{self.BASE}{path}"
-        req = urllib.request.Request(url, headers=self._headers())
-
-        for attempt in range(max_retries + 1):
-            try:
-                with urllib.request.urlopen(req, timeout=20) as resp:
-                    return json.loads(resp.read().decode("utf-8"))
-            except (urllib.error.HTTPError, urllib.error.URLError) as e:
-                if attempt < max_retries:
-                    time.sleep(3 * (attempt + 1))
-                    continue
-                print(f"    HF API error: {e}")
-                return None
-            except Exception as e:
-                print(f"    Request error: {e}")
-                return None
-        return None
+        try:
+            raw = http_get(url, headers=self._headers(), timeout=20,
+                           max_retries=max_retries, owner=self)
+        except AdapterHTTPError as e:
+            print(f"    {e.failure_class}: HF {path}")
+            return None
+        try:
+            return json.loads(raw.decode("utf-8"))
+        except Exception as e:
+            print(f"    HF decode error: {e}")
+            return None
 
     def _is_ai_relevant(self, model: dict) -> bool:
         """Check if a model/dataset is AI/ML relevant.

adapters/reddit.py

@@ -24,7 +24,7 @@ import xml.etree.ElementTree as ET
 from datetime import datetime, timezone
 from html import unescape
 
-from adapters import SourceAdapter
+from adapters import SourceAdapter, http_get, AdapterHTTPError
 
 
 class RedditAdapter(SourceAdapter):
@@ -101,33 +101,27 @@ class RedditAdapter(SourceAdapter):
         return False
 
     def _fetch_rss(self, subreddit: str) -> list[dict]:
-        """Fetch RSS feed for a subreddit."""
-        url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
-        req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
+        """Fetch RSS feed for a subreddit (shared retry helper).
 
-        for attempt in range(2):  # max 2 attempts, fail fast
-            try:
-                with urllib.request.urlopen(req, timeout=10) as resp:
-                    xml_data = resp.read().decode("utf-8")
-                break
-            except urllib.error.HTTPError as e:
-                if e.code in (403,):
-                    print(f"    RSS blocked (HTTP {e.code}) for r/{subreddit}")
-                    return []
-                if e.code == 429:
-                    if attempt == 0:
-                        time.sleep(2)  # single retry with short backoff
-                        continue
-                    print(f"    RSS rate-limited for r/{subreddit}, skip")
-                    return []
-                print(f"    RSS HTTP {e.code} for r/{subreddit}")
-                return []
-            except Exception as e:
-                print(f"    RSS error r/{subreddit}: {e}")
-                return []
-        else:
-            print(f"    r/{subreddit}: still rate limited, skip")
+        Preserves prior fast-bail: 403 -> immediate []; 429 -> single 2s
+        retry then []; 5xx -> helper retry then []. Not slower than before.
+        """
+        url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
+        try:
+            raw = http_get(url, headers={"User-Agent": self.user_agent},
+                           timeout=10, max_retries=1, backoff_base=2, owner=self)
+        except AdapterHTTPError as e:
+            if e.status == 403:
+                print(f"    RSS blocked (HTTP 403) for r/{subreddit}")
+            elif e.status == 429:
+                print(f"    RSS rate-limited for r/{subreddit}, skip")
+            else:
+                print(f"    RSS {e.failure_class} for r/{subreddit}")
             return []
+        except Exception as e:
+            print(f"    RSS error r/{subreddit}: {e}")
+            return []
+        xml_data = raw.decode("utf-8")
 
         # Parse Atom XML
         entries = []
@@ -303,33 +297,27 @@ class RedditAdapter(SourceAdapter):
         return tags
 
     def _fetch_rss(self, subreddit: str) -> list[dict]:
-        """Fetch RSS feed for a subreddit."""
-        url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
-        req = urllib.request.Request(url, headers={"User-Agent": self.user_agent})
+        """Fetch RSS feed for a subreddit (shared retry helper).
 
-        for attempt in range(2):  # max 2 attempts, fail fast
-            try:
-                with urllib.request.urlopen(req, timeout=10) as resp:
-                    xml_data = resp.read().decode("utf-8")
-                break
-            except urllib.error.HTTPError as e:
-                if e.code in (403,):
-                    print(f"    RSS blocked (HTTP {e.code}) for r/{subreddit}")
-                    return []
-                if e.code == 429:
-                    if attempt == 0:
-                        time.sleep(2)  # single retry with short backoff
-                        continue
-                    print(f"    RSS rate-limited for r/{subreddit}, skip")
-                    return []
-                print(f"    RSS HTTP {e.code} for r/{subreddit}")
-                return []
-            except Exception as e:
-                print(f"    RSS error r/{subreddit}: {e}")
-                return []
-        else:
-            print(f"    r/{subreddit}: still rate limited, skip")
+        Preserves prior fast-bail: 403 -> immediate []; 429 -> single 2s
+        retry then []; 5xx -> helper retry then []. Not slower than before.
+        """
+        url = f"https://www.reddit.com/r/{subreddit}/hot/.rss?limit=50"
+        try:
+            raw = http_get(url, headers={"User-Agent": self.user_agent},
+                           timeout=10, max_retries=1, backoff_base=2, owner=self)
+        except AdapterHTTPError as e:
+            if e.status == 403:
+                print(f"    RSS blocked (HTTP 403) for r/{subreddit}")
+            elif e.status == 429:
+                print(f"    RSS rate-limited for r/{subreddit}, skip")
+            else:
+                print(f"    RSS {e.failure_class} for r/{subreddit}")
             return []
+        except Exception as e:
+            print(f"    RSS error r/{subreddit}: {e}")
+            return []
+        xml_data = raw.decode("utf-8")
 
         # Parse Atom XML
         entries = []

docs/Deployment-Plan.md

@@ -0,0 +1,42 @@
+# Deployment Plan — Athena MVP
+
+## Prerequisites
+- Docker installed on the target host
+- Network access to the 6 source APIs/feeds (arxiv, github, huggingface, hackernews, reddit, rss_feeds)
+- `GITHUB_TOKEN` available as an environment variable (optional, but raises the GitHub rate limit from 60/hr to 5000/hr)
+- `HUGGINGFACE_TOKEN` available as an environment variable
+- A local Ollama instance running `llama3.2:1b`, or an alternate reachable inference backend if swapping — per the model-agnostic `summarize(text) -> (summary, model)` contract
+- Hermes cron infrastructure available and able to invoke `oracle-pipeline.sh`
+
+## Setup
+1. Clone `main` (not a milestone branch) onto the target host
+2. `pip install -r requirements.txt` if present; otherwise confirm stdlib + `requests` are available
+3. Run `schema.sql` against a fresh `oracle.db` — this file is git-ignored and created locally, never committed
+4. Export required environment variables (`GITHUB_TOKEN`, `HUGGINGFACE_TOKEN`) — never hardcode these
+5. Build and run inside Docker with the 150MB memory cap and non-root user enforced, per the PRD platform requirements
+6. Manual smoke test: run `python3 pipeline.py` once and confirm ingest → store → summarize → score completes without errors before handing off to cron
+
+## Cron / Scheduling
+1. Confirm `oracle-pipeline.sh` is the entry point Hermes cron calls
+2. Schedule for 13:00 UTC daily
+3. Add a lock file or PID check so overlapping runs can't happen if a prior run is still in progress
+4. Confirm the cron environment actually carries the exported tokens — cron environments are frequently minimal and won't inherit an interactive shell's exports
+
+## Hermes Integration
+1. Confirm the wrapper script's exit codes are meaningful (0 = success, non-zero = failure) so Hermes can act on them
+2. Define where Hermes should look for pipeline output/logs
+3. Decide on a failure-notification path (Hermes alert, log flag, etc.) — **not yet specified, needs a decision**
+
+## Monitoring
+1. Aggregate logs from each pipeline stage (ingest, store, summarize, score)
+2. Track theme-scan new-arrival counts per cycle per theme — this is the core signal the falsification logic depends on, so it deserves visibility beyond raw logs
+3. Add a heartbeat/dead-man's-switch alert if a scheduled run doesn't fire, rather than relying on someone noticing missing data days later
+4. Watch memory usage against the 150MB cap under real production load, not just dev conditions
+
+## Rollback
+1. Back up `oracle.db` before any schema change — it's git-ignored and not recoverable from the repo itself
+2. If a bad deploy breaks the pipeline, revert to the last known-good commit on `main` and redeploy the Docker image
+3. Check falsification state (new-arrivals counters) after any rollback — rolling back mid-window could distort the 7-day dead-thesis calculation if not handled carefully
+
+---
+*Draft prepared by Claude from the README, whitepaper falsification logic, and MVP-PRD platform requirements on `main`. The Hermes failure-notification path is the one open decision blocking this from being final.*

docs/MVP-PRD.md

@@ -0,0 +1,161 @@
+## Chapter 1: Vision and Scope
+
+### Elevator Pitch
+
+Athena is an autonomous research intelligence engine that cuts through high-volume, fragmented signals by ingesting from multiple sources, surfacing cross-source convergence, and using falsification to distinguish real momentum from noise. While the initial focus is on AI signals, the system is designed to work with any class of signals. It delivers actionable insight into emerging trends and capability gaps while remaining model-agnostic and lightweight enough to run autonomously.
+
+### 1.1 Vision
+
+#### Why are we building it?
+
+The AI space produces an overwhelming volume of new research, tools, discussions, and model releases every day. Individual sources only provide partial views, making it difficult to distinguish genuine, sustained trends from one-day spikes. Without a system that can detect convergence across sources and validate momentum over time, real opportunities tied to emerging capability gaps are missed.
+
+#### What happens if we don’t build it?
+
+Without this capability, builders and researchers will continue to operate with fragmented, noisy signals. Early indicators of meaningful trends will remain hidden, decisions will stay reactive, and the ability to spot validated cross-source momentum before it becomes obvious will be lost.
+
+#### When must it be done?
+
+The foundational ability to reliably ingest, score, and validate signals through falsification must be established before meaningful trend detection and opportunity mapping can occur. This forms the core of the MVP and must be in place to enable the system to deliver on its intended value.
+
+### 1.2 Personas and Archetypes
+
+See committed document:  
+**`docs/Personas-and-Archetypes.md`** (on `MVP-milestone` branch)
+
+**Summary of scoped personas and archetypes for MVP:**
+
+**Personas**
+- Pers-1 (Bob) – Sector Trend Tracker (New to AI)
+- Pers-2 (Alice) – Content Creator
+- Pers-3 (Sam) – Hermes Research Agent
+
+**Archetypes**
+- Arch-1 (Small Scrappy VPS)
+- Arch-2 (Research Consumption Layer)
+
+All user stories in this PRD are scoped to combinations of the above.
+
+### 1.3 Use Case Priority Taxonomy
+
+This PRD focuses on defining the core functionality required for MVP. It also catalogs use cases and requirements across V1.0 – V1.5 to maintain context. The primary goal is to deliver a working MVP, with future PRDs derived from the remaining prioritized content.
+
+We will use the following prioritization model:
+
+- **MVP**: The short list of P1 use cases required to prove the concept with a working prototype.
+- **P1**: Use cases that are fundamental to successfully implementing the product vision.
+- **P2**: Use cases that add strength, convenience, and quality to the product vision.
+- **P3**: Use cases that bring additional value but can be cut if time or resource constrained.
+
+## Chapter 2: User Stories (Bob)
+These user stories are based on the personas and archetypes document contained in this repo.  
+
+Chapter 2.1 - Bob's user stories
+
+**As Bob, I want to…**
+
+**Bob-1.** Automatically receive daily updates on new AI innovations without having to manually check multiple sources.
+
+**Bob-5.** See emerging trends and differentiate durable signal from temporary or artificial hype.
+
+**Bob-10.** See when the same idea or pattern is appearing across multiple independent sources (GitHub, arXiv, Reddit, HN, HF).
+
+**Bob-15.** Identify emerging capability gaps or opportunities early, before they become widely obvious.
+
+**Bob-20.** Have research that gives me confidence it is exhaustive and vetted.
+
+**Bob-25.** Adjust or alter the underlying data feeds and weights so I can tune the accuracy and relevance of the output.
+
+**Bob-30.** Understand why a particular signal is considered strong or weak (e.g., cross-source convergence or falsification results).
+
+Chapter 2.2 - Alice's user stories
+As Alice, I want to…
+Alice-1. Integrate deep, vetted research directly into my existing content production pipeline so I can reduce manual research time.
+Alice-5. Query the research system with follow-up questions to explore specific angles or topics on demand.
+Alice-10. Have my tools automatically receive curated, high-signal research so I can focus on content creation instead of information filtering.
+Alice-15. Get research outputs in a structured format that my existing AI tools and workflows can consume without manual reformatting.
+Alice-20. Quickly surface non-obvious insights and patterns from research data to develop more compelling content angles.
+Alice-25. Control which research sources and signals are prioritized so the output stays aligned with my content focus and audience.
+Alice-30. Understand the reasoning and supporting evidence behind key research findings so I can speak to them confidently in my content.
+
+## Chapter 3: Requirements
+Requirements defined as what the product / system must do, differentiated from what the persona can accomplish. Requirements are defined to meet the needs of use cases as well as the architectural system design.
+
+High level design (refer to ***TBD_Design.MD for full design details)
+High-Level Design
+.
+├── Runtime Environment
+│   ├── Linux
+│   └── Docker (containerized)
+│
+├── Core Components
+│   ├── Database: SQLite
+│   ├── Scheduling: Cron
+│   └── Runtime: Python
+│
+├── Connectivity
+│   ├── Outbound (Internet)
+│   │   ├── HTTP client for data feeds (RSS, cURL, optional Playwright)
+│   │   └── OpenAI-compatible inference endpoints
+│   ├── Inbound (Internet)
+│   │   └── HTTP server endpoint (MCP + external consumers)
+│   └── Internal (Intranet)
+│       └── HTTP client for local inference (e.g. Hermes)
+│
+├── Storage
+│   ├── File system (daily digest artifacts stored outside container)
+│   └── Temporary working storage during pipeline execution
+│
+├── Configuration & Secrets
+│   ├── Research topic manifest (feeds, URLs, declarations)
+│   ├── System settings (YAML)
+│   └── Secrets (.env)
+│
+├── Business Logic / Pipeline Flow
+│   ├── Starting trigger
+│   ├── Preflight checks
+│   ├── Query feeds → Temporary result storage
+│   ├── Vet and promote final results to database
+│   ├── Optional daily digest generation
+│   └── Cleanup and sleep
+│
+└── Observability
+├── Structured logging
+├── Diagnostics and instrumentation (inside Docker)
+└── Health/status reporting
+
+3.1 Setup and configuration (REQ-SNC-XX)
+Requirements for initial setup, deployment configs, updating, and uninstall
+    REQ-SNC-05 -, with outbound access to the internet and in/outbound access to the underlying OS network
+    REQ-SNC-10 - The installation process shall be a single command which can be run interactively or silently
+    REQ-SNC-15 - The insallation shall utilize best-practice settings and secrets storage
+    REQ-SNC-20 - 
+
+3.2 Platform requirements (REQ-PLT-XX)
+    REQ-PLT-05 -  All processes will run as standard user (no admin / sudo elevation necessary)
+    REQ-PLT-10 -  ...
+    REQ-PLT-15 -  The system shall be Docker based limited to 150MB of memory
+Requirements addressing what OS and hardware support is in scope
+3.3 Performance and scalability (REQ-PERF-XX)
+3.4 Instrumenation and diagnostics (REQ-DIAG-XX)
+3.5 
+### 3.1 Reliability
+
+REQ-REL-05: Once setup and configured, the system will reliably operate without interaction from the user.
+REQ-REL-10: The system shall automatically retry failed source fetches with exponential backoff.
+REQ-REL-15: The system shall not lose previously stored data on restart or failure.
+REQ-REL-20: The daily pipeline shall complete successfully even if one or more sources are unavailable.
+
+3.3 Observability and Diagnostics
+REQ-DIAG-05: The system shall produce structured logs with timestamps and severity levels.
+REQ-DIAG-10: A health check endpoint or command shall report overall system status and last successful run.
+REQ-DIAG-15: Run logs shall capture per-source success/failure and basic metrics (items fetched, stored, failed).
+
+3.4 Security 
+REQ-SEC-05: All external HTTP calls shall use TLS.
+REQ-SEC-10: No secrets shall be hardcoded or stored in plaintext.
+REQ-SEC-15: The system shall support least-privilege access for outbound API calls.
+
+3.5 Integration and extensibility 
+REQ-INT-05: The system shall expose research output in a structured, machine-readable format (e.g., JSON files or API) consumable by external tools.
+REQ-INT-10: The adapter layer shall support adding new sources without modifying core pipeline logic.

docs/Personas-and-Archetypes.md

@@ -0,0 +1,59 @@
+# Personas and Archetypes (Derived from /main)
+
+**Status**: First Draft – Derived from existing docs on `main`  
+**Source**: README.md + whitepaper.md (branch: main)  
+**Date**: 2026-07-08
+
+## Overview
+
+This document extracts the implied users and operating contexts directly from the current documentation on the `main` branch. It serves as the baseline before we expand or refine.
+
+---
+
+## Personas
+
+### Pers-1 (Bob) – Sector Trend Tracker (New to AI)
+- Is relatively new to AI and the broader space.
+- Wants to stay current with trends in AI (and potentially other sectors) without getting overwhelmed.
+- Needs a way to keep up with the high volume of new research, tools, and discussions with minimal ongoing effort.
+- Benefits from a system that filters noise and surfaces what actually matters.
+
+### Pers-2 (Alice) – Content Creator
+- Runs a YouTube channel and an X account with 25k followers.
+- Goal is to grow her audience significantly (targeting 1M followers).
+- Needs help doing research across AI and related topics.
+- Wants to convert research signals into interesting, timely, and compelling content for her audience.
+
+### Pers-3 (Sam) – Hermes Research Agent
+- Is a Hermes agent profile with its own memory and endpoint connection.
+- Acts as the dedicated research team member for an AI-first development team.
+- Needs to stay current on a defined market segment (AI for the MVP; extensible to other segments later).
+- Consumes structured signals from Athena to support ongoing research and decision-making within the team.
+
+---
+
+## Archetypes (Operating Environments)
+
+### Arch-1 (Small Scrappy VPS)
+- Small, low-budget, and scrappy VPS environment.
+- Used primarily for learning and early prototyping.
+- Requires a small-footprint workload that can be memory-constrained so it doesn’t destabilize the host system.
+
+### Arch-2 (Research Consumption Layer)
+- Functions as a consumption layer for Athena’s research output.
+- Designed to support downstream AI systems (examples: MCP tools, LoRA adapters, or other agent profiles).
+- Focuses on making Athena’s signals and summaries easily consumable by other systems rather than direct human use.
+
+---
+
+## Notes & Limitations (from /main)
+
+- The current documentation does **not** describe team or multi-user usage.
+- Emphasis is on autonomous operation and signal integrity.
+- Polished human-facing interfaces (e.g., daily digest) are not yet built.
+
+---
+
+## Next Steps
+
+This version incorporates the updated personas and archetypes.

docs/Prioritized-Task-List.md

@@ -0,0 +1,42 @@
+# Prioritized Task List — Athena MVP
+
+Tied to the personas and requirements in `MVP-PRD.md`. Ordered by phase; within each phase, roughly in the order they should be tackled.
+
+## Phase 1: Get Running Daily
+- [ ] Verify the cron entry (`oracle-pipeline.sh`) fires reliably at 13:00 UTC under Hermes
+- [ ] Confirm `pipeline.py` runs the full ingest → store → summarize → score cycle without manual intervention
+- [ ] Add a lock/guard so a slow run can't overlap with the next day's cron trigger
+- [ ] Validate all 6 adapters (arxiv, github, huggingface, hackernews, reddit, rss_feeds) independently — one adapter failing shouldn't kill the whole run
+- [ ] Confirm environment-only secrets (`GITHUB_TOKEN`, `HUGGINGFACE_TOKEN`) resolve correctly in the cron context (cron environments are often stripped down compared to an interactive shell)
+
+## Phase 2: Core Functionality
+- [ ] Confirm `schema.sql` initializes `oracle.db` cleanly and stays idempotent across repeated runs
+- [ ] Verify `theme_scan.py`'s 4-theme tagging (tool-call, context, compute, trust) against a few real days of data
+- [ ] Confirm the falsification counter (new arrivals per cycle) is genuinely idempotent — re-running against unchanged data must yield 0 new
+- [ ] Wire `summarize.py` to degrade gracefully when the Ollama endpoint (`llama3.2:1b`) isn't reachable — ingestion, scoring, and theme-scan must keep running without it
+- [ ] Confirm `archive.py`'s cold-storage rotation doesn't delete data still needed inside the 7-day falsification window
+
+## Phase 3: Observability & Reliability
+- [ ] Add structured logging per pipeline stage (ingest, store, summarize, score) with pass/fail per adapter
+- [ ] Surface theme-scan counts (new arrivals per theme per cycle) somewhere inspectable, not just buried in log files
+- [ ] Add a daily heartbeat/health check so a silent failure (e.g. cron didn't fire at all) is detectable rather than just showing up as missing data later
+- [ ] Decide and implement retry/backoff behavior for adapters that hit rate limits (especially GitHub without a token: 60/hr)
+
+## Phase 4: Human Consumption Layer
+- [ ] Extend `query.py` to support Bob's cross-source convergence lookups and Alice's curated-research pulls
+- [ ] Define the output format(s) for a "trend confirmed" vs. "trend killed" verdict (per the 7-day dead-thesis rule)
+- [ ] Decide how Alice's content pipeline actually consumes Athena's output — file drop, API, direct DB read — this is currently undefined
+
+## Phase 5: Validation & UAT
+- [ ] Run the pipeline unattended for at least one full 7-day falsification window
+- [ ] Manually verify at least one theme through to a real "confirmed" or "killed" verdict
+- [ ] Walk Bob's and Alice's user stories from `MVP-PRD.md` end-to-end against real output, not synthetic data
+- [ ] Confirm memory stays under the 150MB cap under real daily load, not just in a light dev test
+
+## Phase 6: Hermes Integration
+- [ ] Confirm `oracle-pipeline.sh`'s contract matches what Hermes cron expects (exit codes, output location)
+- [ ] Decide how Hermes is notified on pipeline failure vs. success — not yet specified
+- [ ] Confirm the non-root execution requirement is actually satisfied inside the Hermes-invoked environment, not just in local Docker testing
+
+---
+*Draft prepared by Claude from the MVP-PRD, Personas doc, and README/whitepaper on `main`. Open items flagged "not yet specified" need a decision before Phase 4–6 can be considered done.*

pipeline.py

@@ -37,7 +37,7 @@ ADAPTERS = {
 }
 
 # Default enabled sources
-ENABLED_SOURCES = ["github", "arxiv", "reddit", "hackernews", "huggingface"]
+ENABLED_SOURCES = ["github", "arxiv", "reddit", "hackernews", "huggingface", "rss"]
 
 
 def init_db(db_path: str, schema_path: str) -> sqlite3.Connection:
@@ -252,9 +252,14 @@ def run_pipeline(sources: list[str] | None = None, limit: int = 20, dry_run: boo
             entries = adapter.fetch(limit=limit)
         except Exception as e:
             print(f"  ✗ {source_name} failed: {e}")
-            source_stats[source_name] = {"fetched": 0, "stored": 0, "error": str(e)}
+            source_stats[source_name] = {"fetched": 0, "stored": 0,
+                                         "error": str(e),
+                                         "failure_class": "error"}
             continue
 
+        # Capture classification from the adapter (set by http_get on failure)
+        fc = getattr(adapter, "last_failure_class", None)
+
         # Add adapter_version to metadata
         for entry in entries:
             meta = json.loads(entry["raw_metadata"]) if isinstance(entry["raw_metadata"], str) else entry["raw_metadata"]
@@ -262,7 +267,8 @@ def run_pipeline(sources: list[str] | None = None, limit: int = 20, dry_run: boo
             entry["raw_metadata"] = json.dumps(meta)
 
         all_entries.extend(entries)
-        source_stats[source_name] = {"fetched": len(entries), "stored": 0}
+        source_stats[source_name] = {"fetched": len(entries), "stored": 0,
+                                     "failure_class": fc or "ok"}
         print(f"    Fetched: {len(entries)} entries")
 
         # Small spacing between sources
@@ -294,16 +300,31 @@ def run_pipeline(sources: list[str] | None = None, limit: int = 20, dry_run: boo
         # Zero-fetch (e.g. Reddit fully rate-limited) raises no exception but
         # is still a degraded run — record it so run_log can tell
         # "intermittent vs consistently-broken" apart over time.
-        zero = [s for s, st in source_stats.items() if st.get("fetched", 0) == 0 and not st.get("error")]
+        zero = [s for s, st in source_stats.items()
+                if st.get("fetched", 0) == 0 and not st.get("error")]
         notes_parts = [f"{s}: {st['error']}" for s, st in source_stats.items() if st.get("error")]
         if zero:
             notes_parts.append(f"no-fetch (degraded): {', '.join(zero)}")
         notes = "; ".join(notes_parts) or "all sources ok"
+
+        # Rollup failure_class (issue #2): most severe across sources.
+        # Priority: 5xx > 4xx > 429 > error > zero_fetch > ok
+        rank = {"5xx": 5, "4xx": 4, "429": 3, "error": 2, "zero_fetch": 1, "ok": 0}
+        classes = [st.get("failure_class", "ok") for st in source_stats.values()]
+        if any(c in ("5xx", "4xx", "429", "error") for c in classes):
+            run_fc = max((c for c in classes if c in rank),
+                         key=lambda c: rank[c])
+        elif zero:
+            run_fc = "zero_fetch"
+        else:
+            run_fc = "ok"
         try:
             conn.execute("""
-                INSERT INTO run_log (total_fetched, total_stored, sources_ok, sources_failed, notes)
-                VALUES (?, ?, ?, ?, ?)
-            """, (len(all_entries), stored, json.dumps(ok), json.dumps(failed), notes))
+                INSERT INTO run_log (total_fetched, total_stored, sources_ok,
+                                     sources_failed, failure_class, notes)
+                VALUES (?, ?, ?, ?, ?, ?)
+            """, (len(all_entries), stored, json.dumps(ok), json.dumps(failed),
+                  run_fc, notes))
             conn.commit()
         except Exception as e:
             print(f"  ⚠ run_log write failed: {e}")

schema.sql

@@ -22,6 +22,8 @@ CREATE INDEX IF NOT EXISTS idx_entries_category ON entries(category_tags);
 -- Partial failures (e.g. Reddit rate-limited) are detectable here, not hidden
 -- as a "complete" run. Also enables future pruning decisions (entries older
 -- than N days with no re-fetch can be archived).
+-- failure_class (issue #2): one of 4xx / 5xx / 429 / zero_fetch / ok, derived
+-- from the real HTTP response via adapters.http_get, not guessed after the fact.
 CREATE TABLE IF NOT EXISTS run_log (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     run_time TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
@@ -29,6 +31,7 @@ CREATE TABLE IF NOT EXISTS run_log (
     total_stored INTEGER DEFAULT 0,
     sources_ok TEXT,       -- JSON list of sources that succeeded
     sources_failed TEXT,   -- JSON list of sources that errored/skipped
+    failure_class TEXT,    -- 4xx / 5xx / 429 / zero_fetch / ok
     notes TEXT
 );