FIX: first_seen re-stamp bug — persist true source publish date, idempotent upsert, backfill DB

adapters/_store.py

@@ -0,0 +1,127 @@
+"""Shared DB storage helper for Athena adapters.
+
+Fixes the first_seen re-stamp bug (2026-07-13, Tony):
+  - Previously every adapter did INSERT OR REPLACE with first_seen = harvest
+    time, so re-harvesting an existing URL OVERWROTE the true publish date
+    with today's date. The recency guard then believed stale stories were new.
+  - true_first_seen() derives the REAL publish date from raw_metadata:
+        hackernews -> raw_meta['time']      (unix epoch)
+        reddit     -> raw_meta['published'] (ISO)
+        rss        -> raw_meta['published'] (RFC822 / ISO)
+        arxiv      -> raw_meta['published'] (ISO)
+        huggingface-> raw_meta['createdAt'] (ISO)
+    Falls back to harvest time only if no source date exists.
+  - upsert_entries() is idempotent: first sight stores the true first_seen;
+    re-encounter PRESERVES the original first_seen and only bumps last_updated.
+"""
+import json
+import sqlite3
+from datetime import datetime, timezone
+
+
+def _epoch_to_iso(ts):
+    try:
+        return datetime.fromtimestamp(float(ts), tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    except (TypeError, ValueError):
+        return None
+
+
+def _norm_iso(s):
+    """Best-effort normalize an arbitrary date string to our ISO 'Z' format."""
+    if not s:
+        return None
+    s = str(s).strip()
+    # already ISO-ish
+    try:
+        return datetime.fromisoformat(s.replace("Z", "+00:00")).astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    except ValueError:
+        pass
+    # RFC822 e.g. 'Sat, 11 Jul 2026 14:13:00 +000' -> normalize offset to +0000
+    import re as _re
+    s_norm = _re.sub(r"([+-]\d{2})(\d{2})$", r"\1:\2", s)   # +0000 -> +00:00
+    if _re.search(r"[+-]\d{3}$", s_norm):                    # +000 -> +0000
+        s_norm = s_norm[:-3] + "0" + s_norm[-3:]
+    for candidate in (s_norm, s):
+        for fmt in ("%a, %d %b %Y %H:%M:%S %z", "%a, %d %b %Y %H:%M:%S %Z"):
+            try:
+                return datetime.strptime(candidate, fmt).astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+            except ValueError:
+                continue
+    # Email parser fallback (most robust for RFC822)
+    try:
+        from email.utils import parsedate_to_datetime
+        d = parsedate_to_datetime(s)
+        if d is not None:
+            return d.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    except Exception:
+        pass
+    return None
+
+
+def true_first_seen(raw_meta, source, now_str):
+    """Derive the real publish date (ISO 'Z') for an entry, or harvest time."""
+    if isinstance(raw_meta, str):
+        try:
+            raw_meta = json.loads(raw_meta)
+        except (json.JSONDecodeError, TypeError):
+            raw_meta = {}
+    if not isinstance(raw_meta, dict):
+        raw_meta = {}
+    src = (source or "").lower()
+    try:
+        if src == "hackernews":
+            return _epoch_to_iso(raw_meta.get("time")) or now_str
+        if src == "reddit":
+            return _norm_iso(raw_meta.get("published")) or now_str
+        if src == "rss":
+            return _norm_iso(raw_meta.get("published")) or now_str
+        if src == "arxiv":
+            return _norm_iso(raw_meta.get("published")) or now_str
+        if src == "huggingface":
+            return _norm_iso(raw_meta.get("createdAt")) or now_str
+        if src == "github":
+            return _norm_iso(raw_meta.get("created_at") or raw_meta.get("pushed_at") or raw_meta.get("published_at")) or now_str
+    except Exception:
+        return now_str
+    return now_str
+
+
+UPSERT_SQL = """
+INSERT INTO entries
+  (source, source_id, url, title, extracted_text, summary,
+   category_tags, signal_score, raw_metadata, first_seen, last_updated)
+VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ON CONFLICT(url) DO UPDATE SET
+  source      = excluded.source,
+  source_id   = excluded.source_id,
+  title       = excluded.title,
+  extracted_text = excluded.extracted_text,
+  summary     = excluded.summary,
+  category_tags  = excluded.category_tags,
+  signal_score   = excluded.signal_score,
+  raw_metadata    = excluded.raw_metadata,
+  last_updated    = excluded.last_updated,
+  first_seen  = COALESCE((SELECT first_seen FROM entries WHERE url = excluded.url), excluded.first_seen)
+"""
+
+
+def upsert_entries(conn, entries):
+    """Idempotent store. Preserves original first_seen on re-harvest.
+
+    `entries` is the list of dicts as built by each adapter; each dict must
+    already have first_seen set to the TRUE publish date (via true_first_seen)
+    and last_updated to the harvest time.
+    Returns count of rows written.
+    """
+    cur = conn.cursor()
+    written = 0
+    for e in entries:
+        cur.execute(UPSERT_SQL, (
+            e["source"], e["source_id"], e["url"], e["title"],
+            e.get("extracted_text"), e.get("summary"),
+            e.get("category_tags"), e.get("signal_score"),
+            e.get("raw_metadata"), e["first_seen"], e["last_updated"],
+        ))
+        written += 1
+    conn.commit()
+    return written