athena-oracle/adapters/_store.py

"""Shared DB storage helper for Athena adapters.

Fixes the first_seen re-stamp bug (2026-07-13, Tony):
  - Previously every adapter did INSERT OR REPLACE with first_seen = harvest
    time, so re-harvesting an existing URL OVERWROTE the true publish date
    with today's date. The recency guard then believed stale stories were new.
  - true_first_seen() derives the REAL publish date from raw_metadata:
        hackernews -> raw_meta['time']      (unix epoch)
        reddit     -> raw_meta['published'] (ISO)
        rss        -> raw_meta['published'] (RFC822 / ISO)
        arxiv      -> raw_meta['published'] (ISO)
        huggingface-> raw_meta['createdAt'] (ISO)
    Falls back to harvest time only if no source date exists.
  - upsert_entries() is idempotent: first sight stores the true first_seen;
    re-encounter PRESERVES the original first_seen and only bumps last_updated.
"""
import json
import sqlite3
from datetime import datetime, timezone


def _epoch_to_iso(ts):
    try:
        return datetime.fromtimestamp(float(ts), tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
    except (TypeError, ValueError):
        return None


def _norm_iso(s):
    """Best-effort normalize an arbitrary date string to our ISO 'Z' format."""
    if not s:
        return None
    s = str(s).strip()
    # already ISO-ish
    try:
        return datetime.fromisoformat(s.replace("Z", "+00:00")).astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
    except ValueError:
        pass
    # RFC822 e.g. 'Sat, 11 Jul 2026 14:13:00 +000' -> normalize offset to +0000
    import re as _re
    s_norm = _re.sub(r"([+-]\d{2})(\d{2})$", r"\1:\2", s)   # +0000 -> +00:00
    if _re.search(r"[+-]\d{3}$", s_norm):                    # +000 -> +0000
        s_norm = s_norm[:-3] + "0" + s_norm[-3:]
    for candidate in (s_norm, s):
        for fmt in ("%a, %d %b %Y %H:%M:%S %z", "%a, %d %b %Y %H:%M:%S %Z"):
            try:
                return datetime.strptime(candidate, fmt).astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
            except ValueError:
                continue
    # Email parser fallback (most robust for RFC822)
    try:
        from email.utils import parsedate_to_datetime
        d = parsedate_to_datetime(s)
        if d is not None:
            return d.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
    except Exception:
        pass
    return None


def true_first_seen(raw_meta, source, now_str):
    """Derive the real publish date (ISO 'Z') for an entry, or harvest time."""
    if isinstance(raw_meta, str):
        try:
            raw_meta = json.loads(raw_meta)
        except (json.JSONDecodeError, TypeError):
            raw_meta = {}
    if not isinstance(raw_meta, dict):
        raw_meta = {}
    src = (source or "").lower()
    try:
        if src == "hackernews":
            return _epoch_to_iso(raw_meta.get("time")) or now_str
        if src == "reddit":
            return _norm_iso(raw_meta.get("published")) or now_str
        if src == "rss":
            return _norm_iso(raw_meta.get("published")) or now_str
        if src == "arxiv":
            return _norm_iso(raw_meta.get("published")) or now_str
        if src == "huggingface":
            return _norm_iso(raw_meta.get("createdAt")) or now_str
        if src == "github":
            return _norm_iso(raw_meta.get("created_at") or raw_meta.get("pushed_at") or raw_meta.get("published_at")) or now_str
    except Exception:
        return now_str
    return now_str


UPSERT_SQL = """
INSERT INTO entries
  (source, source_id, url, title, extracted_text, summary,
   category_tags, signal_score, raw_metadata, first_seen, last_updated,
   content_hash, source_tier, verdict)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(source, source_id) DO UPDATE SET
  source      = excluded.source,
  source_id   = excluded.source_id,
  title       = excluded.title,
  extracted_text = excluded.extracted_text,
  summary     = excluded.summary,
  category_tags  = excluded.category_tags,
  signal_score   = excluded.signal_score,
  raw_metadata    = excluded.raw_metadata,
  last_updated    = excluded.last_updated,
  content_hash    = excluded.content_hash,
  source_tier     = excluded.source_tier,
  verdict         = excluded.verdict,
  first_seen  = COALESCE((SELECT first_seen FROM entries WHERE source = excluded.source AND source_id = excluded.source_id), excluded.first_seen)
"""


def upsert_entries(conn, entries):
    """Idempotent store. Preserves original first_seen on re-harvest.

    `entries` is the list of dicts as built by each adapter; each dict must
    already have first_seen set to the TRUE publish date (via true_first_seen)
    and last_updated to the harvest time.

    Now also sets content_hash and source_tier at insertion time.
    Returns count of rows written.
    """
    from oracle.dedup import content_hash, get_source_tier, compute_verdict, age_hours

    cur = conn.cursor()
    written = 0
    for e in entries:
        # Compute content hash
        title = e.get("title", "")
        url = e.get("url", "")
        body = e.get("extracted_text", "")[:500]
        h = content_hash(title, url, body)

        # Get source tier
        source = e.get("source", "")
        tier_info = get_source_tier(source)
        tier = tier_info["tier"]

        # Compute verdict
        first_seen = e.get("first_seen", "")
        score = float(e.get("signal_score") or 0)
        age = age_hours(first_seen)
        verdict = compute_verdict(score, age)

        cur.execute(UPSERT_SQL, (
            e["source"], e["source_id"], e["url"], e["title"],
            e.get("extracted_text"), e.get("summary"),
            e.get("category_tags"), e.get("signal_score"),
            e.get("raw_metadata"), e["first_seen"], e["last_updated"],
            h, tier, verdict,
        ))
        written += 1
    conn.commit()
    return written