athena-oracle/oracle/recency.py

"""Recency guard — Athena "how old is this news?" gate.

Age is the dominant gate. TODAY's items are always eligible.
Older items are eligible ONLY if never posted before.
"""
import json
import os
import re
from datetime import datetime, timezone

from oracle.config import SEEN_JSON

ORACLE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


def _parse(ts):
    if not ts:
        return None
    try:
        return datetime.fromisoformat(ts.replace("Z", "+00:00"))
    except Exception:
        return None


def _norm_url(u):
    if not u:
        return ""
    return u.split("?")[0].split("#")[0].rstrip("/").lower()


def _norm_title(t):
    if not t:
        return ""
    t = t.lower()
    t = re.sub(r"[^a-z0-9 ]", " ", t)
    return re.sub(r"\s+", " ", t).strip()[:60]


def load_posted(md_dir: str = ORACLE_DIR, seen_json: str = str(SEEN_JSON)):
    """Return (md_urls:set, md_titles:set, seen_urls:set)."""
    md_urls, md_titles, seen_urls = set(), set(), set()

    for fn in sorted(os.listdir(md_dir)):
        if re.match(r"athena_top.*\.md$", fn):
            try:
                txt = open(os.path.join(md_dir, fn), encoding="utf-8", errors="replace").read()
            except OSError:
                continue
            for m in re.findall(r"\]\((https?://[^)\s]+)\)", txt):
                nu = _norm_url(m)
                if nu:
                    md_urls.add(nu)
            for t in re.findall(r"^\|\s*\d+\s*\|\s*(.+?)\s*\|", txt, re.M):
                nt = _norm_title(t)
                if nt:
                    md_titles.add(nt)

    if os.path.exists(seen_json):
        try:
            with open(seen_json, encoding="utf-8") as f:
                for u in json.load(f):
                    nu = _norm_url(u)
                    if nu:
                        seen_urls.add(nu)
        except (json.JSONDecodeError, OSError):
            pass

    return md_urls, md_titles, seen_urls


def is_today(first_seen, now=None):
    now = now or datetime.now(timezone.utc)
    d = _parse(first_seen)
    return bool(d) and d.strftime("%Y-%m-%d") == now.strftime("%Y-%m-%d")


def age_days(first_seen, now=None):
    now = now or datetime.now(timezone.utc)
    d = _parse(first_seen)
    if not d:
        return 9999.0
    return max((now - d).total_seconds() / 86400.0, 0.0)


def day_bucket(first_seen, now=None):
    days = age_days(first_seen, now)
    if days < 1:
        return "today"
    if days < 2:
        return "yesterday"
    if days <= 6:
        return "this-week"
    return "older"


def already_posted(url, title, first_seen, now=None,
                   md_urls=None, md_titles=None, seen_urls=None):
    """Age-aware dedup. Today's items are NEVER flagged."""
    if md_urls is None or md_titles is None or seen_urls is None:
        md_urls, md_titles, seen_urls = load_posted()
    if _norm_url(url) in md_urls:
        return True
    nt = _norm_title(title)
    if nt and nt in md_titles:
        return True
    if is_today(first_seen, now):
        return False
    if _norm_url(url) in seen_urls:
        return True
    return False


def recency_weight(first_seen, now=None, half_life_days=2.0):
    """1.0 for today, decays ~halving every 2 days."""
    return 0.5 ** (age_days(first_seen, now) / half_life_days)


def blend_score(item, now=None):
    """clickability_decayed * recency_weight."""
    base = item.get("clickability_decayed", 0) or 0
    return base * recency_weight(item.get("first_seen"), now)


def filter_fresh(items, now=None, recent_window_days=4.0):
    """Split into (today_items, older_new_items, dropped_items)."""
    now = now or datetime.now(timezone.utc)
    md_urls, md_titles, seen_urls = load_posted()
    today_items, older_new, dropped = [], [], []
    for it in items:
        fs = it.get("first_seen")
        if is_today(fs, now):
            today_items.append(it)
            continue
        posted = already_posted(it.get("url"), it.get("title"), fs, now,
                                md_urls, md_titles, seen_urls)
        if posted and age_days(fs, now) > recent_window_days:
            dropped.append(it)
        else:
            older_new.append(it)
    return today_items, older_new, dropped