"""Recency guard — Athena "how old is this news?" gate. Age is the dominant gate. TODAY's items are always eligible. Older items are eligible ONLY if never posted before. """ import json import os import re from datetime import datetime, timezone from oracle.config import SEEN_JSON ORACLE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) def _parse(ts): if not ts: return None try: return datetime.fromisoformat(ts.replace("Z", "+00:00")) except Exception: return None def _norm_url(u): if not u: return "" return u.split("?")[0].split("#")[0].rstrip("/").lower() def _norm_title(t): if not t: return "" t = t.lower() t = re.sub(r"[^a-z0-9 ]", " ", t) return re.sub(r"\s+", " ", t).strip()[:60] def load_posted(md_dir: str = ORACLE_DIR, seen_json: str = str(SEEN_JSON)): """Return (md_urls:set, md_titles:set, seen_urls:set).""" md_urls, md_titles, seen_urls = set(), set(), set() for fn in sorted(os.listdir(md_dir)): if re.match(r"athena_top.*\.md$", fn): try: txt = open(os.path.join(md_dir, fn), encoding="utf-8", errors="replace").read() except OSError: continue for m in re.findall(r"\]\((https?://[^)\s]+)\)", txt): nu = _norm_url(m) if nu: md_urls.add(nu) for t in re.findall(r"^\|\s*\d+\s*\|\s*(.+?)\s*\|", txt, re.M): nt = _norm_title(t) if nt: md_titles.add(nt) if os.path.exists(seen_json): try: with open(seen_json, encoding="utf-8") as f: for u in json.load(f): nu = _norm_url(u) if nu: seen_urls.add(nu) except (json.JSONDecodeError, OSError): pass return md_urls, md_titles, seen_urls def is_today(first_seen, now=None): now = now or datetime.now(timezone.utc) d = _parse(first_seen) return bool(d) and d.strftime("%Y-%m-%d") == now.strftime("%Y-%m-%d") def age_days(first_seen, now=None): now = now or datetime.now(timezone.utc) d = _parse(first_seen) if not d: return 9999.0 return max((now - d).total_seconds() / 86400.0, 0.0) def day_bucket(first_seen, now=None): days = age_days(first_seen, now) if days < 1: return "today" if days < 2: return "yesterday" if days <= 6: return "this-week" return "older" def already_posted(url, title, first_seen, now=None, md_urls=None, md_titles=None, seen_urls=None): """Age-aware dedup. Today's items are NEVER flagged.""" if md_urls is None or md_titles is None or seen_urls is None: md_urls, md_titles, seen_urls = load_posted() if _norm_url(url) in md_urls: return True nt = _norm_title(title) if nt and nt in md_titles: return True if is_today(first_seen, now): return False if _norm_url(url) in seen_urls: return True return False def recency_weight(first_seen, now=None, half_life_days=2.0): """1.0 for today, decays ~halving every 2 days.""" return 0.5 ** (age_days(first_seen, now) / half_life_days) def blend_score(item, now=None): """clickability_decayed * recency_weight.""" base = item.get("clickability_decayed", 0) or 0 return base * recency_weight(item.get("first_seen"), now) def filter_fresh(items, now=None, recent_window_days=4.0): """Split into (today_items, older_new_items, dropped_items).""" now = now or datetime.now(timezone.utc) md_urls, md_titles, seen_urls = load_posted() today_items, older_new, dropped = [], [], [] for it in items: fs = it.get("first_seen") if is_today(fs, now): today_items.append(it) continue posted = already_posted(it.get("url"), it.get("title"), fs, now, md_urls, md_titles, seen_urls) if posted and age_days(fs, now) > recent_window_days: dropped.append(it) else: older_new.append(it) return today_items, older_new, dropped