#!/usr/bin/env python3 """ recency_guard.py — Athena "how old is this news?" gate (Tony's correction, 2026-07-13). THE PROBLEM IT SOLVES: The old pipeline ranked by virality + editorial fit ONLY. That surfaced the SAME stories week after week (Apple-vs-OpenAI, GPT-5.6, etc.) because the curation had NO memory of what was already posted. Tony's rule: "when you receive the news stories, morality [editorial fit] is only ONE measure. The OTHER measure -- maybe MORE important -- is how OLD is the news. Today is Monday; the week's news is just beginning, so we begin the week with stories dated for TODAY." So AGE is the dominant gate. The algorithm (editorial, not render-time): * TODAY's items -> ALWAYS eligible. They are this week's fresh news; a Monday stack leads with them even if rendered earlier today. * OLDER items -> eligible ONLY if NEVER posted before (not in the markdown Top-N stack history AND not in seen_urls.json). This kills the re-post problem at the source. WHAT IS "ALREADY POSTED": Two signals, differing in authority: - athena_top*.md = the CURATED/PUBLIC stack history (authoritative) - seen_urls.json = the live-site render dedup (secondary; ALSO flags items rendered in prior runs TODAY, which we must NOT drop) Because seen_urls.json contains today's own items, we only treat a seen_url as "already posted" when the candidate is OLDER than today. Today's items are exempt from the seen_urls gate entirely (fresh by definition). EXPORTS: load_posted() -> (md_urls, md_titles, seen_urls) sets is_today(first_seen, now) age_days(first_seen, now) day_bucket(first_seen, now) already_posted_fs(url,title,fs,now) -> bool (age-aware dedup) filter_fresh(items, now) -> (today_items, older_new_items, dropped_items) recency_weight(first_seen, now) -> float (1.0 today -> ~0 over 7d) blend_score(item, now) -> clickability_decayed * recency_weight Read-only against markdown + json + passed-in items. No DB writes. """ import os, re, json from datetime import datetime, timezone ORACLE = os.path.dirname(os.path.abspath(__file__)) SEEN_JSON = "/home/vpsadmin/ai-oracle-site/seen_urls.json" def _parse(ts): if not ts: return None try: return datetime.fromisoformat(ts.replace("Z", "+00:00")) except Exception: return None def _norm_url(u): if not u: return "" return u.split("?")[0].split("#")[0].rstrip("/").lower() def _norm_title(t): if not t: return "" t = t.lower() t = re.sub(r"[^a-z0-9 ]", " ", t) t = re.sub(r"\s+", " ", t).strip() return t[:60] def load_posted(md_dir=ORACLE, seen_json=SEEN_JSON): """Return (md_urls:set, md_titles:set, seen_urls:set).""" md_urls, md_titles, seen_urls = set(), set(), set() for fn in sorted(os.listdir(md_dir)): if re.match(r"athena_top.*\.md$", fn): try: txt = open(os.path.join(md_dir, fn), encoding="utf-8", errors="replace").read() except OSError: continue for m in re.findall(r"\]\((https?://[^)\s]+)\)", txt): nu = _norm_url(m) if nu: md_urls.add(nu) for t in re.findall(r"^\|\s*\d+\s*\|\s*(.+?)\s*\|", txt, re.M): nt = _norm_title(t) if nt: md_titles.add(nt) if os.path.exists(seen_json): try: with open(seen_json, encoding="utf-8") as f: for u in json.load(f): nu = _norm_url(u) if nu: seen_urls.add(nu) except (json.JSONDecodeError, OSError): pass return md_urls, md_titles, seen_urls def is_today(first_seen, now=None): now = now or datetime.now(timezone.utc) d = _parse(first_seen) return bool(d) and d.strftime("%Y-%m-%d") == now.strftime("%Y-%m-%d") def age_days(first_seen, now=None): now = now or datetime.now(timezone.utc) d = _parse(first_seen) if not d: return 9999.0 return max((now - d).total_seconds() / 86400.0, 0.0) def day_bucket(first_seen, now=None): """'today' | 'yesterday' | 'this-week' (<=6d) | 'older'.""" days = age_days(first_seen, now) if days < 1: return "today" if days < 2: return "yesterday" if days <= 6: return "this-week" return "older" def already_posted_fs(url, title, first_seen, now=None, md_urls=None, md_titles=None, seen_urls=None): """Age-aware dedup. A candidate is 'already posted' iff: (a) it matches the curated md-stack history, OR (b) it is OLDER than today AND its URL is in seen_urls.json. Today's items are NEVER flagged -- they are fresh by definition. """ if md_urls is None or md_titles is None or seen_urls is None: md_urls, md_titles, seen_urls = load_posted() if _norm_url(url) in md_urls: return True nt = _norm_title(title) if nt and nt in md_titles: return True if is_today(first_seen, now): return False if _norm_url(url) in seen_urls: return True return False def recency_weight(first_seen, now=None, half_life_days=2.0): """1.0 for today, decays ~halving every 2 days. The 'age' measure.""" return 0.5 ** (age_days(first_seen, now) / half_life_days) def blend_score(item, now=None): """clickability_decayed * recency_weight. Today's items dominate; old sink.""" base = item.get("clickability_decayed", 0) or 0 return base * recency_weight(item.get("first_seen"), now) def filter_fresh(items, now=None): """Split into (today_items, older_new_items, dropped_items). today_items = first_seen == today (always eligible; the week-open lead) older_new_items= older, but never before posted (md/seen) dropped_items = older AND already posted (the re-posts we are killing) """ now = now or datetime.now(timezone.utc) md_urls, md_titles, seen_urls = load_posted() today_items, older_new, dropped = [], [], [] for it in items: fs = it.get("first_seen") if is_today(fs, now): today_items.append(it) continue if already_posted_fs(it.get("url"), it.get("title"), fs, now, md_urls, md_titles, seen_urls): dropped.append(it) else: older_new.append(it) return today_items, older_new, dropped if __name__ == "__main__": import sys sys.path.insert(0, ORACLE) import clickability as cb DB = os.path.join(ORACLE, "oracle.db") conn = sqlite3.connect(f"file:{DB}?mode=ro", uri=True) items = cb.fetch_items(conn); conn.close() items = cb.compute_index(items); items = cb.decay_index(items) today_items, older_new, dropped = filter_fresh(items) print(f"TODAY-new (week-open lead): {len(today_items)}") print(f"OLDER-but-never-posted: {len(older_new)}") print(f"DROPPED (already posted): {len(dropped)}") print("\nSample dropped (the re-posts that caused the problem):") for it in sorted(dropped, key=lambda x: -x["clickability_decayed"])[:6]: print(f" - [{it['clickability_decayed']:.3f}] {it['title'][:66]}")