FIX: first_seen re-stamp bug — persist true source publish date, idempotent upsert, backfill DB

This commit is contained in:
Epictetus
2026-07-13 22:16:59 +00:00
parent 7e33ffd36f
commit 424da91364
8 changed files with 167 additions and 138 deletions
+7 -19
View File
@@ -25,6 +25,7 @@ from datetime import datetime, timedelta, timezone
from email.utils import parsedate_to_datetime
from adapters import SourceAdapter
from adapters._store import true_first_seen, upsert_entries
# Curated feed list — AI-focused, reliable, diverse publishers.
@@ -192,6 +193,9 @@ class RSSFeedsAdapter(SourceAdapter):
score = self._score(entry_dict, label, age_hours)
entry_dict["score"] = score
now_iso = now.strftime("%Y-%m-%dT%H:%M:%SZ")
first_seen = true_first_seen(
{"published": published}, "rss", now_iso)
all_entries.append({
"source": "rss",
"source_id": f"{source_key}:{entry.get('id', entry.get('link', ''))[-30:]}",
@@ -210,8 +214,8 @@ class RSSFeedsAdapter(SourceAdapter):
"tags": tags,
"score_type": "estimated",
}),
"first_seen": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
"last_updated": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
"first_seen": first_seen,
"last_updated": now_iso,
})
time.sleep(0.5) # polite spacing
@@ -260,23 +264,7 @@ if __name__ == "__main__":
conn.commit()
cur = conn.cursor()
stored = 0
for entry in entries:
try:
cur.execute("""
INSERT OR REPLACE INTO entries
(source, source_id, url, title, extracted_text, summary,
category_tags, signal_score, raw_metadata, first_seen, last_updated)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (
entry["source"], entry["source_id"], entry["url"], entry["title"],
entry["extracted_text"], entry["summary"],
entry["category_tags"], entry["signal_score"],
entry["raw_metadata"], entry["first_seen"], entry["last_updated"],
))
stored += 1
except Exception as e:
print(f" DB error: {e}")
stored = upsert_entries(conn, entries)
conn.commit()
conn.close()
print(f"\n Stored {stored} entries")