FIX: first_seen re-stamp bug — persist true source publish date, idempotent upsert, backfill DB
This commit is contained in:
+7
-19
@@ -25,6 +25,7 @@ from datetime import datetime, timedelta, timezone
|
||||
from email.utils import parsedate_to_datetime
|
||||
|
||||
from adapters import SourceAdapter
|
||||
from adapters._store import true_first_seen, upsert_entries
|
||||
|
||||
|
||||
# Curated feed list — AI-focused, reliable, diverse publishers.
|
||||
@@ -192,6 +193,9 @@ class RSSFeedsAdapter(SourceAdapter):
|
||||
score = self._score(entry_dict, label, age_hours)
|
||||
entry_dict["score"] = score
|
||||
|
||||
now_iso = now.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
first_seen = true_first_seen(
|
||||
{"published": published}, "rss", now_iso)
|
||||
all_entries.append({
|
||||
"source": "rss",
|
||||
"source_id": f"{source_key}:{entry.get('id', entry.get('link', ''))[-30:]}",
|
||||
@@ -210,8 +214,8 @@ class RSSFeedsAdapter(SourceAdapter):
|
||||
"tags": tags,
|
||||
"score_type": "estimated",
|
||||
}),
|
||||
"first_seen": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"last_updated": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"first_seen": first_seen,
|
||||
"last_updated": now_iso,
|
||||
})
|
||||
|
||||
time.sleep(0.5) # polite spacing
|
||||
@@ -260,23 +264,7 @@ if __name__ == "__main__":
|
||||
conn.commit()
|
||||
|
||||
cur = conn.cursor()
|
||||
stored = 0
|
||||
for entry in entries:
|
||||
try:
|
||||
cur.execute("""
|
||||
INSERT OR REPLACE INTO entries
|
||||
(source, source_id, url, title, extracted_text, summary,
|
||||
category_tags, signal_score, raw_metadata, first_seen, last_updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
entry["source"], entry["source_id"], entry["url"], entry["title"],
|
||||
entry["extracted_text"], entry["summary"],
|
||||
entry["category_tags"], entry["signal_score"],
|
||||
entry["raw_metadata"], entry["first_seen"], entry["last_updated"],
|
||||
))
|
||||
stored += 1
|
||||
except Exception as e:
|
||||
print(f" DB error: {e}")
|
||||
stored = upsert_entries(conn, entries)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"\n Stored {stored} entries")
|
||||
|
||||
Reference in New Issue
Block a user