Sprint 0+1: Package restructure, source tiers, verdicts, multi-variant editions
- New oracle/ package (11 modules) with unified CLI (python -m oracle) - Source tiers: Tier 1 (arxiv/github/hf), Tier 2 (rss/hn), Tier 3 (reddit) - Composite verdicts: PUBLISH/WATCH/ARCHIVE/DROP based on signal score + age - Content-hash dedup: SHA-256[:16] normalized, atomic at insert time - Multi-variant editions: 4 YAML configs (default/research/devops/brief) - Variant engine: filter → rank → render (HTML + JSON, themed) - Per-adapter timeout (10s) + threading fallback - Consolidated 12 root scripts → thin wrappers + oracle/ package - Archived stale scripts (_engagement, _live_compare, reddit_proof) - Updated .gitignore, README.md, schema.sql
This commit is contained in:
+28
-2
@@ -89,8 +89,9 @@ def true_first_seen(raw_meta, source, now_str):
|
||||
UPSERT_SQL = """
|
||||
INSERT INTO entries
|
||||
(source, source_id, url, title, extracted_text, summary,
|
||||
category_tags, signal_score, raw_metadata, first_seen, last_updated)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
category_tags, signal_score, raw_metadata, first_seen, last_updated,
|
||||
content_hash, source_tier, verdict)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(source, source_id) DO UPDATE SET
|
||||
source = excluded.source,
|
||||
source_id = excluded.source_id,
|
||||
@@ -101,6 +102,9 @@ ON CONFLICT(source, source_id) DO UPDATE SET
|
||||
signal_score = excluded.signal_score,
|
||||
raw_metadata = excluded.raw_metadata,
|
||||
last_updated = excluded.last_updated,
|
||||
content_hash = excluded.content_hash,
|
||||
source_tier = excluded.source_tier,
|
||||
verdict = excluded.verdict,
|
||||
first_seen = COALESCE((SELECT first_seen FROM entries WHERE source = excluded.source AND source_id = excluded.source_id), excluded.first_seen)
|
||||
"""
|
||||
|
||||
@@ -111,16 +115,38 @@ def upsert_entries(conn, entries):
|
||||
`entries` is the list of dicts as built by each adapter; each dict must
|
||||
already have first_seen set to the TRUE publish date (via true_first_seen)
|
||||
and last_updated to the harvest time.
|
||||
|
||||
Now also sets content_hash and source_tier at insertion time.
|
||||
Returns count of rows written.
|
||||
"""
|
||||
from oracle.dedup import content_hash, get_source_tier, compute_verdict, age_hours
|
||||
|
||||
cur = conn.cursor()
|
||||
written = 0
|
||||
for e in entries:
|
||||
# Compute content hash
|
||||
title = e.get("title", "")
|
||||
url = e.get("url", "")
|
||||
body = e.get("extracted_text", "")[:500]
|
||||
h = content_hash(title, url, body)
|
||||
|
||||
# Get source tier
|
||||
source = e.get("source", "")
|
||||
tier_info = get_source_tier(source)
|
||||
tier = tier_info["tier"]
|
||||
|
||||
# Compute verdict
|
||||
first_seen = e.get("first_seen", "")
|
||||
score = float(e.get("signal_score") or 0)
|
||||
age = age_hours(first_seen)
|
||||
verdict = compute_verdict(score, age)
|
||||
|
||||
cur.execute(UPSERT_SQL, (
|
||||
e["source"], e["source_id"], e["url"], e["title"],
|
||||
e.get("extracted_text"), e.get("summary"),
|
||||
e.get("category_tags"), e.get("signal_score"),
|
||||
e.get("raw_metadata"), e["first_seen"], e["last_updated"],
|
||||
h, tier, verdict,
|
||||
))
|
||||
written += 1
|
||||
conn.commit()
|
||||
|
||||
Reference in New Issue
Block a user