Add Hugging Face adapter — model releases & adoption signal

- adapters/huggingface.py: HF API adapter (models + datasets)
- Dual fetch: sort=likes (popularity) + sort=lastModified (fresh)
- AI relevance filter: pipeline_tag, library_name, tag matching
- Score: adoption (likes/downloads log-scale) + relevance (pipeline/library/tags)
- score_type: actual (real likes/downloads from HF API)
- Cross-source signal: GLM-5.2 top on both HN and HF
- Wired into ENABLED_SOURCES + verification in pipeline.py
- Live verify: DB likes match live API exactly (GLM-5.2: 3607, DeepSeek-R1: 13448)
- 99 total entries across 5 sources, all pipeline green
This commit is contained in:
Epictetus
2026-07-08 05:54:27 +00:00
parent 6396b66b45
commit 4ba270c166
2 changed files with 430 additions and 1 deletions
+23 -1
View File
@@ -32,10 +32,11 @@ ADAPTERS = {
"arxiv": lambda: __import__("adapters.arxiv", fromlist=["ArxivAdapter"]).ArxivAdapter(),
"reddit": lambda: __import__("adapters.reddit", fromlist=["RedditAdapter"]).RedditAdapter(),
"hackernews": lambda: __import__("adapters.hackernews", fromlist=["HackerNewsAdapter"]).HackerNewsAdapter(),
"huggingface": lambda: __import__("adapters.huggingface", fromlist=["HuggingFaceAdapter"]).HuggingFaceAdapter(),
}
# Default enabled sources
ENABLED_SOURCES = ["github", "arxiv", "reddit", "hackernews"]
ENABLED_SOURCES = ["github", "arxiv", "reddit", "hackernews", "huggingface"]
def init_db(db_path: str, schema_path: str) -> sqlite3.Connection:
@@ -166,6 +167,27 @@ def verify_entries(conn: sqlite3.Connection, source: str, sample_size: int = 3):
except Exception as e:
print(f" ? [{eid}] {title[:60]}... verify failed: {e}")
elif src == "huggingface":
model_id = meta.get("model_id", "")
if model_id:
try:
req = urllib.request.Request(
f"https://huggingface.co/api/models/{model_id}",
headers={"User-Agent": "athena/0.1"},
)
with urllib.request.urlopen(req, timeout=10) as resp:
live = json.loads(resp.read().decode("utf-8"))
live_likes = live.get("likes", 0)
db_likes = meta.get("likes", 0)
drift = abs(live_likes - db_likes)
if drift <= 10: # HF likes update in real-time, allow small drift
print(f" ✓ [{eid}] {title[:60]}... hf_likes={db_likes} drift={drift}")
passed += 1
else:
print(f" ? [{eid}] {title[:60]}... hf_likes={db_likes} vs live={live_likes} DRIFT={drift}")
except Exception as e:
print(f" ? [{eid}] {title[:60]}... verify failed: {e}")
elif src == "reddit":
# For Reddit, just check the URL is reachable (no easy verification of scores via RSS)
if url: