Add Hugging Face adapter — model releases & adoption signal
- adapters/huggingface.py: HF API adapter (models + datasets) - Dual fetch: sort=likes (popularity) + sort=lastModified (fresh) - AI relevance filter: pipeline_tag, library_name, tag matching - Score: adoption (likes/downloads log-scale) + relevance (pipeline/library/tags) - score_type: actual (real likes/downloads from HF API) - Cross-source signal: GLM-5.2 top on both HN and HF - Wired into ENABLED_SOURCES + verification in pipeline.py - Live verify: DB likes match live API exactly (GLM-5.2: 3607, DeepSeek-R1: 13448) - 99 total entries across 5 sources, all pipeline green
This commit is contained in:
+23
-1
@@ -32,10 +32,11 @@ ADAPTERS = {
|
||||
"arxiv": lambda: __import__("adapters.arxiv", fromlist=["ArxivAdapter"]).ArxivAdapter(),
|
||||
"reddit": lambda: __import__("adapters.reddit", fromlist=["RedditAdapter"]).RedditAdapter(),
|
||||
"hackernews": lambda: __import__("adapters.hackernews", fromlist=["HackerNewsAdapter"]).HackerNewsAdapter(),
|
||||
"huggingface": lambda: __import__("adapters.huggingface", fromlist=["HuggingFaceAdapter"]).HuggingFaceAdapter(),
|
||||
}
|
||||
|
||||
# Default enabled sources
|
||||
ENABLED_SOURCES = ["github", "arxiv", "reddit", "hackernews"]
|
||||
ENABLED_SOURCES = ["github", "arxiv", "reddit", "hackernews", "huggingface"]
|
||||
|
||||
|
||||
def init_db(db_path: str, schema_path: str) -> sqlite3.Connection:
|
||||
@@ -166,6 +167,27 @@ def verify_entries(conn: sqlite3.Connection, source: str, sample_size: int = 3):
|
||||
except Exception as e:
|
||||
print(f" ? [{eid}] {title[:60]}... verify failed: {e}")
|
||||
|
||||
elif src == "huggingface":
|
||||
model_id = meta.get("model_id", "")
|
||||
if model_id:
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"https://huggingface.co/api/models/{model_id}",
|
||||
headers={"User-Agent": "athena/0.1"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
live = json.loads(resp.read().decode("utf-8"))
|
||||
live_likes = live.get("likes", 0)
|
||||
db_likes = meta.get("likes", 0)
|
||||
drift = abs(live_likes - db_likes)
|
||||
if drift <= 10: # HF likes update in real-time, allow small drift
|
||||
print(f" ✓ [{eid}] {title[:60]}... hf_likes={db_likes} drift={drift}")
|
||||
passed += 1
|
||||
else:
|
||||
print(f" ? [{eid}] {title[:60]}... hf_likes={db_likes} vs live={live_likes} DRIFT={drift}")
|
||||
except Exception as e:
|
||||
print(f" ? [{eid}] {title[:60]}... verify failed: {e}")
|
||||
|
||||
elif src == "reddit":
|
||||
# For Reddit, just check the URL is reachable (no easy verification of scores via RSS)
|
||||
if url:
|
||||
|
||||
Reference in New Issue
Block a user