Fix GitHub adapter: sort by velocity (stars/day), not absolute stars
Bug: sort=stars returned the same viral repos every cycle (ponytail 77K⭐
dominated for 26 days). A 5-day-old repo with 3K stars (600/d) was
ranked below it despite having nearly 2x the growth velocity.
Fix:
- Fetch repos created in last 30 days (was 14)
- Re-sort by velocity (stars/day) instead of total stars
- Score formula: 70% velocity + 30% absolute stars
- Added 3rd query for breadth (autonomous agent OR AI tool)
- Added stars_per_day to raw_metadata for queryability
- Added velocity field to metadata
This commit is contained in:
+40
-13
@@ -107,14 +107,15 @@ class GitHubAdapter(SourceAdapter):
|
||||
return ""
|
||||
|
||||
def _score(self, repo: dict, age_days: float) -> float:
|
||||
"""Score: stars weighted by recency. Higher = better signal."""
|
||||
"""Score: growth velocity (stars/day) on log scale. Higher = better signal."""
|
||||
stars = repo.get("stargazers_count", 0)
|
||||
# Normalize: log scale on stars, decay by age
|
||||
import math
|
||||
star_score = min(math.log1p(stars) / 2.0, 10.0) # log(1000) ≈ 6.9 → ~3.5
|
||||
# Recency bonus: newer repos get a slight boost
|
||||
recency_bonus = max(0, 1.0 - age_days / 30.0) * 1.5 # up to +1.5 for very recent
|
||||
return min(star_score + recency_bonus, 10.0)
|
||||
# Velocity: stars per day — the real signal
|
||||
velocity = stars / max(age_days, 1)
|
||||
vel_score = min(math.log1p(velocity) / 1.8, 10.0) # log(1000/d) ≈ 6.9 → ~3.8
|
||||
# Absolute stars still matter (a 100K star repo is legit even if slow)
|
||||
star_score = min(math.log1p(stars) / 3.0, 5.0) # max contribution: 5.0
|
||||
return min(vel_score * 0.7 + star_score * 0.3, 10.0)
|
||||
|
||||
def _tags(self, repo: dict) -> list:
|
||||
"""Generate category tags from repo metadata."""
|
||||
@@ -148,12 +149,21 @@ class GitHubAdapter(SourceAdapter):
|
||||
print(f" Searching: '{query}'")
|
||||
repos = self._search_repos(query, sort="stars", per_page=min(limit * 2, 100))
|
||||
else:
|
||||
# "Trending" = AI repos created in last 14 days, sorted by stars
|
||||
cutoff = (now - timedelta(days=14)).strftime("%Y-%m-%d")
|
||||
# Two queries to get breadth
|
||||
# "Trending" = repos created in last 30 days, sorted by GROWTH VELOCITY
|
||||
# (stars per day), NOT absolute stars. A 5-day-old repo with 3K stars
|
||||
# (600/d) is more interesting than a 25-day-old repo with 77K stars (3K/d).
|
||||
#
|
||||
# GitHub API sort=stars ranks total stars. We fetch by stars to get
|
||||
# broad coverage, then re-sort by velocity locally.
|
||||
cutoff = (now - timedelta(days=30)).strftime("%Y-%m-%d")
|
||||
# Three queries for breadth: agents, LLM/infra, and security/tools
|
||||
repos = []
|
||||
for q in ["ai agents created:>=2026-06-01", "llm inference created:>=2026-06-01"]:
|
||||
batch = self._search_repos(q, sort="stars", per_page=50)
|
||||
for q in [
|
||||
f"ai agent created:>{cutoff}",
|
||||
f"llm OR inference OR rag created:>{cutoff}",
|
||||
f"autonomous agent OR AI tool created:>{cutoff}",
|
||||
]:
|
||||
batch = self._search_repos(q, sort="stars", per_page=30)
|
||||
repos.extend(batch)
|
||||
time.sleep(1) # polite spacing
|
||||
|
||||
@@ -167,8 +177,22 @@ class GitHubAdapter(SourceAdapter):
|
||||
unique.append(r)
|
||||
repos = unique
|
||||
|
||||
# Sort by stars descending, take top limit
|
||||
repos.sort(key=lambda r: r.get("stargazers_count", 0), reverse=True)
|
||||
# Compute velocity (stars/day) and sort by that, not total stars
|
||||
def _velocity(repo):
|
||||
created = repo.get("created_at", "")
|
||||
if created:
|
||||
try:
|
||||
created_dt = datetime.fromisoformat(
|
||||
created.replace("Z", "+00:00")
|
||||
)
|
||||
age_days = max((now - created_dt).days, 1)
|
||||
except (ValueError, TypeError):
|
||||
age_days = 1
|
||||
else:
|
||||
age_days = 1
|
||||
return repo.get("stargazers_count", 0) / age_days
|
||||
|
||||
repos.sort(key=_velocity, reverse=True)
|
||||
repos = repos[:limit]
|
||||
|
||||
entries = []
|
||||
@@ -212,6 +236,8 @@ class GitHubAdapter(SourceAdapter):
|
||||
time.sleep(0.5) # polite spacing between README fetches
|
||||
|
||||
# Structured metadata
|
||||
stars = repo.get("stargazers_count", 0)
|
||||
velocity = stars / max(age_days, 1)
|
||||
raw_meta = {
|
||||
"full_name": repo.get("full_name", ""),
|
||||
"owner": repo.get("owner", {}).get("login", ""),
|
||||
@@ -224,6 +250,7 @@ class GitHubAdapter(SourceAdapter):
|
||||
"updated_at": repo.get("updated_at", ""),
|
||||
"pushed_at": repo.get("pushed_at", ""),
|
||||
"age_days": age_days,
|
||||
"stars_per_day": round(velocity, 1),
|
||||
"readme_length": len(readme_text),
|
||||
"score_type": "actual", # based on real star counts
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user