Fix GitHub adapter: sort by velocity (stars/day), not absolute stars
Bug: sort=stars returned the same viral repos every cycle (ponytail 77K⭐
dominated for 26 days). A 5-day-old repo with 3K stars (600/d) was
ranked below it despite having nearly 2x the growth velocity.
Fix:
- Fetch repos created in last 30 days (was 14)
- Re-sort by velocity (stars/day) instead of total stars
- Score formula: 70% velocity + 30% absolute stars
- Added 3rd query for breadth (autonomous agent OR AI tool)
- Added stars_per_day to raw_metadata for queryability
- Added velocity field to metadata
This commit is contained in:
+40
-13
@@ -107,14 +107,15 @@ class GitHubAdapter(SourceAdapter):
|
|||||||
return ""
|
return ""
|
||||||
|
|
||||||
def _score(self, repo: dict, age_days: float) -> float:
|
def _score(self, repo: dict, age_days: float) -> float:
|
||||||
"""Score: stars weighted by recency. Higher = better signal."""
|
"""Score: growth velocity (stars/day) on log scale. Higher = better signal."""
|
||||||
stars = repo.get("stargazers_count", 0)
|
stars = repo.get("stargazers_count", 0)
|
||||||
# Normalize: log scale on stars, decay by age
|
|
||||||
import math
|
import math
|
||||||
star_score = min(math.log1p(stars) / 2.0, 10.0) # log(1000) ≈ 6.9 → ~3.5
|
# Velocity: stars per day — the real signal
|
||||||
# Recency bonus: newer repos get a slight boost
|
velocity = stars / max(age_days, 1)
|
||||||
recency_bonus = max(0, 1.0 - age_days / 30.0) * 1.5 # up to +1.5 for very recent
|
vel_score = min(math.log1p(velocity) / 1.8, 10.0) # log(1000/d) ≈ 6.9 → ~3.8
|
||||||
return min(star_score + recency_bonus, 10.0)
|
# Absolute stars still matter (a 100K star repo is legit even if slow)
|
||||||
|
star_score = min(math.log1p(stars) / 3.0, 5.0) # max contribution: 5.0
|
||||||
|
return min(vel_score * 0.7 + star_score * 0.3, 10.0)
|
||||||
|
|
||||||
def _tags(self, repo: dict) -> list:
|
def _tags(self, repo: dict) -> list:
|
||||||
"""Generate category tags from repo metadata."""
|
"""Generate category tags from repo metadata."""
|
||||||
@@ -148,12 +149,21 @@ class GitHubAdapter(SourceAdapter):
|
|||||||
print(f" Searching: '{query}'")
|
print(f" Searching: '{query}'")
|
||||||
repos = self._search_repos(query, sort="stars", per_page=min(limit * 2, 100))
|
repos = self._search_repos(query, sort="stars", per_page=min(limit * 2, 100))
|
||||||
else:
|
else:
|
||||||
# "Trending" = AI repos created in last 14 days, sorted by stars
|
# "Trending" = repos created in last 30 days, sorted by GROWTH VELOCITY
|
||||||
cutoff = (now - timedelta(days=14)).strftime("%Y-%m-%d")
|
# (stars per day), NOT absolute stars. A 5-day-old repo with 3K stars
|
||||||
# Two queries to get breadth
|
# (600/d) is more interesting than a 25-day-old repo with 77K stars (3K/d).
|
||||||
|
#
|
||||||
|
# GitHub API sort=stars ranks total stars. We fetch by stars to get
|
||||||
|
# broad coverage, then re-sort by velocity locally.
|
||||||
|
cutoff = (now - timedelta(days=30)).strftime("%Y-%m-%d")
|
||||||
|
# Three queries for breadth: agents, LLM/infra, and security/tools
|
||||||
repos = []
|
repos = []
|
||||||
for q in ["ai agents created:>=2026-06-01", "llm inference created:>=2026-06-01"]:
|
for q in [
|
||||||
batch = self._search_repos(q, sort="stars", per_page=50)
|
f"ai agent created:>{cutoff}",
|
||||||
|
f"llm OR inference OR rag created:>{cutoff}",
|
||||||
|
f"autonomous agent OR AI tool created:>{cutoff}",
|
||||||
|
]:
|
||||||
|
batch = self._search_repos(q, sort="stars", per_page=30)
|
||||||
repos.extend(batch)
|
repos.extend(batch)
|
||||||
time.sleep(1) # polite spacing
|
time.sleep(1) # polite spacing
|
||||||
|
|
||||||
@@ -167,8 +177,22 @@ class GitHubAdapter(SourceAdapter):
|
|||||||
unique.append(r)
|
unique.append(r)
|
||||||
repos = unique
|
repos = unique
|
||||||
|
|
||||||
# Sort by stars descending, take top limit
|
# Compute velocity (stars/day) and sort by that, not total stars
|
||||||
repos.sort(key=lambda r: r.get("stargazers_count", 0), reverse=True)
|
def _velocity(repo):
|
||||||
|
created = repo.get("created_at", "")
|
||||||
|
if created:
|
||||||
|
try:
|
||||||
|
created_dt = datetime.fromisoformat(
|
||||||
|
created.replace("Z", "+00:00")
|
||||||
|
)
|
||||||
|
age_days = max((now - created_dt).days, 1)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
age_days = 1
|
||||||
|
else:
|
||||||
|
age_days = 1
|
||||||
|
return repo.get("stargazers_count", 0) / age_days
|
||||||
|
|
||||||
|
repos.sort(key=_velocity, reverse=True)
|
||||||
repos = repos[:limit]
|
repos = repos[:limit]
|
||||||
|
|
||||||
entries = []
|
entries = []
|
||||||
@@ -212,6 +236,8 @@ class GitHubAdapter(SourceAdapter):
|
|||||||
time.sleep(0.5) # polite spacing between README fetches
|
time.sleep(0.5) # polite spacing between README fetches
|
||||||
|
|
||||||
# Structured metadata
|
# Structured metadata
|
||||||
|
stars = repo.get("stargazers_count", 0)
|
||||||
|
velocity = stars / max(age_days, 1)
|
||||||
raw_meta = {
|
raw_meta = {
|
||||||
"full_name": repo.get("full_name", ""),
|
"full_name": repo.get("full_name", ""),
|
||||||
"owner": repo.get("owner", {}).get("login", ""),
|
"owner": repo.get("owner", {}).get("login", ""),
|
||||||
@@ -224,6 +250,7 @@ class GitHubAdapter(SourceAdapter):
|
|||||||
"updated_at": repo.get("updated_at", ""),
|
"updated_at": repo.get("updated_at", ""),
|
||||||
"pushed_at": repo.get("pushed_at", ""),
|
"pushed_at": repo.get("pushed_at", ""),
|
||||||
"age_days": age_days,
|
"age_days": age_days,
|
||||||
|
"stars_per_day": round(velocity, 1),
|
||||||
"readme_length": len(readme_text),
|
"readme_length": len(readme_text),
|
||||||
"score_type": "actual", # based on real star counts
|
"score_type": "actual", # based on real star counts
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user