Fix GitHub adapter: sort by velocity (stars/day), not absolute stars

Bug: sort=stars returned the same viral repos every cycle (ponytail 77K
dominated for 26 days). A 5-day-old repo with 3K stars (600/d) was
ranked below it despite having nearly 2x the growth velocity.

Fix:
- Fetch repos created in last 30 days (was 14)
- Re-sort by velocity (stars/day) instead of total stars
- Score formula: 70% velocity + 30% absolute stars
- Added 3rd query for breadth (autonomous agent OR AI tool)
- Added stars_per_day to raw_metadata for queryability
- Added velocity field to metadata
This commit is contained in:
Epictetus
2026-07-08 06:46:40 +00:00
parent 8d0c832e20
commit 62031bef3f
+41 -14
View File
@@ -107,14 +107,15 @@ class GitHubAdapter(SourceAdapter):
return "" return ""
def _score(self, repo: dict, age_days: float) -> float: def _score(self, repo: dict, age_days: float) -> float:
"""Score: stars weighted by recency. Higher = better signal.""" """Score: growth velocity (stars/day) on log scale. Higher = better signal."""
stars = repo.get("stargazers_count", 0) stars = repo.get("stargazers_count", 0)
# Normalize: log scale on stars, decay by age
import math import math
star_score = min(math.log1p(stars) / 2.0, 10.0) # log(1000) ≈ 6.9 → ~3.5 # Velocity: stars per day — the real signal
# Recency bonus: newer repos get a slight boost velocity = stars / max(age_days, 1)
recency_bonus = max(0, 1.0 - age_days / 30.0) * 1.5 # up to +1.5 for very recent vel_score = min(math.log1p(velocity) / 1.8, 10.0) # log(1000/d) ≈ 6.9 → ~3.8
return min(star_score + recency_bonus, 10.0) # Absolute stars still matter (a 100K star repo is legit even if slow)
star_score = min(math.log1p(stars) / 3.0, 5.0) # max contribution: 5.0
return min(vel_score * 0.7 + star_score * 0.3, 10.0)
def _tags(self, repo: dict) -> list: def _tags(self, repo: dict) -> list:
"""Generate category tags from repo metadata.""" """Generate category tags from repo metadata."""
@@ -148,12 +149,21 @@ class GitHubAdapter(SourceAdapter):
print(f" Searching: '{query}'") print(f" Searching: '{query}'")
repos = self._search_repos(query, sort="stars", per_page=min(limit * 2, 100)) repos = self._search_repos(query, sort="stars", per_page=min(limit * 2, 100))
else: else:
# "Trending" = AI repos created in last 14 days, sorted by stars # "Trending" = repos created in last 30 days, sorted by GROWTH VELOCITY
cutoff = (now - timedelta(days=14)).strftime("%Y-%m-%d") # (stars per day), NOT absolute stars. A 5-day-old repo with 3K stars
# Two queries to get breadth # (600/d) is more interesting than a 25-day-old repo with 77K stars (3K/d).
#
# GitHub API sort=stars ranks total stars. We fetch by stars to get
# broad coverage, then re-sort by velocity locally.
cutoff = (now - timedelta(days=30)).strftime("%Y-%m-%d")
# Three queries for breadth: agents, LLM/infra, and security/tools
repos = [] repos = []
for q in ["ai agents created:>=2026-06-01", "llm inference created:>=2026-06-01"]: for q in [
batch = self._search_repos(q, sort="stars", per_page=50) f"ai agent created:>{cutoff}",
f"llm OR inference OR rag created:>{cutoff}",
f"autonomous agent OR AI tool created:>{cutoff}",
]:
batch = self._search_repos(q, sort="stars", per_page=30)
repos.extend(batch) repos.extend(batch)
time.sleep(1) # polite spacing time.sleep(1) # polite spacing
@@ -167,9 +177,23 @@ class GitHubAdapter(SourceAdapter):
unique.append(r) unique.append(r)
repos = unique repos = unique
# Sort by stars descending, take top limit # Compute velocity (stars/day) and sort by that, not total stars
repos.sort(key=lambda r: r.get("stargazers_count", 0), reverse=True) def _velocity(repo):
repos = repos[:limit] created = repo.get("created_at", "")
if created:
try:
created_dt = datetime.fromisoformat(
created.replace("Z", "+00:00")
)
age_days = max((now - created_dt).days, 1)
except (ValueError, TypeError):
age_days = 1
else:
age_days = 1
return repo.get("stargazers_count", 0) / age_days
repos.sort(key=_velocity, reverse=True)
repos = repos[:limit]
entries = [] entries = []
readme_budget = min(10, limit) # Only fetch README for top 10 to stay under rate limit readme_budget = min(10, limit) # Only fetch README for top 10 to stay under rate limit
@@ -212,6 +236,8 @@ class GitHubAdapter(SourceAdapter):
time.sleep(0.5) # polite spacing between README fetches time.sleep(0.5) # polite spacing between README fetches
# Structured metadata # Structured metadata
stars = repo.get("stargazers_count", 0)
velocity = stars / max(age_days, 1)
raw_meta = { raw_meta = {
"full_name": repo.get("full_name", ""), "full_name": repo.get("full_name", ""),
"owner": repo.get("owner", {}).get("login", ""), "owner": repo.get("owner", {}).get("login", ""),
@@ -224,6 +250,7 @@ class GitHubAdapter(SourceAdapter):
"updated_at": repo.get("updated_at", ""), "updated_at": repo.get("updated_at", ""),
"pushed_at": repo.get("pushed_at", ""), "pushed_at": repo.get("pushed_at", ""),
"age_days": age_days, "age_days": age_days,
"stars_per_day": round(velocity, 1),
"readme_length": len(readme_text), "readme_length": len(readme_text),
"score_type": "actual", # based on real star counts "score_type": "actual", # based on real star counts
} }