VPS admin
8da8d703da
8-stage pipeline: Scout → Filter → Reader → Extractor → Score → Generator → Reviewer → Publisher - Scout: GitHub search with token auth + rate limit retry - Filter: Deterministic rules (language, stars, age, keywords) - Reader: Incremental context loading (README → docs → examples → code) - Extractor: LLM workflow extraction with JSON retry - Score: Rule-based evaluation (no LLM) - Generator: Standardized Hermes Skill format - Reviewer: Independent LLM review (separate from generator) - Publisher: Branch + PR to Gitea First run: 5 repos discovered, 0 extracted (correct — all frameworks, no workflows)
81 lines
2.6 KiB
Python
81 lines
2.6 KiB
Python
"""Stage 2: Filter — Deterministic noise removal before LLM."""
|
|
|
|
def filter_repos(repos, config):
|
|
"""
|
|
Apply deterministic rules to filter out irrelevant repos.
|
|
Goal: eliminate obvious noise, not perfect classification.
|
|
"""
|
|
filter_config = config.get("filter", {})
|
|
reject_categories = set(filter_config.get("reject", []))
|
|
keep_categories = set(filter_config.get("keep", []))
|
|
min_stars = filter_config.get("min_stars", 50)
|
|
max_age_days = filter_config.get("max_age_days", 365)
|
|
|
|
from datetime import datetime, timedelta
|
|
|
|
kept = []
|
|
rejected = []
|
|
now = datetime.now()
|
|
|
|
for repo in repos:
|
|
if repo.get("status") == "ERROR":
|
|
continue
|
|
|
|
reasons = []
|
|
|
|
# Archived
|
|
if repo.get("archived", False):
|
|
reasons.append("archived")
|
|
|
|
# Stars too low
|
|
if repo.get("stars", 0) < min_stars:
|
|
reasons.append(f"stars {repo.get('stars', 0)} < {min_stars}")
|
|
|
|
# Too old
|
|
updated = repo.get("updated_at", "")
|
|
if updated:
|
|
try:
|
|
updated_dt = datetime.fromisoformat(updated.replace("Z", "+00:00"))
|
|
if (now - updated_dt).days > max_age_days:
|
|
reasons.append(f"too old ({(now - updated_dt).days} days)")
|
|
except:
|
|
pass
|
|
|
|
# Wrong category
|
|
lang = repo.get("language", "")
|
|
if lang in reject_categories:
|
|
reasons.append(f"rejected language: {lang}")
|
|
|
|
# Size check — too small to have meaningful workflow
|
|
size_kb = repo.get("size_kb", 0)
|
|
if size_kb < 20:
|
|
reasons.append(f"too small ({size_kb}KB)")
|
|
|
|
# Check description for obvious non-AI content
|
|
desc_lower = (repo.get("description") or "").lower()
|
|
skip_keywords = ["css", "animation library", "color picker", "bootstrap theme",
|
|
"game", "minecraft", "pygame", "flappy bird", "snake game",
|
|
"dataset", "kaggle", "csv only", "data dump"]
|
|
for kw in skip_keywords:
|
|
if kw in desc_lower:
|
|
reasons.append(f"description contains: {kw}")
|
|
break
|
|
|
|
if reasons:
|
|
rejected.append({
|
|
"repo": repo,
|
|
"decision": "REJECT",
|
|
"reasons": reasons,
|
|
})
|
|
else:
|
|
repo["status"] = "FILTERED"
|
|
kept.append(repo)
|
|
|
|
return {
|
|
"status": "OK",
|
|
"kept": kept,
|
|
"rejected": rejected,
|
|
"kept_count": len(kept),
|
|
"rejected_count": len(rejected),
|
|
}
|