Agent Skills Pipeline v1.0

8-stage pipeline: Scout → Filter → Reader → Extractor → Score → Generator → Reviewer → Publisher - Scout: GitHub search with token auth + rate limit retry - Filter: Deterministic rules (language, stars, age, keywords) - Reader: Incremental context loading (README → docs → examples → code) - Extractor: LLM workflow extraction with JSON retry - Score: Rule-based evaluation (no LLM) - Generator: Standardized Hermes Skill format - Reviewer: Independent LLM review (separate from generator) - Publisher: Branch + PR to Gitea First run: 5 repos discovered, 0 extracted (correct — all frameworks, no workflows)
2026-08-05 05:51:06 +00:00
commit 8da8d703da
25 changed files with 1159 additions and 0 deletions
@@ -0,0 +1,80 @@
+"""Stage 2: Filter — Deterministic noise removal before LLM."""
+
+def filter_repos(repos, config):
+    """
+    Apply deterministic rules to filter out irrelevant repos.
+    Goal: eliminate obvious noise, not perfect classification.
+    """
+    filter_config = config.get("filter", {})
+    reject_categories = set(filter_config.get("reject", []))
+    keep_categories = set(filter_config.get("keep", []))
+    min_stars = filter_config.get("min_stars", 50)
+    max_age_days = filter_config.get("max_age_days", 365)
+
+    from datetime import datetime, timedelta
+
+    kept = []
+    rejected = []
+    now = datetime.now()
+
+    for repo in repos:
+        if repo.get("status") == "ERROR":
+            continue
+
+        reasons = []
+
+        # Archived
+        if repo.get("archived", False):
+            reasons.append("archived")
+
+        # Stars too low
+        if repo.get("stars", 0) < min_stars:
+            reasons.append(f"stars {repo.get('stars', 0)} < {min_stars}")
+
+        # Too old
+        updated = repo.get("updated_at", "")
+        if updated:
+            try:
+                updated_dt = datetime.fromisoformat(updated.replace("Z", "+00:00"))
+                if (now - updated_dt).days > max_age_days:
+                    reasons.append(f"too old ({(now - updated_dt).days} days)")
+            except:
+                pass
+
+        # Wrong category
+        lang = repo.get("language", "")
+        if lang in reject_categories:
+            reasons.append(f"rejected language: {lang}")
+
+        # Size check — too small to have meaningful workflow
+        size_kb = repo.get("size_kb", 0)
+        if size_kb < 20:
+            reasons.append(f"too small ({size_kb}KB)")
+
+        # Check description for obvious non-AI content
+        desc_lower = (repo.get("description") or "").lower()
+        skip_keywords = ["css", "animation library", "color picker", "bootstrap theme",
+                         "game", "minecraft", "pygame", "flappy bird", "snake game",
+                         "dataset", "kaggle", "csv only", "data dump"]
+        for kw in skip_keywords:
+            if kw in desc_lower:
+                reasons.append(f"description contains: {kw}")
+                break
+
+        if reasons:
+            rejected.append({
+                "repo": repo,
+                "decision": "REJECT",
+                "reasons": reasons,
+            })
+        else:
+            repo["status"] = "FILTERED"
+            kept.append(repo)
+
+    return {
+        "status": "OK",
+        "kept": kept,
+        "rejected": rejected,
+        "kept_count": len(kept),
+        "rejected_count": len(rejected),
+    }