Agent Skills Pipeline v1.0

8-stage pipeline: Scout → Filter → Reader → Extractor → Score → Generator → Reviewer → Publisher - Scout: GitHub search with token auth + rate limit retry - Filter: Deterministic rules (language, stars, age, keywords) - Reader: Incremental context loading (README → docs → examples → code) - Extractor: LLM workflow extraction with JSON retry - Score: Rule-based evaluation (no LLM) - Generator: Standardized Hermes Skill format - Reviewer: Independent LLM review (separate from generator) - Publisher: Branch + PR to Gitea First run: 5 repos discovered, 0 extracted (correct — all frameworks, no workflows)
2026-08-05 05:51:06 +00:00
commit 8da8d703da
25 changed files with 1159 additions and 0 deletions
@@ -0,0 +1,95 @@
+"""Stage 1: Scout — Discover candidate repos from GitHub."""
+import json
+import requests
+from datetime import datetime, timedelta
+
+def scout(config, state=None):
+    """
+    Search GitHub for repos matching AI workflow queries.
+    Returns list of discovered repos with metadata.
+    """
+    queries = config.get("scout", {}).get("queries", [])
+    filters = config.get("scout", {}).get("filters", {})
+    max_results = config.get("scout", {}).get("max_results", 30)
+    cooldown_hours = config.get("scout", {}).get("cooldown_hours", 24)
+
+    # Check cooldown
+    if state is None:
+        state = {}
+    if "last_run" in state:
+        last = datetime.fromisoformat(state["last_run"])
+        if datetime.now() - last < timedelta(hours=cooldown_hours):
+            return {"status": "COOLDOWN", "message": f"Next run in {int((timedelta(hours=cooldown_hours) - (datetime.now() - last)).total_seconds() / 3600)}h"}
+
+    discovered = []
+    seen_urls = set()
+
+    for query in queries:
+        stars_min = filters.get("stars_min", 50)
+        pushed_after = filters.get("pushed_after", "2026-01-01")
+        language = filters.get("language", "Python")
+
+        # Build query string safely — requests handles URL encoding of params
+        url = "https://api.github.com/search/repositories"
+        search_q = f"{query} stars:>{stars_min} pushed:>{pushed_after} language:{language}"
+        params = {
+            "q": search_q,
+            "sort": "updated",
+            "order": "desc",
+            "per_page": min(max_results, 30),
+        }
+
+        headers = {}
+        github_token = config.get("github", {}).get("token", "")
+        if github_token:
+            headers["Authorization"] = f"token {github_token}"
+            headers["Accept"] = "application/vnd.github.v3+json"
+
+        try:
+            resp = requests.get(url, params=params, headers=headers, timeout=15)
+            if resp.status_code == 403:
+                import time
+                time.sleep(60)  # Wait for rate limit window
+                resp = requests.get(url, params=params, headers=headers, timeout=15)
+                if resp.status_code == 403:
+                    return {"status": "RATE_LIMITED", "message": "GitHub API rate limit hit after retry."}
+            if resp.status_code != 200:
+                continue
+
+            data = resp.json()
+            for item in data.get("items", []):
+                repo_url = item.get("html_url", "")
+                if repo_url in seen_urls:
+                    continue
+                seen_urls.add(repo_url)
+
+                discovered.append({
+                    "name": item.get("name", ""),
+                    "full_name": item.get("full_name", ""),
+                    "url": repo_url,
+                    "clone_url": item.get("clone_url", ""),
+                    "stars": item.get("stargazers_count", 0),
+                    "language": item.get("language", ""),
+                    "description": item.get("description", ""),
+                    "updated_at": item.get("updated_at", ""),
+                    "created_at": item.get("created_at", ""),
+                    "archived": item.get("archived", False),
+                    "size_kb": item.get("size", 0),
+                    "status": "DISCOVERED",
+                    "discovered_at": datetime.now().isoformat(),
+                })
+
+            if len(discovered) >= max_results:
+                break
+
+        except Exception as e:
+            discovered.append({"error": str(e), "query": query, "status": "ERROR"})
+
+    state["last_run"] = datetime.now().isoformat()
+
+    return {
+        "status": "OK",
+        "count": len(discovered),
+        "repos": discovered,
+        "state": state,
+    }