VPS admin
8da8d703da
8-stage pipeline: Scout → Filter → Reader → Extractor → Score → Generator → Reviewer → Publisher - Scout: GitHub search with token auth + rate limit retry - Filter: Deterministic rules (language, stars, age, keywords) - Reader: Incremental context loading (README → docs → examples → code) - Extractor: LLM workflow extraction with JSON retry - Score: Rule-based evaluation (no LLM) - Generator: Standardized Hermes Skill format - Reviewer: Independent LLM review (separate from generator) - Publisher: Branch + PR to Gitea First run: 5 repos discovered, 0 extracted (correct — all frameworks, no workflows)
96 lines
3.6 KiB
Python
96 lines
3.6 KiB
Python
"""Stage 1: Scout — Discover candidate repos from GitHub."""
|
|
import json
|
|
import requests
|
|
from datetime import datetime, timedelta
|
|
|
|
def scout(config, state=None):
|
|
"""
|
|
Search GitHub for repos matching AI workflow queries.
|
|
Returns list of discovered repos with metadata.
|
|
"""
|
|
queries = config.get("scout", {}).get("queries", [])
|
|
filters = config.get("scout", {}).get("filters", {})
|
|
max_results = config.get("scout", {}).get("max_results", 30)
|
|
cooldown_hours = config.get("scout", {}).get("cooldown_hours", 24)
|
|
|
|
# Check cooldown
|
|
if state is None:
|
|
state = {}
|
|
if "last_run" in state:
|
|
last = datetime.fromisoformat(state["last_run"])
|
|
if datetime.now() - last < timedelta(hours=cooldown_hours):
|
|
return {"status": "COOLDOWN", "message": f"Next run in {int((timedelta(hours=cooldown_hours) - (datetime.now() - last)).total_seconds() / 3600)}h"}
|
|
|
|
discovered = []
|
|
seen_urls = set()
|
|
|
|
for query in queries:
|
|
stars_min = filters.get("stars_min", 50)
|
|
pushed_after = filters.get("pushed_after", "2026-01-01")
|
|
language = filters.get("language", "Python")
|
|
|
|
# Build query string safely — requests handles URL encoding of params
|
|
url = "https://api.github.com/search/repositories"
|
|
search_q = f"{query} stars:>{stars_min} pushed:>{pushed_after} language:{language}"
|
|
params = {
|
|
"q": search_q,
|
|
"sort": "updated",
|
|
"order": "desc",
|
|
"per_page": min(max_results, 30),
|
|
}
|
|
|
|
headers = {}
|
|
github_token = config.get("github", {}).get("token", "")
|
|
if github_token:
|
|
headers["Authorization"] = f"token {github_token}"
|
|
headers["Accept"] = "application/vnd.github.v3+json"
|
|
|
|
try:
|
|
resp = requests.get(url, params=params, headers=headers, timeout=15)
|
|
if resp.status_code == 403:
|
|
import time
|
|
time.sleep(60) # Wait for rate limit window
|
|
resp = requests.get(url, params=params, headers=headers, timeout=15)
|
|
if resp.status_code == 403:
|
|
return {"status": "RATE_LIMITED", "message": "GitHub API rate limit hit after retry."}
|
|
if resp.status_code != 200:
|
|
continue
|
|
|
|
data = resp.json()
|
|
for item in data.get("items", []):
|
|
repo_url = item.get("html_url", "")
|
|
if repo_url in seen_urls:
|
|
continue
|
|
seen_urls.add(repo_url)
|
|
|
|
discovered.append({
|
|
"name": item.get("name", ""),
|
|
"full_name": item.get("full_name", ""),
|
|
"url": repo_url,
|
|
"clone_url": item.get("clone_url", ""),
|
|
"stars": item.get("stargazers_count", 0),
|
|
"language": item.get("language", ""),
|
|
"description": item.get("description", ""),
|
|
"updated_at": item.get("updated_at", ""),
|
|
"created_at": item.get("created_at", ""),
|
|
"archived": item.get("archived", False),
|
|
"size_kb": item.get("size", 0),
|
|
"status": "DISCOVERED",
|
|
"discovered_at": datetime.now().isoformat(),
|
|
})
|
|
|
|
if len(discovered) >= max_results:
|
|
break
|
|
|
|
except Exception as e:
|
|
discovered.append({"error": str(e), "query": query, "status": "ERROR"})
|
|
|
|
state["last_run"] = datetime.now().isoformat()
|
|
|
|
return {
|
|
"status": "OK",
|
|
"count": len(discovered),
|
|
"repos": discovered,
|
|
"state": state,
|
|
}
|