Agent Skills Pipeline v1.0

8-stage pipeline: Scout → Filter → Reader → Extractor → Score → Generator → Reviewer → Publisher - Scout: GitHub search with token auth + rate limit retry - Filter: Deterministic rules (language, stars, age, keywords) - Reader: Incremental context loading (README → docs → examples → code) - Extractor: LLM workflow extraction with JSON retry - Score: Rule-based evaluation (no LLM) - Generator: Standardized Hermes Skill format - Reviewer: Independent LLM review (separate from generator) - Publisher: Branch + PR to Gitea First run: 5 repos discovered, 0 extracted (correct — all frameworks, no workflows)
2026-08-05 05:51:06 +00:00
commit 8da8d703da
25 changed files with 1159 additions and 0 deletions
@@ -0,0 +1,84 @@
+"""Stage 7: Reviewer — LLM review of generated skill."""
+import json
+from pipeline.extractor import call_llm
+
+def review_skill(generator_output, config):
+    """
+    Review a generated skill. Generation and review are separated.
+    The reviewer never modifies — only approves or rejects with feedback.
+    """
+    if generator_output.get("status") != "GENERATED":
+        return {
+            "status": "BLOCKED",
+            "reason": "Generation failed",
+        }
+
+    files = generator_output.get("files", {})
+    skill_md = files.get("SKILL.md", "")
+
+    prompt = f"""You are reviewing an AI Agent Skill that was automatically extracted from a GitHub repository.
+
+Would an experienced engineer install this Skill without editing it?
+
+Answer with ONLY valid JSON in this format:
+{{
+  "decision": "YES" or "NO",
+  "confidence": 0.0-1.0,
+  "reason": "One paragraph explaining your decision",
+  "missing_assumptions": ["List any unclear steps or assumptions"],
+  "minimum_changes": ["If NO, list the minimum changes for approval"]
+}}
+
+Skill to review:
+
+{skill_md}
+
+Remember:
+- The skill must be clearly documented
+- It must be reusable outside the original repository
+- Steps must be specific enough to execute
+- Inputs and outputs must be well-defined
+- Failure modes should be documented
+
+Return ONLY valid JSON. No markdown."""
+
+    result_text = call_llm(prompt, config)
+
+    try:
+        cleaned = result_text.strip()
+        if cleaned.startswith("```"):
+            cleaned = cleaned.split("```")[1]
+            if cleaned.startswith("json"):
+                cleaned = cleaned[4:]
+            cleaned = cleaned.rstrip("```")
+        cleaned = cleaned.strip()
+
+        review = json.loads(cleaned)
+
+        decision = review.get("decision", "NO").upper()
+        confidence = review.get("confidence", 0)
+        min_confidence = config.get("reviewer", {}).get("confidence_min", 0.80)
+
+        if decision == "YES" and confidence >= min_confidence:
+            status = "APPROVED"
+        elif decision == "YES" and confidence < min_confidence:
+            status = "LOW_CONFIDENCE"
+        else:
+            status = "REJECTED"
+
+        return {
+            "status": status,
+            "decision": decision,
+            "confidence": confidence,
+            "reason": review.get("reason", ""),
+            "missing_assumptions": review.get("missing_assumptions", []),
+            "minimum_changes": review.get("minimum_changes", []),
+            "generator_output": generator_output,
+        }
+
+    except json.JSONDecodeError:
+        return {
+            "status": "REVIEW_ERROR",
+            "raw": result_text[:500],
+            "generator_output": generator_output,
+        }