VPS admin
8da8d703da
8-stage pipeline: Scout → Filter → Reader → Extractor → Score → Generator → Reviewer → Publisher - Scout: GitHub search with token auth + rate limit retry - Filter: Deterministic rules (language, stars, age, keywords) - Reader: Incremental context loading (README → docs → examples → code) - Extractor: LLM workflow extraction with JSON retry - Score: Rule-based evaluation (no LLM) - Generator: Standardized Hermes Skill format - Reviewer: Independent LLM review (separate from generator) - Publisher: Branch + PR to Gitea First run: 5 repos discovered, 0 extracted (correct — all frameworks, no workflows)
85 lines
2.6 KiB
Python
85 lines
2.6 KiB
Python
"""Stage 7: Reviewer — LLM review of generated skill."""
|
|
import json
|
|
from pipeline.extractor import call_llm
|
|
|
|
def review_skill(generator_output, config):
|
|
"""
|
|
Review a generated skill. Generation and review are separated.
|
|
The reviewer never modifies — only approves or rejects with feedback.
|
|
"""
|
|
if generator_output.get("status") != "GENERATED":
|
|
return {
|
|
"status": "BLOCKED",
|
|
"reason": "Generation failed",
|
|
}
|
|
|
|
files = generator_output.get("files", {})
|
|
skill_md = files.get("SKILL.md", "")
|
|
|
|
prompt = f"""You are reviewing an AI Agent Skill that was automatically extracted from a GitHub repository.
|
|
|
|
Would an experienced engineer install this Skill without editing it?
|
|
|
|
Answer with ONLY valid JSON in this format:
|
|
{{
|
|
"decision": "YES" or "NO",
|
|
"confidence": 0.0-1.0,
|
|
"reason": "One paragraph explaining your decision",
|
|
"missing_assumptions": ["List any unclear steps or assumptions"],
|
|
"minimum_changes": ["If NO, list the minimum changes for approval"]
|
|
}}
|
|
|
|
Skill to review:
|
|
|
|
{skill_md}
|
|
|
|
Remember:
|
|
- The skill must be clearly documented
|
|
- It must be reusable outside the original repository
|
|
- Steps must be specific enough to execute
|
|
- Inputs and outputs must be well-defined
|
|
- Failure modes should be documented
|
|
|
|
Return ONLY valid JSON. No markdown."""
|
|
|
|
result_text = call_llm(prompt, config)
|
|
|
|
try:
|
|
cleaned = result_text.strip()
|
|
if cleaned.startswith("```"):
|
|
cleaned = cleaned.split("```")[1]
|
|
if cleaned.startswith("json"):
|
|
cleaned = cleaned[4:]
|
|
cleaned = cleaned.rstrip("```")
|
|
cleaned = cleaned.strip()
|
|
|
|
review = json.loads(cleaned)
|
|
|
|
decision = review.get("decision", "NO").upper()
|
|
confidence = review.get("confidence", 0)
|
|
min_confidence = config.get("reviewer", {}).get("confidence_min", 0.80)
|
|
|
|
if decision == "YES" and confidence >= min_confidence:
|
|
status = "APPROVED"
|
|
elif decision == "YES" and confidence < min_confidence:
|
|
status = "LOW_CONFIDENCE"
|
|
else:
|
|
status = "REJECTED"
|
|
|
|
return {
|
|
"status": status,
|
|
"decision": decision,
|
|
"confidence": confidence,
|
|
"reason": review.get("reason", ""),
|
|
"missing_assumptions": review.get("missing_assumptions", []),
|
|
"minimum_changes": review.get("minimum_changes", []),
|
|
"generator_output": generator_output,
|
|
}
|
|
|
|
except json.JSONDecodeError:
|
|
return {
|
|
"status": "REVIEW_ERROR",
|
|
"raw": result_text[:500],
|
|
"generator_output": generator_output,
|
|
}
|