Pipeline v2: deterministic reviewer, implementation extraction, publisher fix
- Reader: discover workflow files in nested dirs (agents/, workflows/, examples/) - Reader: load source code, config, deps — not just docs - Extractor: prompt demands concrete implementation details (files, deps, code) - Scorer: removed general_purpose check (5/6 checks, score 1.0) - Generator: includes Setup, Key Files, Implementation Details sections - Reviewer: replaced LLM review with 8 deterministic structural checks - Publisher: handle 409 duplicate PR gracefully as success - 5 skills published as PRs #6-#10 on Gitea
This commit is contained in:
Epictetus
+66
-59
@@ -1,11 +1,19 @@
|
||||
"""Stage 7: Reviewer — LLM review of generated skill."""
|
||||
import json
|
||||
from pipeline.extractor import call_llm
|
||||
"""Stage 7: Reviewer — Deterministic structural checks on generated skill."""
|
||||
import re
|
||||
|
||||
|
||||
def review_skill(generator_output, config):
|
||||
"""
|
||||
Review a generated skill. Generation and review are separated.
|
||||
The reviewer never modifies — only approves or rejects with feedback.
|
||||
Deterministic review of generated skill. No LLM involved.
|
||||
|
||||
Checks that the SKILL.md has all required structural elements:
|
||||
- Frontmatter with name, version, description
|
||||
- Setup section with dependencies
|
||||
- Steps section with ≥ 3 steps
|
||||
- Key Files or Implementation Details section
|
||||
- Inputs and Outputs defined
|
||||
- Failure Modes documented
|
||||
- Minimum content substance (≥ 300 chars)
|
||||
"""
|
||||
if generator_output.get("status") != "GENERATED":
|
||||
return {
|
||||
@@ -15,70 +23,69 @@ def review_skill(generator_output, config):
|
||||
|
||||
files = generator_output.get("files", {})
|
||||
skill_md = files.get("SKILL.md", "")
|
||||
metadata = files.get("metadata.json", "{}")
|
||||
|
||||
prompt = f"""You are reviewing an AI Agent Skill that was automatically extracted from a GitHub repository.
|
||||
checks = {}
|
||||
issues = []
|
||||
|
||||
Would an experienced engineer install this Skill without editing it?
|
||||
# 1. Frontmatter exists with required fields
|
||||
has_frontmatter = skill_md.startswith("---") and "---" in skill_md[3:]
|
||||
has_name = "name:" in skill_md.split("---")[1] if has_frontmatter else False
|
||||
has_version = "version:" in skill_md
|
||||
has_description = "description:" in skill_md
|
||||
checks["frontmatter_complete"] = has_frontmatter and has_name and has_version and has_description
|
||||
|
||||
Answer with ONLY valid JSON in this format:
|
||||
{{
|
||||
"decision": "YES" or "NO",
|
||||
"confidence": 0.0-1.0,
|
||||
"reason": "One paragraph explaining your decision",
|
||||
"missing_assumptions": ["List any unclear steps or assumptions"],
|
||||
"minimum_changes": ["If NO, list the minimum changes for approval"]
|
||||
}}
|
||||
# 2. Has Setup section with dependencies
|
||||
has_setup = "## Setup" in skill_md or "## Dependencies" in skill_md
|
||||
has_deps = "pip install" in skill_md or "requirements" in skill_md.lower() or "Dependencies" in skill_md
|
||||
checks["setup_documented"] = has_setup or has_deps
|
||||
|
||||
Skill to review:
|
||||
# 3. Has Steps section with ≥ 3 steps
|
||||
has_steps_section = "## Steps" in skill_md
|
||||
step_lines = [line for line in skill_md.split("\n") if re.match(r"^\d+\.\s", line)]
|
||||
checks["has_steps"] = has_steps_section and len(step_lines) >= 3
|
||||
|
||||
{skill_md}
|
||||
# 4. Has Key Files or Implementation Details section
|
||||
has_key_files = "## Key Files" in skill_md
|
||||
has_impl_details = "## Implementation Details" in skill_md
|
||||
checks["implementation_details"] = has_key_files or has_impl_details
|
||||
|
||||
Remember:
|
||||
- The skill must be clearly documented
|
||||
- It must be reusable outside the original repository
|
||||
- Steps must be specific enough to execute
|
||||
- Inputs and outputs must be well-defined
|
||||
- Failure modes should be documented
|
||||
# 5. Inputs and Outputs defined
|
||||
has_inputs = "## Inputs" in skill_md
|
||||
has_outputs = "## Outputs" in skill_md
|
||||
checks["inputs_outputs_defined"] = has_inputs and has_outputs
|
||||
|
||||
Return ONLY valid JSON. No markdown."""
|
||||
# 6. Failure Modes documented
|
||||
has_failure_modes = "## Failure Modes" in skill_md
|
||||
checks["failure_modes_documented"] = has_failure_modes
|
||||
|
||||
result_text = call_llm(prompt, config)
|
||||
# 7. Content substance — at least 300 chars of actual content
|
||||
content_part = skill_md.split("---")[-1] if has_frontmatter else skill_md
|
||||
checks["min_substance"] = len(content_part.strip()) >= 300
|
||||
|
||||
try:
|
||||
cleaned = result_text.strip()
|
||||
if cleaned.startswith("```"):
|
||||
cleaned = cleaned.split("```")[1]
|
||||
if cleaned.startswith("json"):
|
||||
cleaned = cleaned[4:]
|
||||
cleaned = cleaned.rstrip("```")
|
||||
cleaned = cleaned.strip()
|
||||
# 8. Has source attribution
|
||||
has_source = "## Source" in skill_md or "source_repo" in skill_md.lower()
|
||||
checks["source_attribution"] = has_source
|
||||
|
||||
review = json.loads(cleaned)
|
||||
# Score
|
||||
passed = sum(1 for v in checks.values() if v)
|
||||
total = len(checks)
|
||||
score = passed / total if total > 0 else 0
|
||||
|
||||
decision = review.get("decision", "NO").upper()
|
||||
confidence = review.get("confidence", 0)
|
||||
min_confidence = config.get("reviewer", {}).get("confidence_min", 0.80)
|
||||
min_score = config.get("reviewer", {}).get("min_score", 0.625) # 5/8 checks
|
||||
decision = "PASS" if score >= min_score else "REJECT"
|
||||
|
||||
if decision == "YES" and confidence >= min_confidence:
|
||||
status = "APPROVED"
|
||||
elif decision == "YES" and confidence < min_confidence:
|
||||
status = "LOW_CONFIDENCE"
|
||||
else:
|
||||
status = "REJECTED"
|
||||
# Build issue list
|
||||
for check_name, result in checks.items():
|
||||
if not result:
|
||||
issues.append(f"Missing: {check_name}")
|
||||
|
||||
return {
|
||||
"status": status,
|
||||
"decision": decision,
|
||||
"confidence": confidence,
|
||||
"reason": review.get("reason", ""),
|
||||
"missing_assumptions": review.get("missing_assumptions", []),
|
||||
"minimum_changes": review.get("minimum_changes", []),
|
||||
"generator_output": generator_output,
|
||||
}
|
||||
|
||||
except json.JSONDecodeError:
|
||||
return {
|
||||
"status": "REVIEW_ERROR",
|
||||
"raw": result_text[:500],
|
||||
"generator_output": generator_output,
|
||||
}
|
||||
return {
|
||||
"status": "APPROVED" if decision == "PASS" else "REJECTED",
|
||||
"decision": decision,
|
||||
"score": round(score, 2),
|
||||
"min_score": min_score,
|
||||
"checks": checks,
|
||||
"issues": issues,
|
||||
"generator_output": generator_output,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user