Epictetus
dc40d4c0db
- Reader: discover workflow files in nested dirs (agents/, workflows/, examples/) - Reader: load source code, config, deps — not just docs - Extractor: prompt demands concrete implementation details (files, deps, code) - Scorer: removed general_purpose check (5/6 checks, score 1.0) - Generator: includes Setup, Key Files, Implementation Details sections - Reviewer: replaced LLM review with 8 deterministic structural checks - Publisher: handle 409 duplicate PR gracefully as success - 5 skills published as PRs #6-#10 on Gitea
92 lines
3.4 KiB
Python
92 lines
3.4 KiB
Python
"""Stage 7: Reviewer — Deterministic structural checks on generated skill."""
|
|
import re
|
|
|
|
|
|
def review_skill(generator_output, config):
|
|
"""
|
|
Deterministic review of generated skill. No LLM involved.
|
|
|
|
Checks that the SKILL.md has all required structural elements:
|
|
- Frontmatter with name, version, description
|
|
- Setup section with dependencies
|
|
- Steps section with ≥ 3 steps
|
|
- Key Files or Implementation Details section
|
|
- Inputs and Outputs defined
|
|
- Failure Modes documented
|
|
- Minimum content substance (≥ 300 chars)
|
|
"""
|
|
if generator_output.get("status") != "GENERATED":
|
|
return {
|
|
"status": "BLOCKED",
|
|
"reason": "Generation failed",
|
|
}
|
|
|
|
files = generator_output.get("files", {})
|
|
skill_md = files.get("SKILL.md", "")
|
|
metadata = files.get("metadata.json", "{}")
|
|
|
|
checks = {}
|
|
issues = []
|
|
|
|
# 1. Frontmatter exists with required fields
|
|
has_frontmatter = skill_md.startswith("---") and "---" in skill_md[3:]
|
|
has_name = "name:" in skill_md.split("---")[1] if has_frontmatter else False
|
|
has_version = "version:" in skill_md
|
|
has_description = "description:" in skill_md
|
|
checks["frontmatter_complete"] = has_frontmatter and has_name and has_version and has_description
|
|
|
|
# 2. Has Setup section with dependencies
|
|
has_setup = "## Setup" in skill_md or "## Dependencies" in skill_md
|
|
has_deps = "pip install" in skill_md or "requirements" in skill_md.lower() or "Dependencies" in skill_md
|
|
checks["setup_documented"] = has_setup or has_deps
|
|
|
|
# 3. Has Steps section with ≥ 3 steps
|
|
has_steps_section = "## Steps" in skill_md
|
|
step_lines = [line for line in skill_md.split("\n") if re.match(r"^\d+\.\s", line)]
|
|
checks["has_steps"] = has_steps_section and len(step_lines) >= 3
|
|
|
|
# 4. Has Key Files or Implementation Details section
|
|
has_key_files = "## Key Files" in skill_md
|
|
has_impl_details = "## Implementation Details" in skill_md
|
|
checks["implementation_details"] = has_key_files or has_impl_details
|
|
|
|
# 5. Inputs and Outputs defined
|
|
has_inputs = "## Inputs" in skill_md
|
|
has_outputs = "## Outputs" in skill_md
|
|
checks["inputs_outputs_defined"] = has_inputs and has_outputs
|
|
|
|
# 6. Failure Modes documented
|
|
has_failure_modes = "## Failure Modes" in skill_md
|
|
checks["failure_modes_documented"] = has_failure_modes
|
|
|
|
# 7. Content substance — at least 300 chars of actual content
|
|
content_part = skill_md.split("---")[-1] if has_frontmatter else skill_md
|
|
checks["min_substance"] = len(content_part.strip()) >= 300
|
|
|
|
# 8. Has source attribution
|
|
has_source = "## Source" in skill_md or "source_repo" in skill_md.lower()
|
|
checks["source_attribution"] = has_source
|
|
|
|
# Score
|
|
passed = sum(1 for v in checks.values() if v)
|
|
total = len(checks)
|
|
score = passed / total if total > 0 else 0
|
|
|
|
min_score = config.get("reviewer", {}).get("min_score", 0.625) # 5/8 checks
|
|
decision = "PASS" if score >= min_score else "REJECT"
|
|
|
|
# Build issue list
|
|
for check_name, result in checks.items():
|
|
if not result:
|
|
issues.append(f"Missing: {check_name}")
|
|
|
|
return {
|
|
"status": "APPROVED" if decision == "PASS" else "REJECTED",
|
|
"decision": decision,
|
|
"score": round(score, 2),
|
|
"min_score": min_score,
|
|
"checks": checks,
|
|
"issues": issues,
|
|
"generator_output": generator_output,
|
|
}
|