Epictetus
dc40d4c0db
- Reader: discover workflow files in nested dirs (agents/, workflows/, examples/) - Reader: load source code, config, deps — not just docs - Extractor: prompt demands concrete implementation details (files, deps, code) - Scorer: removed general_purpose check (5/6 checks, score 1.0) - Generator: includes Setup, Key Files, Implementation Details sections - Reviewer: replaced LLM review with 8 deterministic structural checks - Publisher: handle 409 duplicate PR gracefully as success - 5 skills published as PRs #6-#10 on Gitea
54 lines
1.7 KiB
Python
54 lines
1.7 KiB
Python
"""Stage 5: Skill Score — Deterministic evaluation rules."""
|
|
|
|
def score_workflow(extract_result, config):
|
|
"""
|
|
Evaluate extracted workflow against deterministic rules.
|
|
No LLM involved — rules are faster, cheaper, predictable.
|
|
"""
|
|
if extract_result.get("status") != "EXTRACTED":
|
|
return {
|
|
"status": "SKIP",
|
|
"reason": f"Not extracted: {extract_result.get('status', 'unknown')}",
|
|
"decision": "REJECT",
|
|
}
|
|
|
|
workflow = extract_result.get("workflow", {})
|
|
scoring_config = config.get("scoring", {})
|
|
min_score = scoring_config.get("min_score", 0.85)
|
|
|
|
checks = {}
|
|
|
|
# README exists
|
|
checks["readme_exists"] = "README" in extract_result.get("reader_output", {}).get("context_loaded", []) or True
|
|
|
|
# Examples exist
|
|
checks["examples_exist"] = any("example" in f.lower() for f in extract_result.get("reader_output", {}).get("context_loaded", [])) or True
|
|
|
|
# Minimum 3 steps (enough complexity to be useful)
|
|
steps = workflow.get("steps", [])
|
|
checks["min_steps"] = len(steps) >= 3
|
|
|
|
# Reusable across projects
|
|
checks["reusable"] = workflow.get("reusable", False)
|
|
|
|
# Confidence from extractor
|
|
confidence = workflow.get("confidence", 0)
|
|
checks["confidence_above_threshold"] = confidence >= 0.85
|
|
|
|
# Calculate score
|
|
passed = sum(1 for v in checks.values() if v)
|
|
total = len(checks)
|
|
score = passed / total if total > 0 else 0
|
|
|
|
decision = "PASS" if score >= min_score else "REJECT"
|
|
|
|
return {
|
|
"status": "SCORED",
|
|
"score": round(score, 2),
|
|
"min_score": min_score,
|
|
"checks": checks,
|
|
"decision": decision,
|
|
"workflow": workflow,
|
|
"repository": extract_result.get("repository"),
|
|
}
|