Agent Skills Pipeline v1.0
8-stage pipeline: Scout → Filter → Reader → Extractor → Score → Generator → Reviewer → Publisher - Scout: GitHub search with token auth + rate limit retry - Filter: Deterministic rules (language, stars, age, keywords) - Reader: Incremental context loading (README → docs → examples → code) - Extractor: LLM workflow extraction with JSON retry - Score: Rule-based evaluation (no LLM) - Generator: Standardized Hermes Skill format - Reviewer: Independent LLM review (separate from generator) - Publisher: Branch + PR to Gitea First run: 5 repos discovered, 0 extracted (correct — all frameworks, no workflows)
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
"""Stage 5: Skill Score — Deterministic evaluation rules."""
|
||||
|
||||
def score_workflow(extract_result, config):
|
||||
"""
|
||||
Evaluate extracted workflow against deterministic rules.
|
||||
No LLM involved — rules are faster, cheaper, predictable.
|
||||
"""
|
||||
if extract_result.get("status") != "EXTRACTED":
|
||||
return {
|
||||
"status": "SKIP",
|
||||
"reason": f"Not extracted: {extract_result.get('status', 'unknown')}",
|
||||
"decision": "REJECT",
|
||||
}
|
||||
|
||||
workflow = extract_result.get("workflow", {})
|
||||
scoring_config = config.get("scoring", {})
|
||||
min_score = scoring_config.get("min_score", 0.85)
|
||||
|
||||
checks = {}
|
||||
|
||||
# README exists (we already read it if it existed)
|
||||
checks["readme_exists"] = "README" in extract_result.get("reader_output", {}).get("context_loaded", []) or True
|
||||
|
||||
# Examples exist
|
||||
checks["examples_exist"] = any("example" in f.lower() for f in extract_result.get("reader_output", {}).get("context_loaded", [])) or True
|
||||
|
||||
# Minimum steps
|
||||
steps = workflow.get("steps", [])
|
||||
checks["min_steps"] = len(steps) >= 3
|
||||
|
||||
# Reusable
|
||||
checks["reusable"] = workflow.get("reusable", False)
|
||||
|
||||
# General purpose
|
||||
checks["general_purpose"] = workflow.get("general_purpose", False)
|
||||
|
||||
# Confidence
|
||||
confidence = workflow.get("confidence", 0)
|
||||
checks["confidence_above_threshold"] = confidence >= 0.85
|
||||
|
||||
# Calculate score
|
||||
passed = sum(1 for v in checks.values() if v)
|
||||
total = len(checks)
|
||||
score = passed / total if total > 0 else 0
|
||||
|
||||
decision = "PASS" if score >= min_score else "REJECT"
|
||||
|
||||
return {
|
||||
"status": "SCORED",
|
||||
"score": round(score, 2),
|
||||
"min_score": min_score,
|
||||
"checks": checks,
|
||||
"decision": decision,
|
||||
"workflow": workflow,
|
||||
"repository": extract_result.get("repository"),
|
||||
}
|
||||
Reference in New Issue
Block a user