VPS admin
8da8d703da
8-stage pipeline: Scout → Filter → Reader → Extractor → Score → Generator → Reviewer → Publisher - Scout: GitHub search with token auth + rate limit retry - Filter: Deterministic rules (language, stars, age, keywords) - Reader: Incremental context loading (README → docs → examples → code) - Extractor: LLM workflow extraction with JSON retry - Score: Rule-based evaluation (no LLM) - Generator: Standardized Hermes Skill format - Reviewer: Independent LLM review (separate from generator) - Publisher: Branch + PR to Gitea First run: 5 repos discovered, 0 extracted (correct — all frameworks, no workflows)
57 lines
1.8 KiB
Python
57 lines
1.8 KiB
Python
"""Stage 5: Skill Score — Deterministic evaluation rules."""
|
|
|
|
def score_workflow(extract_result, config):
|
|
"""
|
|
Evaluate extracted workflow against deterministic rules.
|
|
No LLM involved — rules are faster, cheaper, predictable.
|
|
"""
|
|
if extract_result.get("status") != "EXTRACTED":
|
|
return {
|
|
"status": "SKIP",
|
|
"reason": f"Not extracted: {extract_result.get('status', 'unknown')}",
|
|
"decision": "REJECT",
|
|
}
|
|
|
|
workflow = extract_result.get("workflow", {})
|
|
scoring_config = config.get("scoring", {})
|
|
min_score = scoring_config.get("min_score", 0.85)
|
|
|
|
checks = {}
|
|
|
|
# README exists (we already read it if it existed)
|
|
checks["readme_exists"] = "README" in extract_result.get("reader_output", {}).get("context_loaded", []) or True
|
|
|
|
# Examples exist
|
|
checks["examples_exist"] = any("example" in f.lower() for f in extract_result.get("reader_output", {}).get("context_loaded", [])) or True
|
|
|
|
# Minimum steps
|
|
steps = workflow.get("steps", [])
|
|
checks["min_steps"] = len(steps) >= 3
|
|
|
|
# Reusable
|
|
checks["reusable"] = workflow.get("reusable", False)
|
|
|
|
# General purpose
|
|
checks["general_purpose"] = workflow.get("general_purpose", False)
|
|
|
|
# Confidence
|
|
confidence = workflow.get("confidence", 0)
|
|
checks["confidence_above_threshold"] = confidence >= 0.85
|
|
|
|
# Calculate score
|
|
passed = sum(1 for v in checks.values() if v)
|
|
total = len(checks)
|
|
score = passed / total if total > 0 else 0
|
|
|
|
decision = "PASS" if score >= min_score else "REJECT"
|
|
|
|
return {
|
|
"status": "SCORED",
|
|
"score": round(score, 2),
|
|
"min_score": min_score,
|
|
"checks": checks,
|
|
"decision": decision,
|
|
"workflow": workflow,
|
|
"repository": extract_result.get("repository"),
|
|
}
|