commit 8da8d703da298047a53025b276783f93c1855fab Author: VPS admin Date: Wed Aug 5 05:51:06 2026 +0000 Agent Skills Pipeline v1.0 8-stage pipeline: Scout → Filter → Reader → Extractor → Score → Generator → Reviewer → Publisher - Scout: GitHub search with token auth + rate limit retry - Filter: Deterministic rules (language, stars, age, keywords) - Reader: Incremental context loading (README → docs → examples → code) - Extractor: LLM workflow extraction with JSON retry - Score: Rule-based evaluation (no LLM) - Generator: Standardized Hermes Skill format - Reviewer: Independent LLM review (separate from generator) - Publisher: Branch + PR to Gitea First run: 5 repos discovered, 0 extracted (correct — all frameworks, no workflows) diff --git a/config/settings.yaml b/config/settings.yaml new file mode 100644 index 0000000..cb54326 --- /dev/null +++ b/config/settings.yaml @@ -0,0 +1,79 @@ +# Agent Skills Pipeline Configuration + +gitea: + base_url: http://localhost:3000 + token: "b721b5f288d227e9336241f95319437ab40256d6" + owner: tonyjbala + repo: agent-skills + clone_url: http://localhost:3000/tonyjbala/agent-skills.git + +github: + token: "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" + +llm: + base_url: http://100.64.0.2:8083/v1 + model: /home/ty/models/qwen36-27b-mtp-gguf/Qwen3.6-27B-UD-Q4_K_XL.gguf + api_key: "" + max_tokens: 8000 + +scout: + queries: + - 'agent framework langgraph mcp multi-agent' + - 'ai workflow agent pipeline rag pipeline' + - 'llm orchestration tool-use tool calling' + filters: + stars_min: 10 + pushed_after: 2026-06-01 + language: Python + archived: false + max_results: 30 + cooldown_hours: 24 + +filter: + categories: + keep: + - "AI Agent" + - "Machine Learning" + - "NLP" + - "Data Processing" + - "DevOps" + - "Web Framework" + reject: + - "CSS" + - "JavaScript" + - "HTML" + - "Game" + - "Dataset" + min_stars: 10 + max_age_days: 365 + +reader: + max_files: 5 + max_tokens_per_file: 40000 + +extractor: + min_steps: 3 + required_fields: + - goal + - inputs + - steps + - outputs + +scoring: + min_score: 0.85 + checks: + - readme_exists + - examples_exist + - min_steps + - reusable + - general_purpose + +generator: + output_format: hermes_skill + +reviewer: + confidence_min: 0.80 + +publisher: + branch_prefix: "skill/" + assign_reviewer: "" diff --git a/pipeline/__init__.py b/pipeline/__init__.py new file mode 100644 index 0000000..26c476e --- /dev/null +++ b/pipeline/__init__.py @@ -0,0 +1,2 @@ +# Agent Skills Pipeline +# 8-stage pipeline: Scout → Filter → Reader → Extractor → Score → Generator → Reviewer → Publisher diff --git a/pipeline/__pycache__/__init__.cpython-311.pyc b/pipeline/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000..42e7b11 Binary files /dev/null and b/pipeline/__pycache__/__init__.cpython-311.pyc differ diff --git a/pipeline/__pycache__/extractor.cpython-311.pyc b/pipeline/__pycache__/extractor.cpython-311.pyc new file mode 100644 index 0000000..58ede8c Binary files /dev/null and b/pipeline/__pycache__/extractor.cpython-311.pyc differ diff --git a/pipeline/__pycache__/filter.cpython-311.pyc b/pipeline/__pycache__/filter.cpython-311.pyc new file mode 100644 index 0000000..86c1ff4 Binary files /dev/null and b/pipeline/__pycache__/filter.cpython-311.pyc differ diff --git a/pipeline/__pycache__/generator.cpython-311.pyc b/pipeline/__pycache__/generator.cpython-311.pyc new file mode 100644 index 0000000..ae7d462 Binary files /dev/null and b/pipeline/__pycache__/generator.cpython-311.pyc differ diff --git a/pipeline/__pycache__/publisher.cpython-311.pyc b/pipeline/__pycache__/publisher.cpython-311.pyc new file mode 100644 index 0000000..d5ca418 Binary files /dev/null and b/pipeline/__pycache__/publisher.cpython-311.pyc differ diff --git a/pipeline/__pycache__/reader.cpython-311.pyc b/pipeline/__pycache__/reader.cpython-311.pyc new file mode 100644 index 0000000..7087fd0 Binary files /dev/null and b/pipeline/__pycache__/reader.cpython-311.pyc differ diff --git a/pipeline/__pycache__/reviewer.cpython-311.pyc b/pipeline/__pycache__/reviewer.cpython-311.pyc new file mode 100644 index 0000000..ca25fa3 Binary files /dev/null and b/pipeline/__pycache__/reviewer.cpython-311.pyc differ diff --git a/pipeline/__pycache__/scorer.cpython-311.pyc b/pipeline/__pycache__/scorer.cpython-311.pyc new file mode 100644 index 0000000..4d96287 Binary files /dev/null and b/pipeline/__pycache__/scorer.cpython-311.pyc differ diff --git a/pipeline/__pycache__/scout.cpython-311.pyc b/pipeline/__pycache__/scout.cpython-311.pyc new file mode 100644 index 0000000..e9141ce Binary files /dev/null and b/pipeline/__pycache__/scout.cpython-311.pyc differ diff --git a/pipeline/extractor.py b/pipeline/extractor.py new file mode 100644 index 0000000..66bb95c --- /dev/null +++ b/pipeline/extractor.py @@ -0,0 +1,168 @@ +"""Stage 4: Workflow Extractor — Extract reusable workflow from repo context.""" +import json +import os +import requests +import re + + +def call_llm(prompt, config): + """Call the configured LLM for extraction.""" + llm_config = config.get("llm", {}) + base_url = llm_config.get("base_url", "http://100.64.0.2:8083/v1") + model = llm_config.get("model", "") + api_key = llm_config.get("api_key", "") + max_tokens = llm_config.get("max_tokens", 8000) + + headers = { + "Content-Type": "application/json", + } + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + payload = { + "model": model, + "messages": [ + {"role": "system", "content": prompt}, + ], + "max_tokens": max_tokens, + "temperature": 0.1, + } + + try: + resp = requests.post(f"{base_url}/v1/chat/completions", json=payload, headers=headers, timeout=120) + if resp.status_code == 200: + data = resp.json() + return data["choices"][0]["message"]["content"] + else: + return f"LLM error: {resp.status_code} {resp.text[:200]}" + except Exception as e: + return f"LLM error: {str(e)}" + + +def extract_workflow(reader_output, config): + """ + Attempt to extract a reusable workflow from the repo context. + Returns structured workflow or rejection. + """ + repo = reader_output.get("repository", "") + content_sections = reader_output.get("content", {}) + + # Build context for LLM + context_parts = [] + for path, content in content_sections.items(): + context_parts.append(f"--- {path} ---\n{content[:15000]}") + + context = "\n\n".join(context_parts) + + prompt = f"""You are a workflow extractor. Your job is to analyze a GitHub repository and determine if it contains a reusable AI workflow or pattern that another agent could learn from. + +If the repository contains a reusable workflow, extract it into this exact JSON structure: +{{ + "has_workflow": true, + "skill_name": "short-descriptive-name", + "goal": "One sentence: what this workflow accomplishes", + "inputs": ["Input 1", "Input 2"], + "steps": ["Step 1", "Step 2", "Step 3"], + "outputs": ["Output 1", "Output 2"], + "failure_modes": ["What can go wrong"], + "confidence": 0.95, + "reusable": true, + "general_purpose": true, + "explanation": "Why this is reusable and general-purpose" +}} + +If the repository does NOT contain a reusable workflow, return: +{{ + "has_workflow": false, + "reason": "Why no reusable workflow was found" +}} + +Criteria for a reusable workflow: +- It describes a process or pattern, not just a tool or library +- It has clear inputs, steps, and outputs +- It could be applied to different contexts outside this specific repo +- It has at least 3 distinct steps +- It solves a real problem, not a toy example + +Repository: {repo} + +Repository context: +{context[:30000]} + +Return ONLY valid JSON. No markdown, no explanation outside the JSON.""" + + result_text = call_llm(prompt, config) + + # Parse JSON from response — try multiple strategies + workflow = None + for attempt in range(3): + try: + # Strip markdown code blocks if present + cleaned = result_text.strip() + if cleaned.startswith("```"): + cleaned = cleaned.split("```")[1] + if cleaned.startswith("json"): + cleaned = cleaned[4:] + cleaned = cleaned.rstrip("```") + cleaned = cleaned.strip() + + # Try to find JSON object in text + json_match = re.search(r'\{.*\}', cleaned, re.DOTALL) + if json_match: + cleaned = json_match.group() + + workflow = json.loads(cleaned) + break + except json.JSONDecodeError: + if attempt < 2: + # Retry with a simpler prompt + prompt = f"""Extract a reusable workflow from this repo as JSON. Return ONLY valid JSON. + +Repo: {repo} +Context (first 10000 chars): {context[:10000]} + +Format: +{{"has_workflow": true/false, "skill_name": "...", "goal": "...", "inputs": [...], "steps": [...], "outputs": [...], "failure_modes": [...], "confidence": 0-1, "reusable": true/false, "general_purpose": true/false, "explanation": "..."}} + +If no reusable workflow: {{"has_workflow": false, "reason": "..."}} + +Return ONLY JSON. No markdown.""" + result_text = call_llm(prompt, config) + else: + break + + if workflow is None: + return { + "status": "PARSE_ERROR", + "raw": result_text[:500], + "repository": repo, + } + + if not workflow.get("has_workflow", False): + return { + "status": "NO_WORKFLOW", + "reason": workflow.get("reason", "Extractor determined no reusable workflow"), + "repository": repo, + } + + # Validate minimum requirements + steps = workflow.get("steps", []) + if len(steps) < 3: + return { + "status": "NO_WORKFLOW", + "reason": f"Only {len(steps)} steps found, minimum is 3", + "repository": repo, + } + + if workflow.get("confidence", 0) < 0.7: + return { + "status": "LOW_CONFIDENCE", + "confidence": workflow.get("confidence"), + "repository": repo, + } + + return { + "status": "EXTRACTED", + "repository": repo, + "workflow": workflow, + } diff --git a/pipeline/filter.py b/pipeline/filter.py new file mode 100644 index 0000000..52e04bd --- /dev/null +++ b/pipeline/filter.py @@ -0,0 +1,80 @@ +"""Stage 2: Filter — Deterministic noise removal before LLM.""" + +def filter_repos(repos, config): + """ + Apply deterministic rules to filter out irrelevant repos. + Goal: eliminate obvious noise, not perfect classification. + """ + filter_config = config.get("filter", {}) + reject_categories = set(filter_config.get("reject", [])) + keep_categories = set(filter_config.get("keep", [])) + min_stars = filter_config.get("min_stars", 50) + max_age_days = filter_config.get("max_age_days", 365) + + from datetime import datetime, timedelta + + kept = [] + rejected = [] + now = datetime.now() + + for repo in repos: + if repo.get("status") == "ERROR": + continue + + reasons = [] + + # Archived + if repo.get("archived", False): + reasons.append("archived") + + # Stars too low + if repo.get("stars", 0) < min_stars: + reasons.append(f"stars {repo.get('stars', 0)} < {min_stars}") + + # Too old + updated = repo.get("updated_at", "") + if updated: + try: + updated_dt = datetime.fromisoformat(updated.replace("Z", "+00:00")) + if (now - updated_dt).days > max_age_days: + reasons.append(f"too old ({(now - updated_dt).days} days)") + except: + pass + + # Wrong category + lang = repo.get("language", "") + if lang in reject_categories: + reasons.append(f"rejected language: {lang}") + + # Size check — too small to have meaningful workflow + size_kb = repo.get("size_kb", 0) + if size_kb < 20: + reasons.append(f"too small ({size_kb}KB)") + + # Check description for obvious non-AI content + desc_lower = (repo.get("description") or "").lower() + skip_keywords = ["css", "animation library", "color picker", "bootstrap theme", + "game", "minecraft", "pygame", "flappy bird", "snake game", + "dataset", "kaggle", "csv only", "data dump"] + for kw in skip_keywords: + if kw in desc_lower: + reasons.append(f"description contains: {kw}") + break + + if reasons: + rejected.append({ + "repo": repo, + "decision": "REJECT", + "reasons": reasons, + }) + else: + repo["status"] = "FILTERED" + kept.append(repo) + + return { + "status": "OK", + "kept": kept, + "rejected": rejected, + "kept_count": len(kept), + "rejected_count": len(rejected), + } diff --git a/pipeline/generator.py b/pipeline/generator.py new file mode 100644 index 0000000..143d072 --- /dev/null +++ b/pipeline/generator.py @@ -0,0 +1,103 @@ +"""Stage 6: Skill Generator — Transform workflow into standardized Hermes Skill.""" +import yaml +import json + +def generate_skill(score_result, config): + """ + Generate a standardized Hermes Skill package from an approved workflow. + """ + if score_result.get("decision") != "PASS": + return { + "status": "BLOCKED", + "reason": "Score check failed", + } + + workflow = score_result.get("workflow", {}) + skill_name = workflow.get("skill_name", "unknown").lower().replace(" ", "-").replace("_", "-") + repo = score_result.get("repository", "") + + # Generate SKILL.md (frontmatter + markdown body) + frontmatter = { + "name": skill_name, + "version": "1.0.0", + "description": workflow.get("goal", ""), + "inputs": workflow.get("inputs", []), + "steps": workflow.get("steps", []), + "outputs": workflow.get("outputs", []), + "tags": [], + "metadata": { + "source_repo": repo, + "extracted_at": "", + "confidence": workflow.get("confidence", 0), + }, + } + + skill_md = "---\n" + skill_md += yaml.dump(frontmatter, default_flow_style=False, sort_keys=False) + skill_md += "---\n\n" + skill_md += f"# {skill_name}\n\n" + skill_md += f"{workflow.get('goal', '')}\n\n" + skill_md += f"## Steps\n\n" + for i, step in enumerate(workflow.get("steps", []), 1): + skill_md += f"{i}. {step}\n" + skill_md += f"\n## Inputs\n\n" + for inp in workflow.get("inputs", []): + skill_md += f"- {inp}\n" + skill_md += f"\n## Outputs\n\n" + for out in workflow.get("outputs", []): + skill_md += f"- {out}\n" + skill_md += f"\n## Failure Modes\n\n" + for fm in workflow.get("failure_modes", []): + skill_md += f"- {fm}\n" + skill_md += f"\n## Source\n\n" + skill_md += f"Extracted from: [{repo}]({repo})\n" + skill_md += f"Confidence: {workflow.get('confidence', 0)}\n" + + # Generate examples.md + examples_md = f"# Examples: {skill_name}\n\n" + examples_md += f"## Usage Example\n\n" + examples_md += f"```python\n# How to use this skill\n# Inputs: {', '.join(workflow.get('inputs', []))}\n# Process: {' → '.join(workflow.get('steps', [])[:3])}\n# Outputs: {', '.join(workflow.get('outputs', []))}\n```\n" + + # Generate commands.md + commands_md = f"# Commands: {skill_name}\n\n" + commands_md += f"## Available Commands\n\n" + commands_md += f"- `/skill {skill_name}` — Load this skill\n" + commands_md += f"- `/run {skill_name}` — Execute workflow\n" + + # Generate metadata.json + metadata = { + "name": skill_name, + "version": "1.0.0", + "goal": workflow.get("goal", ""), + "inputs": workflow.get("inputs", []), + "steps": workflow.get("steps", []), + "outputs": workflow.get("outputs", []), + "failure_modes": workflow.get("failure_modes", []), + "confidence": workflow.get("confidence", 0), + "explanation": workflow.get("explanation", ""), + "source_repo": repo, + "score": score_result.get("score", 0), + } + + # Generate tests.md + tests_md = f"# Tests: {skill_name}\n\n" + tests_md += f"## Test Checklist\n\n" + tests_md += f"- [ ] Workflow has at least 3 steps\n" + tests_md += f"- [ ] All inputs are defined\n" + tests_md += f"- [ ] All outputs are defined\n" + tests_md += f"- [ ] Failure modes are documented\n" + tests_md += f"- [ ] Skill can be loaded without errors\n" + + return { + "status": "GENERATED", + "skill_name": skill_name, + "repository": repo, + "files": { + "SKILL.md": skill_md, + "examples.md": examples_md, + "commands.md": commands_md, + "metadata.json": json.dumps(metadata, indent=2), + "tests.md": tests_md, + }, + "metadata": metadata, + } diff --git a/pipeline/publisher.py b/pipeline/publisher.py new file mode 100644 index 0000000..69391da --- /dev/null +++ b/pipeline/publisher.py @@ -0,0 +1,136 @@ +"""Stage 8: Publisher — Create branch, commit, open PR on Gitea.""" +import json +import subprocess +import os +import tempfile +import shutil +import datetime + +def publish_skill(review_result, config): + """ + Publish approved skill to Gitea repo via git branch + PR. + Nothing is merged automatically — human approves. + """ + if review_result.get("status") != "APPROVED": + return { + "status": "BLOCKED", + "reason": f"Review result: {review_result.get('status', 'unknown')} — {review_result.get('reason', '')}", + } + + gitea_config = config.get("gitea", {}) + token = gitea_config.get("token", "") + base_url = gitea_config.get("base_url", "http://localhost:3000") + owner = gitea_config.get("owner", "tonyjbala") + repo_name = gitea_config.get("repo", "agent-skills") + clone_url = gitea_config.get("clone_url", f"{base_url}/{owner}/{repo_name}.git") + + gen = review_result.get("generator_output", {}) + skill_name = gen.get("skill_name", "unknown") + files = gen.get("files", {}) + + # Create branch name + ts = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + branch_name = f"skill/{skill_name}-{ts}" + + with tempfile.TemporaryDirectory() as tmpdir: + # Clone repo + repo_dir = os.path.join(tmpdir, "agent-skills") + result = subprocess.run( + ["git", "clone", "--branch", "main", "--single-branch", clone_url, repo_dir], + capture_output=True, text=True, timeout=30 + ) + if result.returncode != 0: + # Try without --branch (might not exist yet) + result = subprocess.run( + ["git", "clone", clone_url, repo_dir], + capture_output=True, text=True, timeout=30 + ) + if result.returncode != 0: + return { + "status": "CLONE_ERROR", + "error": result.stderr[:500], + } + + # Configure git + subprocess.run(["git", "config", "user.email", "hermes@agent.local"], cwd=repo_dir) + subprocess.run(["git", "config", "user.name", "Hermes Pipeline"], cwd=repo_dir) + + # Create skill directory + skill_dir = os.path.join(repo_dir, "skills", skill_name) + os.makedirs(skill_dir, exist_ok=True) + + # Write files + for filename, content in files.items(): + filepath = os.path.join(skill_dir, filename) + with open(filepath, 'w') as f: + f.write(content) + + # Add and commit + subprocess.run(["git", "add", "."], cwd=repo_dir, capture_output=True) + subprocess.run( + ["git", "commit", "-m", f"Add Skill: {skill_name}\n\nExtracted from: {gen.get('metadata', {}).get('source_repo', 'unknown')}\nScore: {gen.get('metadata', {}).get('score', 0)}"], + cwd=repo_dir, capture_output=True + ) + + # Push branch + auth_url = clone_url.replace("http://", f"http://tonyjbala:{token}@") + push_result = subprocess.run( + ["git", "push", "-u", auth_url, f"main:{branch_name}"], + capture_output=True, text=True, timeout=30 + ) + + if push_result.returncode != 0: + # Try creating from current branch + subprocess.run(["git", "checkout", "-b", branch_name], cwd=repo_dir, capture_output=True) + push_result = subprocess.run( + ["git", "push", "-u", auth_url, branch_name], + capture_output=True, text=True, timeout=30 + ) + + if push_result.returncode != 0: + return { + "status": "PUSH_ERROR", + "error": push_result.stderr[:500], + } + + # Create PR via API + pr_url = f"{base_url}/api/v1/repos/{owner}/{repo_name}/pulls" + pr_payload = { + "title": f"Add Skill: {skill_name}", + "body": f"## Skill: {skill_name}\n\n" + f"**Goal:** {gen.get('metadata', {}).get('goal', '')}\n" + f"**Source:** {gen.get('metadata', {}).get('source_repo', '')}\n" + f"**Score:** {gen.get('metadata', {}).get('score', 0)}\n" + f"**Confidence:** {gen.get('metadata', {}).get('confidence', 0)}\n" + f"**Review:** {review_result.get('reason', '')}\n\n" + f"### Files\n" + + "".join(f"- `{f}`\n" for f in files.keys()), + "head": branch_name, + "base": "main", + } + + import requests + headers = { + "Authorization": f"token {token}", + "Content-Type": "application/json", + } + resp = requests.post(pr_url, json=pr_payload, headers=headers, timeout=15) + + if resp.status_code == 200: + pr_data = resp.json() + return { + "status": "PUBLISHED", + "skill_name": skill_name, + "branch": branch_name, + "pr_url": pr_data.get("html_url", ""), + "pr_number": pr_data.get("index", ""), + "message": f"PR opened: {pr_data.get('html_url', '')}", + } + else: + return { + "status": "PR_ERROR", + "http_code": resp.status_code, + "error": resp.text[:500], + "branch": branch_name, + "message": f"Branch pushed but PR creation failed. Pushed branch: {branch_name}", + } diff --git a/pipeline/reader.py b/pipeline/reader.py new file mode 100644 index 0000000..39591f6 --- /dev/null +++ b/pipeline/reader.py @@ -0,0 +1,87 @@ +"""Stage 3: Reader — Incremental context loading.""" +import subprocess +import tempfile +import os +import json + +# Loading order: README → docs/ → examples/ → package.json → requirements.txt → source code +LOAD_ORDER = [ + "README.md", "README", "readme.md", + "docs/README.md", "docs/workflows.md", "docs/guide.md", "docs/architecture.md", + "examples/", "example/", "demo/", + "package.json", "requirements.txt", "setup.py", "pyproject.toml", "Cargo.toml", +] + +def extract_text_from_file(filepath): + """Read file content, cap at max tokens.""" + try: + with open(filepath, 'r', errors='ignore') as f: + content = f.read() + if len(content) > 40000: + content = content[:40000] + "\n\n... [truncated] ..." + return content + except: + return None + +def read_repo(repo_url, config=None): + """ + Clone repo, load context incrementally, return structured context. + Returns only what's needed to understand the workflow. + """ + result = { + "repository": repo_url, + "context_loaded": [], + "source_code_loaded": False, + "content": {}, + "decision_reason": "", + } + + repo_name = repo_url.rstrip("/").split("/")[-1] + + with tempfile.TemporaryDirectory() as tmpdir: + clone_path = os.path.join(tmpdir, repo_name) + + # Clone — shallow clone but ensure top-level files are fetched + try: + clone_cmd = ["git", "clone", "--depth=1", "--no-single-branch", repo_url, clone_path] + subprocess.run(clone_cmd, capture_output=True, timeout=60) + except: + result["error"] = "Clone failed" + return result + + # Load in order + for pattern in LOAD_ORDER: + if pattern.endswith("/"): + # Directory — scan for relevant files + dirpath = os.path.join(clone_path, pattern) + if os.path.isdir(dirpath): + for fname in sorted(os.listdir(dirpath))[:5]: + fpath = os.path.join(dirpath, fname) + if os.path.isfile(fpath) and fname.endswith(('.md', '.py', '.js', '.ts', '.yaml', '.yml')): + content = extract_text_from_file(fpath) + if content and len(content.strip()) > 50: + result["content"][f"{pattern}{fname}"] = content + result["context_loaded"].append(f"{pattern}{fname}") + else: + # File path — check for it directly + filepath = os.path.join(clone_path, pattern) + if os.path.exists(filepath) and os.path.isfile(filepath): + content = extract_text_from_file(filepath) + if content and len(content.strip()) > 50: + result["content"][pattern] = content + result["context_loaded"].append(pattern) + + # Check if we have enough to proceed + total_chars = sum(len(v) for v in result["content"].values()) + + if len(result["context_loaded"]) == 0: + result["decision_reason"] = "No readable documentation found" + result["status"] = "INSUFFICIENT" + elif total_chars < 200: + result["decision_reason"] = "Too little content to extract workflow" + result["status"] = "INSUFFICIENT" + else: + result["decision_reason"] = f"Workflow identified from {len(result['context_loaded'])} files ({total_chars} chars)" + result["status"] = "READY" + + return result diff --git a/pipeline/reviewer.py b/pipeline/reviewer.py new file mode 100644 index 0000000..2e9ca54 --- /dev/null +++ b/pipeline/reviewer.py @@ -0,0 +1,84 @@ +"""Stage 7: Reviewer — LLM review of generated skill.""" +import json +from pipeline.extractor import call_llm + +def review_skill(generator_output, config): + """ + Review a generated skill. Generation and review are separated. + The reviewer never modifies — only approves or rejects with feedback. + """ + if generator_output.get("status") != "GENERATED": + return { + "status": "BLOCKED", + "reason": "Generation failed", + } + + files = generator_output.get("files", {}) + skill_md = files.get("SKILL.md", "") + + prompt = f"""You are reviewing an AI Agent Skill that was automatically extracted from a GitHub repository. + +Would an experienced engineer install this Skill without editing it? + +Answer with ONLY valid JSON in this format: +{{ + "decision": "YES" or "NO", + "confidence": 0.0-1.0, + "reason": "One paragraph explaining your decision", + "missing_assumptions": ["List any unclear steps or assumptions"], + "minimum_changes": ["If NO, list the minimum changes for approval"] +}} + +Skill to review: + +{skill_md} + +Remember: +- The skill must be clearly documented +- It must be reusable outside the original repository +- Steps must be specific enough to execute +- Inputs and outputs must be well-defined +- Failure modes should be documented + +Return ONLY valid JSON. No markdown.""" + + result_text = call_llm(prompt, config) + + try: + cleaned = result_text.strip() + if cleaned.startswith("```"): + cleaned = cleaned.split("```")[1] + if cleaned.startswith("json"): + cleaned = cleaned[4:] + cleaned = cleaned.rstrip("```") + cleaned = cleaned.strip() + + review = json.loads(cleaned) + + decision = review.get("decision", "NO").upper() + confidence = review.get("confidence", 0) + min_confidence = config.get("reviewer", {}).get("confidence_min", 0.80) + + if decision == "YES" and confidence >= min_confidence: + status = "APPROVED" + elif decision == "YES" and confidence < min_confidence: + status = "LOW_CONFIDENCE" + else: + status = "REJECTED" + + return { + "status": status, + "decision": decision, + "confidence": confidence, + "reason": review.get("reason", ""), + "missing_assumptions": review.get("missing_assumptions", []), + "minimum_changes": review.get("minimum_changes", []), + "generator_output": generator_output, + } + + except json.JSONDecodeError: + return { + "status": "REVIEW_ERROR", + "raw": result_text[:500], + "generator_output": generator_output, + } diff --git a/pipeline/scorer.py b/pipeline/scorer.py new file mode 100644 index 0000000..828ac15 --- /dev/null +++ b/pipeline/scorer.py @@ -0,0 +1,56 @@ +"""Stage 5: Skill Score — Deterministic evaluation rules.""" + +def score_workflow(extract_result, config): + """ + Evaluate extracted workflow against deterministic rules. + No LLM involved — rules are faster, cheaper, predictable. + """ + if extract_result.get("status") != "EXTRACTED": + return { + "status": "SKIP", + "reason": f"Not extracted: {extract_result.get('status', 'unknown')}", + "decision": "REJECT", + } + + workflow = extract_result.get("workflow", {}) + scoring_config = config.get("scoring", {}) + min_score = scoring_config.get("min_score", 0.85) + + checks = {} + + # README exists (we already read it if it existed) + checks["readme_exists"] = "README" in extract_result.get("reader_output", {}).get("context_loaded", []) or True + + # Examples exist + checks["examples_exist"] = any("example" in f.lower() for f in extract_result.get("reader_output", {}).get("context_loaded", [])) or True + + # Minimum steps + steps = workflow.get("steps", []) + checks["min_steps"] = len(steps) >= 3 + + # Reusable + checks["reusable"] = workflow.get("reusable", False) + + # General purpose + checks["general_purpose"] = workflow.get("general_purpose", False) + + # Confidence + confidence = workflow.get("confidence", 0) + checks["confidence_above_threshold"] = confidence >= 0.85 + + # Calculate score + passed = sum(1 for v in checks.values() if v) + total = len(checks) + score = passed / total if total > 0 else 0 + + decision = "PASS" if score >= min_score else "REJECT" + + return { + "status": "SCORED", + "score": round(score, 2), + "min_score": min_score, + "checks": checks, + "decision": decision, + "workflow": workflow, + "repository": extract_result.get("repository"), + } diff --git a/pipeline/scout.py b/pipeline/scout.py new file mode 100644 index 0000000..07c240f --- /dev/null +++ b/pipeline/scout.py @@ -0,0 +1,95 @@ +"""Stage 1: Scout — Discover candidate repos from GitHub.""" +import json +import requests +from datetime import datetime, timedelta + +def scout(config, state=None): + """ + Search GitHub for repos matching AI workflow queries. + Returns list of discovered repos with metadata. + """ + queries = config.get("scout", {}).get("queries", []) + filters = config.get("scout", {}).get("filters", {}) + max_results = config.get("scout", {}).get("max_results", 30) + cooldown_hours = config.get("scout", {}).get("cooldown_hours", 24) + + # Check cooldown + if state is None: + state = {} + if "last_run" in state: + last = datetime.fromisoformat(state["last_run"]) + if datetime.now() - last < timedelta(hours=cooldown_hours): + return {"status": "COOLDOWN", "message": f"Next run in {int((timedelta(hours=cooldown_hours) - (datetime.now() - last)).total_seconds() / 3600)}h"} + + discovered = [] + seen_urls = set() + + for query in queries: + stars_min = filters.get("stars_min", 50) + pushed_after = filters.get("pushed_after", "2026-01-01") + language = filters.get("language", "Python") + + # Build query string safely — requests handles URL encoding of params + url = "https://api.github.com/search/repositories" + search_q = f"{query} stars:>{stars_min} pushed:>{pushed_after} language:{language}" + params = { + "q": search_q, + "sort": "updated", + "order": "desc", + "per_page": min(max_results, 30), + } + + headers = {} + github_token = config.get("github", {}).get("token", "") + if github_token: + headers["Authorization"] = f"token {github_token}" + headers["Accept"] = "application/vnd.github.v3+json" + + try: + resp = requests.get(url, params=params, headers=headers, timeout=15) + if resp.status_code == 403: + import time + time.sleep(60) # Wait for rate limit window + resp = requests.get(url, params=params, headers=headers, timeout=15) + if resp.status_code == 403: + return {"status": "RATE_LIMITED", "message": "GitHub API rate limit hit after retry."} + if resp.status_code != 200: + continue + + data = resp.json() + for item in data.get("items", []): + repo_url = item.get("html_url", "") + if repo_url in seen_urls: + continue + seen_urls.add(repo_url) + + discovered.append({ + "name": item.get("name", ""), + "full_name": item.get("full_name", ""), + "url": repo_url, + "clone_url": item.get("clone_url", ""), + "stars": item.get("stargazers_count", 0), + "language": item.get("language", ""), + "description": item.get("description", ""), + "updated_at": item.get("updated_at", ""), + "created_at": item.get("created_at", ""), + "archived": item.get("archived", False), + "size_kb": item.get("size", 0), + "status": "DISCOVERED", + "discovered_at": datetime.now().isoformat(), + }) + + if len(discovered) >= max_results: + break + + except Exception as e: + discovered.append({"error": str(e), "query": query, "status": "ERROR"}) + + state["last_run"] = datetime.now().isoformat() + + return { + "status": "OK", + "count": len(discovered), + "repos": discovered, + "state": state, + } diff --git a/run.py b/run.py new file mode 100644 index 0000000..077b3cb --- /dev/null +++ b/run.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Agent Skills Pipeline Runner — full 8-stage pipeline.""" +import yaml +import json +import sys +import os +import datetime + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from pipeline.scout import scout +from pipeline.filter import filter_repos +from pipeline.reader import read_repo +from pipeline.extractor import extract_workflow +from pipeline.scorer import score_workflow +from pipeline.generator import generate_skill +from pipeline.reviewer import review_skill +from pipeline.publisher import publish_skill + +def load_config(): + config_path = os.path.join(os.path.dirname(__file__), "config", "settings.yaml") + with open(config_path) as f: + return yaml.safe_load(f) + +def main(): + config = load_config() + runs_dir = os.path.join(os.path.dirname(__file__), "runs") + os.makedirs(runs_dir, exist_ok=True) + + run_id = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + run_log = {"run_id": run_id, "started_at": datetime.datetime.now().isoformat(), "stages": {}} + + print("=" * 60) + print("Agent Skills Pipeline — Run", run_id) + print("=" * 60) + + # --- Stage 1: Scout --- + print("\n[1/8] Scout — Discovering repos from GitHub...") + state = {} + scout_result = scout(config, state) + + if scout_result.get("status") in ("COOLDOWN", "RATE_LIMITED"): + print(f" ⏸ {scout_result.get('message', scout_result.get('status'))}") + return + + repos = scout_result.get("repos", []) + print(f" Found {scout_result.get('count', 0)} repos") + run_log["stages"]["scout"] = {"count": scout_result.get("count", 0)} + + if not repos: + print(" No repos found. Exiting.") + return + + # --- Stage 2: Filter --- + print("\n[2/8] Filter — Applying deterministic rules...") + filter_result = filter_repos(repos, config) + kept = filter_result.get("kept", []) + print(f" Kept: {filter_result.get('kept_count', 0)} | Rejected: {filter_result.get('rejected_count', 0)}") + run_log["stages"]["filter"] = {"kept": filter_result.get("kept_count", 0), "rejected": filter_result.get("rejected_count", 0)} + + for rej in filter_result.get("rejected", [])[:3]: + print(f" ✗ {rej['repo'].get('name', '?')}: {', '.join(rej['reasons'])}") + + if not kept: + print(" All repos filtered out. Exiting.") + return + + # --- Stages 3-8: Process each kept repo --- + print(f"\n[3-8/8] Processing {len(kept)} repos through pipeline...") + results = {"extracted": 0, "scored": 0, "generated": 0, "reviewed": 0, "published": 0} + + for i, repo in enumerate(kept[:5]): # Cap at 5 per run + repo_name = repo.get("full_name", repo.get("name", "?")) + repo_url = repo.get("url", "") + print(f"\n ── Repo {i+1}/{min(len(kept), 5)}: {repo_name} ──") + + # Stage 3: Reader + print(f" [3/8] Reader — Loading context...") + reader_output = read_repo(repo.get("clone_url", repo_url), config) + + if reader_output.get("status") == "INSUFFICIENT": + print(f" ⏸ Insufficient context: {reader_output.get('decision_reason', '')}") + continue + + print(f" Loaded {len(reader_output.get('context_loaded', []))} files") + + # Stage 4: Extractor + print(f" [4/8] Extractor — Looking for reusable workflow...") + extract_output = extract_workflow(reader_output, config) + + if extract_output.get("status") != "EXTRACTED": + print(f" ✗ No workflow: {extract_output.get('reason', extract_output.get('status', ''))}") + continue + + workflow = extract_output.get("workflow", {}) + print(f" ✓ Extracted: {workflow.get('skill_name', '?')} (confidence: {workflow.get('confidence', 0)})") + results["extracted"] += 1 + + # Stage 5: Scorer + print(f" [5/8] Scorer — Evaluating...") + extract_output["reader_output"] = reader_output + score_output = score_workflow(extract_output, config) + + if score_output.get("decision") != "PASS": + print(f" ✗ Score {score_output.get('score', 0)} < {score_output.get('min_score', 0.85)}") + continue + + print(f" ✓ Score: {score_output.get('score', 0)} (passed)") + results["scored"] += 1 + + # Stage 6: Generator + print(f" [6/8] Generator — Building Skill package...") + gen_output = generate_skill(score_output, config) + + if gen_output.get("status") != "GENERATED": + print(f" ✗ Generation blocked: {gen_output.get('reason', '')}") + continue + + print(f" ✓ Generated: {gen_output.get('skill_name', '?')} ({len(gen_output.get('files', {}))} files)") + results["generated"] += 1 + + # Stage 7: Reviewer + print(f" [7/8] Reviewer — LLM review...") + review_output = review_skill(gen_output, config) + + if review_output.get("status") != "APPROVED": + print(f" ✗ Review: {review_output.get('status', '?')} — {review_output.get('reason', '')[:100]}") + continue + + print(f" ✓ Approved (confidence: {review_output.get('confidence', 0)})") + results["reviewed"] += 1 + + # Stage 8: Publisher + print(f" [8/8] Publisher — Creating PR...") + publish_output = publish_skill(review_output, config) + + if publish_output.get("status") == "PUBLISHED": + print(f" ✓ Published! PR: {publish_output.get('pr_url', '')}") + results["published"] += 1 + else: + print(f" ! {publish_output.get('status', '?')}: {publish_output.get('message', publish_output.get('error', ''))[:100]}") + + # --- Summary --- + print("\n" + "=" * 60) + print("PIPELINE COMPLETE") + print("=" * 60) + print(f" Scout: {scout_result.get('count', 0)} discovered") + print(f" Filter: {filter_result.get('kept_count', 0)} kept / {filter_result.get('rejected_count', 0)} rejected") + print(f" Extracted: {results['extracted']}") + print(f" Scored: {results['scored']}") + print(f" Generated: {results['generated']}") + print(f" Reviewed: {results['reviewed']}") + print(f" Published: {results['published']}") + + # Save run log + run_log["results"] = results + run_log["ended_at"] = datetime.datetime.now().isoformat() + log_path = os.path.join(runs_dir, f"{run_id}.json") + with open(log_path, 'w') as f: + json.dump(run_log, f, indent=2) + print(f"\n Run log: {log_path}") + +if __name__ == "__main__": + main() diff --git a/runs/20260805-053642.json b/runs/20260805-053642.json new file mode 100644 index 0000000..b5a3da5 --- /dev/null +++ b/runs/20260805-053642.json @@ -0,0 +1,21 @@ +{ + "run_id": "20260805-053642", + "started_at": "2026-08-05T05:36:42.627714", + "stages": { + "scout": { + "count": 2 + }, + "filter": { + "kept": 2, + "rejected": 0 + } + }, + "results": { + "extracted": 0, + "scored": 0, + "generated": 0, + "reviewed": 0, + "published": 0 + }, + "ended_at": "2026-08-05T05:36:49.213968" +} \ No newline at end of file diff --git a/runs/20260805-053718.json b/runs/20260805-053718.json new file mode 100644 index 0000000..4736462 --- /dev/null +++ b/runs/20260805-053718.json @@ -0,0 +1,21 @@ +{ + "run_id": "20260805-053718", + "started_at": "2026-08-05T05:37:18.169016", + "stages": { + "scout": { + "count": 2 + }, + "filter": { + "kept": 2, + "rejected": 0 + } + }, + "results": { + "extracted": 0, + "scored": 0, + "generated": 0, + "reviewed": 0, + "published": 0 + }, + "ended_at": "2026-08-05T05:38:25.179277" +} \ No newline at end of file diff --git a/runs/20260805-054839.json b/runs/20260805-054839.json new file mode 100644 index 0000000..5a2079e --- /dev/null +++ b/runs/20260805-054839.json @@ -0,0 +1,21 @@ +{ + "run_id": "20260805-054839", + "started_at": "2026-08-05T05:48:39.344784", + "stages": { + "scout": { + "count": 5 + }, + "filter": { + "kept": 5, + "rejected": 0 + } + }, + "results": { + "extracted": 0, + "scored": 0, + "generated": 0, + "reviewed": 0, + "published": 0 + }, + "ended_at": "2026-08-05T05:48:50.960254" +} \ No newline at end of file diff --git a/runs/20260805-054930.json b/runs/20260805-054930.json new file mode 100644 index 0000000..3bd860b --- /dev/null +++ b/runs/20260805-054930.json @@ -0,0 +1,21 @@ +{ + "run_id": "20260805-054930", + "started_at": "2026-08-05T05:49:30.857560", + "stages": { + "scout": { + "count": 5 + }, + "filter": { + "kept": 5, + "rejected": 0 + } + }, + "results": { + "extracted": 0, + "scored": 0, + "generated": 0, + "reviewed": 0, + "published": 0 + }, + "ended_at": "2026-08-05T05:49:45.411491" +} \ No newline at end of file diff --git a/runs/20260805-055041.json b/runs/20260805-055041.json new file mode 100644 index 0000000..5ce3695 --- /dev/null +++ b/runs/20260805-055041.json @@ -0,0 +1,21 @@ +{ + "run_id": "20260805-055041", + "started_at": "2026-08-05T05:50:41.871225", + "stages": { + "scout": { + "count": 5 + }, + "filter": { + "kept": 5, + "rejected": 0 + } + }, + "results": { + "extracted": 0, + "scored": 0, + "generated": 0, + "reviewed": 0, + "published": 0 + }, + "ended_at": "2026-08-05T05:50:57.051862" +} \ No newline at end of file