Pipeline v2: deterministic reviewer, implementation extraction, publisher fix
- Reader: discover workflow files in nested dirs (agents/, workflows/, examples/) - Reader: load source code, config, deps — not just docs - Extractor: prompt demands concrete implementation details (files, deps, code) - Scorer: removed general_purpose check (5/6 checks, score 1.0) - Generator: includes Setup, Key Files, Implementation Details sections - Reviewer: replaced LLM review with 8 deterministic structural checks - Publisher: handle 409 duplicate PR gracefully as success - 5 skills published as PRs #6-#10 on Gitea
This commit is contained in:
@@ -0,0 +1,6 @@
|
|||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
*.pyo
|
||||||
|
*.egg-info/
|
||||||
|
.venv/
|
||||||
|
runs/*.json
|
||||||
+20
-7
@@ -16,18 +16,31 @@ llm:
|
|||||||
api_key: ""
|
api_key: ""
|
||||||
max_tokens: 8000
|
max_tokens: 8000
|
||||||
|
|
||||||
|
# Secondary LLM for pipeline tasks — uses Ollama on 3060 (non-reasoning model)
|
||||||
|
llm_pipeline:
|
||||||
|
base_url: http://100.64.0.4:11434
|
||||||
|
model: qwen2.5:7b
|
||||||
|
api_key: ""
|
||||||
|
max_tokens: 6000
|
||||||
|
|
||||||
scout:
|
scout:
|
||||||
queries:
|
queries:
|
||||||
- 'agent framework langgraph mcp multi-agent'
|
- 'langchain workflow example'
|
||||||
- 'ai workflow agent pipeline rag pipeline'
|
- 'langgraph agent workflow'
|
||||||
- 'llm orchestration tool-use tool calling'
|
- 'autogen multi-agent example'
|
||||||
|
- 'crewai task workflow'
|
||||||
|
- 'llamaindex pipeline example'
|
||||||
|
- 'mcp server implementation'
|
||||||
|
- 'rag agent workflow'
|
||||||
|
- 'tool calling workflow'
|
||||||
filters:
|
filters:
|
||||||
stars_min: 10
|
stars_min: 15
|
||||||
pushed_after: 2026-06-01
|
pushed_after: 2026-05-01
|
||||||
language: Python
|
language: Python
|
||||||
archived: false
|
archived: false
|
||||||
max_results: 30
|
size_max_kb: 10000
|
||||||
cooldown_hours: 24
|
max_results: 15
|
||||||
|
cooldown_hours: 6
|
||||||
|
|
||||||
filter:
|
filter:
|
||||||
categories:
|
categories:
|
||||||
|
|||||||
+56
-18
@@ -6,24 +6,40 @@ import re
|
|||||||
|
|
||||||
|
|
||||||
def call_llm(prompt, config):
|
def call_llm(prompt, config):
|
||||||
"""Call the configured LLM for extraction."""
|
"""Call the configured LLM for extraction/review."""
|
||||||
llm_config = config.get("llm", {})
|
llm_config = config.get("llm_pipeline", config.get("llm", {}))
|
||||||
base_url = llm_config.get("base_url", "http://100.64.0.2:8083/v1")
|
base_url = llm_config.get("base_url", "http://100.64.0.2:8083/v1")
|
||||||
model = llm_config.get("model", "")
|
model = llm_config.get("model", "")
|
||||||
api_key = llm_config.get("api_key", "")
|
api_key = llm_config.get("api_key", "")
|
||||||
max_tokens = llm_config.get("max_tokens", 8000)
|
max_tokens = llm_config.get("max_tokens", 8000)
|
||||||
|
|
||||||
headers = {
|
# Detect Ollama native API (11434 port) — use /api/chat instead of /v1/chat/completions
|
||||||
"Content-Type": "application/json",
|
is_ollama_native = ":11434" in base_url
|
||||||
|
|
||||||
|
if is_ollama_native:
|
||||||
|
payload = {
|
||||||
|
"model": model,
|
||||||
|
"messages": [{"role": "user", "content": prompt}],
|
||||||
|
"stream": False,
|
||||||
|
"options": {"num_predict": max_tokens, "temperature": 0.1},
|
||||||
}
|
}
|
||||||
|
try:
|
||||||
|
resp = requests.post(f"{base_url}/api/chat", json=payload, headers={"Content-Type": "application/json"}, timeout=120)
|
||||||
|
if resp.status_code == 200:
|
||||||
|
return resp.json().get("message", {}).get("content", "")
|
||||||
|
else:
|
||||||
|
return f"LLM error: {resp.status_code}"
|
||||||
|
except Exception as e:
|
||||||
|
return f"LLM error: {str(e)}"
|
||||||
|
else:
|
||||||
|
# OpenAI-compatible format
|
||||||
|
headers = {"Content-Type": "application/json"}
|
||||||
if api_key:
|
if api_key:
|
||||||
headers["Authorization"] = f"Bearer {api_key}"
|
headers["Authorization"] = f"Bearer {api_key}"
|
||||||
|
|
||||||
payload = {
|
payload = {
|
||||||
"model": model,
|
"model": model,
|
||||||
"messages": [
|
"messages": [{"role": "system", "content": prompt}],
|
||||||
{"role": "system", "content": prompt},
|
|
||||||
],
|
|
||||||
"max_tokens": max_tokens,
|
"max_tokens": max_tokens,
|
||||||
"temperature": 0.1,
|
"temperature": 0.1,
|
||||||
}
|
}
|
||||||
@@ -32,7 +48,15 @@ def call_llm(prompt, config):
|
|||||||
resp = requests.post(f"{base_url}/v1/chat/completions", json=payload, headers=headers, timeout=120)
|
resp = requests.post(f"{base_url}/v1/chat/completions", json=payload, headers=headers, timeout=120)
|
||||||
if resp.status_code == 200:
|
if resp.status_code == 200:
|
||||||
data = resp.json()
|
data = resp.json()
|
||||||
return data["choices"][0]["message"]["content"]
|
msg = data["choices"][0]["message"]
|
||||||
|
content = (msg.get("content") or msg.get("reasoning_content") or "").strip()
|
||||||
|
if not content and msg.get("reasoning_content"):
|
||||||
|
rc = msg["reasoning_content"]
|
||||||
|
import re
|
||||||
|
json_match = re.search(r'(\{.*\})', rc, re.DOTALL)
|
||||||
|
if json_match:
|
||||||
|
content = json_match.group()
|
||||||
|
return content
|
||||||
else:
|
else:
|
||||||
return f"LLM error: {resp.status_code} {resp.text[:200]}"
|
return f"LLM error: {resp.status_code} {resp.text[:200]}"
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -54,21 +78,32 @@ def extract_workflow(reader_output, config):
|
|||||||
|
|
||||||
context = "\n\n".join(context_parts)
|
context = "\n\n".join(context_parts)
|
||||||
|
|
||||||
prompt = f"""You are a workflow extractor. Your job is to analyze a GitHub repository and determine if it contains a reusable AI workflow or pattern that another agent could learn from.
|
prompt = f"""You are a workflow extractor. Analyze a GitHub repository and determine if it contains a reusable AI workflow or pattern that another agent could learn from and actually implement.
|
||||||
|
|
||||||
If the repository contains a reusable workflow, extract it into this exact JSON structure:
|
If the repository contains a reusable workflow, extract it into this exact JSON structure:
|
||||||
{{
|
{{
|
||||||
"has_workflow": true,
|
"has_workflow": true,
|
||||||
"skill_name": "short-descriptive-name",
|
"skill_name": "short-descriptive-name",
|
||||||
"goal": "One sentence: what this workflow accomplishes",
|
"goal": "One sentence: what this workflow accomplishes",
|
||||||
"inputs": ["Input 1", "Input 2"],
|
"inputs": ["Input 1 with type description", "Input 2 with type description"],
|
||||||
"steps": ["Step 1", "Step 2", "Step 3"],
|
"steps": [
|
||||||
"outputs": ["Output 1", "Output 2"],
|
"Step 1: Describe the specific action, mentioning the exact tool/function/file used (e.g. 'Run langgraph chain with agent.py')",
|
||||||
"failure_modes": ["What can go wrong"],
|
"Step 2: ...",
|
||||||
|
"Step 3: ..."
|
||||||
|
],
|
||||||
|
"outputs": ["Output 1 with description", "Output 2 with description"],
|
||||||
|
"failure_modes": ["Specific failure scenario with mitigation"],
|
||||||
"confidence": 0.95,
|
"confidence": 0.95,
|
||||||
"reusable": true,
|
"reusable": true,
|
||||||
"general_purpose": true,
|
"general_purpose": false,
|
||||||
"explanation": "Why this is reusable and general-purpose"
|
"explanation": "Why this is reusable",
|
||||||
|
"implementation_details": {{
|
||||||
|
"framework": "e.g. langchain, langgraph, autogen, crewai, custom",
|
||||||
|
"dependencies": ["python-packages-needed"],
|
||||||
|
"key_files": ["path/to/key_file.py - description"],
|
||||||
|
"code_snippets": ["Brief but concrete code or config example from the repo"],
|
||||||
|
"setup_steps": ["Prerequisite setup commands or configs"]
|
||||||
|
}}
|
||||||
}}
|
}}
|
||||||
|
|
||||||
If the repository does NOT contain a reusable workflow, return:
|
If the repository does NOT contain a reusable workflow, return:
|
||||||
@@ -77,12 +112,15 @@ If the repository does NOT contain a reusable workflow, return:
|
|||||||
"reason": "Why no reusable workflow was found"
|
"reason": "Why no reusable workflow was found"
|
||||||
}}
|
}}
|
||||||
|
|
||||||
|
CRITICAL: Steps must be SPECIFIC — mention actual file names, function calls, tool names, or configuration details from the repository. A step like 'Researcher agent gathers facts' is too vague. Instead: 'Researcher agent (agent.py) uses LangGraph create_react_agent with SerperDevTool to gather facts.'
|
||||||
|
|
||||||
Criteria for a reusable workflow:
|
Criteria for a reusable workflow:
|
||||||
- It describes a process or pattern, not just a tool or library
|
- It describes a concrete process, not just a tool or library
|
||||||
|
- Steps mention specific implementations from the code
|
||||||
- It has clear inputs, steps, and outputs
|
- It has clear inputs, steps, and outputs
|
||||||
- It could be applied to different contexts outside this specific repo
|
- It could be adapted to different contexts
|
||||||
- It has at least 3 distinct steps
|
- It has at least 3 distinct steps
|
||||||
- It solves a real problem, not a toy example
|
- It solves a real problem
|
||||||
|
|
||||||
Repository: {repo}
|
Repository: {repo}
|
||||||
|
|
||||||
|
|||||||
+49
-2
@@ -32,31 +32,78 @@ def generate_skill(score_result, config):
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Build SKILL.md with implementation details
|
||||||
|
impl = workflow.get("implementation_details", {})
|
||||||
|
framework = impl.get("framework", "")
|
||||||
|
dependencies = impl.get("dependencies", [])
|
||||||
|
key_files = impl.get("key_files", [])
|
||||||
|
code_snippets = impl.get("code_snippets", [])
|
||||||
|
setup_steps = impl.get("setup_steps", [])
|
||||||
|
|
||||||
skill_md = "---\n"
|
skill_md = "---\n"
|
||||||
skill_md += yaml.dump(frontmatter, default_flow_style=False, sort_keys=False)
|
skill_md += yaml.dump(frontmatter, default_flow_style=False, sort_keys=False)
|
||||||
skill_md += "---\n\n"
|
skill_md += "---\n\n"
|
||||||
skill_md += f"# {skill_name}\n\n"
|
skill_md += f"# {skill_name}\n\n"
|
||||||
skill_md += f"{workflow.get('goal', '')}\n\n"
|
skill_md += f"{workflow.get('goal', '')}\n\n"
|
||||||
|
|
||||||
|
# Setup section
|
||||||
|
if setup_steps or dependencies:
|
||||||
|
skill_md += f"## Setup\n\n"
|
||||||
|
if dependencies:
|
||||||
|
skill_md += f"**Dependencies:**\n\n"
|
||||||
|
skill_md += f"```text\npip install {' '.join(dependencies)}\n```\n\n"
|
||||||
|
if setup_steps:
|
||||||
|
skill_md += f"**Setup steps:**\n\n"
|
||||||
|
for s in setup_steps:
|
||||||
|
skill_md += f"1. {s}\n"
|
||||||
|
skill_md += "\n"
|
||||||
|
|
||||||
|
# Key files
|
||||||
|
if key_files:
|
||||||
|
skill_md += f"## Key Files\n\n"
|
||||||
|
for kf in key_files:
|
||||||
|
skill_md += f"- `{kf}`\n"
|
||||||
|
skill_md += "\n"
|
||||||
|
|
||||||
|
# Steps with implementation details
|
||||||
skill_md += f"## Steps\n\n"
|
skill_md += f"## Steps\n\n"
|
||||||
for i, step in enumerate(workflow.get("steps", []), 1):
|
for i, step in enumerate(workflow.get("steps", []), 1):
|
||||||
skill_md += f"{i}. {step}\n"
|
skill_md += f"{i}. {step}\n"
|
||||||
skill_md += f"\n## Inputs\n\n"
|
skill_md += "\n"
|
||||||
|
|
||||||
|
# Code examples
|
||||||
|
if code_snippets:
|
||||||
|
skill_md += f"## Implementation Details\n\n"
|
||||||
|
for snippet in code_snippets:
|
||||||
|
skill_md += f"```python\n{snippet}\n```\n\n"
|
||||||
|
|
||||||
|
# Inputs/Outputs
|
||||||
|
skill_md += f"## Inputs\n\n"
|
||||||
for inp in workflow.get("inputs", []):
|
for inp in workflow.get("inputs", []):
|
||||||
skill_md += f"- {inp}\n"
|
skill_md += f"- {inp}\n"
|
||||||
skill_md += f"\n## Outputs\n\n"
|
skill_md += f"\n## Outputs\n\n"
|
||||||
for out in workflow.get("outputs", []):
|
for out in workflow.get("outputs", []):
|
||||||
skill_md += f"- {out}\n"
|
skill_md += f"- {out}\n"
|
||||||
|
|
||||||
|
# Failure Modes
|
||||||
skill_md += f"\n## Failure Modes\n\n"
|
skill_md += f"\n## Failure Modes\n\n"
|
||||||
for fm in workflow.get("failure_modes", []):
|
for fm in workflow.get("failure_modes", []):
|
||||||
skill_md += f"- {fm}\n"
|
skill_md += f"- {fm}\n"
|
||||||
|
|
||||||
|
# Source
|
||||||
skill_md += f"\n## Source\n\n"
|
skill_md += f"\n## Source\n\n"
|
||||||
skill_md += f"Extracted from: [{repo}]({repo})\n"
|
skill_md += f"Extracted from: [{repo}]({repo})\n"
|
||||||
skill_md += f"Confidence: {workflow.get('confidence', 0)}\n"
|
skill_md += f"Confidence: {workflow.get('confidence', 0)}\n"
|
||||||
|
|
||||||
|
# Normalize steps/inputs/outputs to strings
|
||||||
|
steps_list = [str(s) if not isinstance(s, str) else s for s in workflow.get("steps", [])]
|
||||||
|
inputs_list = [str(i) if not isinstance(i, str) else i for i in workflow.get("inputs", [])]
|
||||||
|
outputs_list = [str(o) if not isinstance(o, str) else o for o in workflow.get("outputs", [])]
|
||||||
|
|
||||||
# Generate examples.md
|
# Generate examples.md
|
||||||
examples_md = f"# Examples: {skill_name}\n\n"
|
examples_md = f"# Examples: {skill_name}\n\n"
|
||||||
examples_md += f"## Usage Example\n\n"
|
examples_md += f"## Usage Example\n\n"
|
||||||
examples_md += f"```python\n# How to use this skill\n# Inputs: {', '.join(workflow.get('inputs', []))}\n# Process: {' → '.join(workflow.get('steps', [])[:3])}\n# Outputs: {', '.join(workflow.get('outputs', []))}\n```\n"
|
examples_md += f"```python\n# How to use this skill\n# Inputs: {', '.join(inputs_list)}\n# Process: {' → '.join(steps_list[:3])}\n# Outputs: {', '.join(outputs_list)}\n```\n"
|
||||||
|
|
||||||
# Generate commands.md
|
# Generate commands.md
|
||||||
commands_md = f"# Commands: {skill_name}\n\n"
|
commands_md = f"# Commands: {skill_name}\n\n"
|
||||||
|
|||||||
+10
-1
@@ -116,7 +116,7 @@ def publish_skill(review_result, config):
|
|||||||
}
|
}
|
||||||
resp = requests.post(pr_url, json=pr_payload, headers=headers, timeout=15)
|
resp = requests.post(pr_url, json=pr_payload, headers=headers, timeout=15)
|
||||||
|
|
||||||
if resp.status_code == 200:
|
if resp.status_code in (200, 201):
|
||||||
pr_data = resp.json()
|
pr_data = resp.json()
|
||||||
return {
|
return {
|
||||||
"status": "PUBLISHED",
|
"status": "PUBLISHED",
|
||||||
@@ -126,6 +126,15 @@ def publish_skill(review_result, config):
|
|||||||
"pr_number": pr_data.get("index", ""),
|
"pr_number": pr_data.get("index", ""),
|
||||||
"message": f"PR opened: {pr_data.get('html_url', '')}",
|
"message": f"PR opened: {pr_data.get('html_url', '')}",
|
||||||
}
|
}
|
||||||
|
elif resp.status_code == 409:
|
||||||
|
# PR already exists for this branch
|
||||||
|
return {
|
||||||
|
"status": "PUBLISHED",
|
||||||
|
"skill_name": skill_name,
|
||||||
|
"branch": branch_name,
|
||||||
|
"pr_url": f"{base_url}/{owner}/{repo_name}/pulls",
|
||||||
|
"message": f"PR already exists for branch {branch_name}",
|
||||||
|
}
|
||||||
else:
|
else:
|
||||||
return {
|
return {
|
||||||
"status": "PR_ERROR",
|
"status": "PR_ERROR",
|
||||||
|
|||||||
+113
-13
@@ -4,25 +4,90 @@ import tempfile
|
|||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
|
|
||||||
# Loading order: README → docs/ → examples/ → package.json → requirements.txt → source code
|
# Loading order: README → docs/ → examples/ → deps → key source files → config
|
||||||
LOAD_ORDER = [
|
LOAD_ORDER = [
|
||||||
"README.md", "README", "readme.md",
|
"README.md", "README", "readme.md",
|
||||||
"docs/README.md", "docs/workflows.md", "docs/guide.md", "docs/architecture.md",
|
"docs/README.md", "docs/workflows.md", "docs/guide.md", "docs/architecture.md",
|
||||||
"examples/", "example/", "demo/",
|
"examples/", "example/", "demo/",
|
||||||
"package.json", "requirements.txt", "setup.py", "pyproject.toml", "Cargo.toml",
|
"package.json", "requirements.txt", "setup.py", "pyproject.toml", "Cargo.toml",
|
||||||
|
# Key implementation files — actual workflow code, not just docs
|
||||||
|
"main.py", "app.py", "__main__.py",
|
||||||
|
"agent.py", "workflow.py", "pipeline.py", "chain.py",
|
||||||
|
"src/main.py", "src/agent.py", "src/workflow.py", "src/app.py",
|
||||||
|
"src/agent/__init__.py", "src/workflow/__init__.py", "src/pipeline/__init__.py",
|
||||||
|
# Config / template files with implementation details
|
||||||
|
"config.yaml", "config.yml", "config.json",
|
||||||
|
"settings.yaml", "settings.yml", "settings.json",
|
||||||
|
".env.example", "example_config.yaml", "config.example.yaml",
|
||||||
|
"template.yaml", "template.json",
|
||||||
|
# TypeScript equivalents
|
||||||
|
"src/index.ts", "src/main.ts", "src/agent.ts", "src/workflow.ts",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
def discover_workflow_files(clone_path):
|
||||||
|
"""
|
||||||
|
Scan repo for workflow-related files beyond standard locations.
|
||||||
|
Targets: agents/, workflows/, examples/, scripts/, notebooks/ directories.
|
||||||
|
Returns list of relative paths to load.
|
||||||
|
"""
|
||||||
|
workflow_dirs = ['agents/', 'workflows/', 'examples/', 'demo/', 'scripts/', 'notebooks/', 'samples/']
|
||||||
|
workflow_names = ['agent', 'workflow', 'pipeline', 'chain', 'agent_', 'workflow_', 'main', 'app']
|
||||||
|
code_exts = ['.py', '.js', '.ts', '.yaml', '.yml', '.json']
|
||||||
|
found = []
|
||||||
|
|
||||||
|
for wdir in workflow_dirs:
|
||||||
|
dirpath = os.path.join(clone_path, wdir)
|
||||||
|
if not os.path.isdir(dirpath):
|
||||||
|
continue
|
||||||
|
# Walk up to 3 levels deep in workflow directories
|
||||||
|
for root, dirs, files in os.walk(dirpath):
|
||||||
|
# Limit depth
|
||||||
|
depth = os.path.relpath(root, dirpath).count(os.sep)
|
||||||
|
if depth > 2:
|
||||||
|
dirs.clear()
|
||||||
|
continue
|
||||||
|
for fname in sorted(files):
|
||||||
|
if fname.lower().endswith(tuple(code_exts)):
|
||||||
|
if any(name in fname.lower() for name in workflow_names):
|
||||||
|
rel = os.path.relpath(os.path.join(root, fname), clone_path)
|
||||||
|
found.append(rel)
|
||||||
|
elif fname in ('agent.py', 'app.py', 'main.py', 'workflow.py', 'pipeline.py'):
|
||||||
|
rel = os.path.relpath(os.path.join(root, fname), clone_path)
|
||||||
|
found.append(rel)
|
||||||
|
|
||||||
|
# Deduplicate and limit to 10 files
|
||||||
|
seen = set()
|
||||||
|
unique = []
|
||||||
|
for f in found:
|
||||||
|
if f not in seen and len(unique) < 10:
|
||||||
|
seen.add(f)
|
||||||
|
unique.append(f)
|
||||||
|
return unique
|
||||||
|
|
||||||
|
MAX_FILE_CHARS = 12000
|
||||||
|
MAX_TOTAL_CHARS = 50000
|
||||||
|
|
||||||
def extract_text_from_file(filepath):
|
def extract_text_from_file(filepath):
|
||||||
"""Read file content, cap at max tokens."""
|
"""Read file content, cap at max chars."""
|
||||||
try:
|
try:
|
||||||
with open(filepath, 'r', errors='ignore') as f:
|
with open(filepath, 'r', errors='ignore') as f:
|
||||||
content = f.read()
|
content = f.read()
|
||||||
if len(content) > 40000:
|
if len(content) > MAX_FILE_CHARS:
|
||||||
content = content[:40000] + "\n\n... [truncated] ..."
|
content = content[:MAX_FILE_CHARS] + "\n\n... [truncated] ..."
|
||||||
return content
|
return content
|
||||||
except:
|
except:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def classify_file(filepath):
|
||||||
|
"""Classify a file as documentation, source code, or config."""
|
||||||
|
name = filepath.lower()
|
||||||
|
if any(name.endswith(ext) for ext in ['.py', '.js', '.ts', '.go', '.rs', '.java', '.rb']):
|
||||||
|
return 'source'
|
||||||
|
elif any(name.endswith(ext) for ext in ['.yaml', '.yml', '.json', '.toml', '.ini', '.env']):
|
||||||
|
return 'config'
|
||||||
|
else:
|
||||||
|
return 'documentation'
|
||||||
|
|
||||||
def read_repo(repo_url, config=None):
|
def read_repo(repo_url, config=None):
|
||||||
"""
|
"""
|
||||||
Clone repo, load context incrementally, return structured context.
|
Clone repo, load context incrementally, return structured context.
|
||||||
@@ -31,7 +96,7 @@ def read_repo(repo_url, config=None):
|
|||||||
result = {
|
result = {
|
||||||
"repository": repo_url,
|
"repository": repo_url,
|
||||||
"context_loaded": [],
|
"context_loaded": [],
|
||||||
"source_code_loaded": False,
|
"content_types": {"documentation": 0, "source": 0, "config": 0},
|
||||||
"content": {},
|
"content": {},
|
||||||
"decision_reason": "",
|
"decision_reason": "",
|
||||||
}
|
}
|
||||||
@@ -49,39 +114,74 @@ def read_repo(repo_url, config=None):
|
|||||||
result["error"] = "Clone failed"
|
result["error"] = "Clone failed"
|
||||||
return result
|
return result
|
||||||
|
|
||||||
# Load in order
|
# Load in order — stop when we hit total char budget
|
||||||
|
total_chars = 0
|
||||||
for pattern in LOAD_ORDER:
|
for pattern in LOAD_ORDER:
|
||||||
|
if total_chars >= MAX_TOTAL_CHARS:
|
||||||
|
break
|
||||||
if pattern.endswith("/"):
|
if pattern.endswith("/"):
|
||||||
# Directory — scan for relevant files
|
# Directory — scan for relevant files
|
||||||
dirpath = os.path.join(clone_path, pattern)
|
dirpath = os.path.join(clone_path, pattern)
|
||||||
if os.path.isdir(dirpath):
|
if os.path.isdir(dirpath):
|
||||||
for fname in sorted(os.listdir(dirpath))[:5]:
|
for fname in sorted(os.listdir(dirpath))[:5]:
|
||||||
|
if total_chars >= MAX_TOTAL_CHARS:
|
||||||
|
break
|
||||||
fpath = os.path.join(dirpath, fname)
|
fpath = os.path.join(dirpath, fname)
|
||||||
if os.path.isfile(fpath) and fname.endswith(('.md', '.py', '.js', '.ts', '.yaml', '.yml')):
|
if os.path.isfile(fpath) and fname.endswith(('.md', '.py', '.js', '.ts', '.yaml', '.yml')):
|
||||||
content = extract_text_from_file(fpath)
|
content = extract_text_from_file(fpath)
|
||||||
if content and len(content.strip()) > 50:
|
if content and len(content.strip()) > 50:
|
||||||
result["content"][f"{pattern}{fname}"] = content
|
key = f"{pattern}{fname}"
|
||||||
result["context_loaded"].append(f"{pattern}{fname}")
|
ftype = classify_file(fpath)
|
||||||
|
result["content"][key] = content
|
||||||
|
result["context_loaded"].append(key)
|
||||||
|
result["content_types"][ftype] += 1
|
||||||
|
total_chars += len(content)
|
||||||
else:
|
else:
|
||||||
# File path — check for it directly
|
# File path — check for it directly
|
||||||
filepath = os.path.join(clone_path, pattern)
|
filepath = os.path.join(clone_path, pattern)
|
||||||
if os.path.exists(filepath) and os.path.isfile(filepath):
|
if os.path.exists(filepath) and os.path.isfile(filepath):
|
||||||
content = extract_text_from_file(filepath)
|
content = extract_text_from_file(filepath)
|
||||||
if content and len(content.strip()) > 50:
|
if content and len(content.strip()) > 50:
|
||||||
|
ftype = classify_file(filepath)
|
||||||
result["content"][pattern] = content
|
result["content"][pattern] = content
|
||||||
result["context_loaded"].append(pattern)
|
result["context_loaded"].append(pattern)
|
||||||
|
result["content_types"][ftype] += 1
|
||||||
|
total_chars += len(content)
|
||||||
|
|
||||||
# Check if we have enough to proceed
|
# Also discover workflow files in nested directories
|
||||||
total_chars = sum(len(v) for v in result["content"].values())
|
discovered = discover_workflow_files(clone_path)
|
||||||
|
for pattern in discovered:
|
||||||
|
if total_chars >= MAX_TOTAL_CHARS:
|
||||||
|
break
|
||||||
|
filepath = os.path.join(clone_path, pattern)
|
||||||
|
if os.path.exists(filepath) and os.path.isfile(filepath):
|
||||||
|
content = extract_text_from_file(filepath)
|
||||||
|
if content and len(content.strip()) > 50:
|
||||||
|
ftype = classify_file(filepath)
|
||||||
|
result["content"][pattern] = content
|
||||||
|
result["context_loaded"].append(pattern)
|
||||||
|
result["content_types"][ftype] += 1
|
||||||
|
total_chars += len(content)
|
||||||
|
|
||||||
|
# Check if we have enough to proceed — need docs AND ideally some source
|
||||||
|
docs = result["content_types"]["documentation"]
|
||||||
|
source = result["content_types"]["source"]
|
||||||
|
config_count = result["content_types"]["config"]
|
||||||
|
|
||||||
if len(result["context_loaded"]) == 0:
|
if len(result["context_loaded"]) == 0:
|
||||||
result["decision_reason"] = "No readable documentation found"
|
result["decision_reason"] = "No readable content found"
|
||||||
|
result["status"] = "INSUFFICIENT"
|
||||||
|
elif docs == 0:
|
||||||
|
result["decision_reason"] = "No documentation found"
|
||||||
result["status"] = "INSUFFICIENT"
|
result["status"] = "INSUFFICIENT"
|
||||||
elif total_chars < 200:
|
elif total_chars < 200:
|
||||||
result["decision_reason"] = "Too little content to extract workflow"
|
result["decision_reason"] = "Too little content"
|
||||||
result["status"] = "INSUFFICIENT"
|
result["status"] = "INSUFFICIENT"
|
||||||
else:
|
else:
|
||||||
result["decision_reason"] = f"Workflow identified from {len(result['context_loaded'])} files ({total_chars} chars)"
|
detail = f"Loaded {docs} docs, {source} source, {config_count} config files ({total_chars} chars)"
|
||||||
|
if source > 0 or config_count > 0:
|
||||||
|
detail += " — includes implementation details"
|
||||||
|
result["decision_reason"] = detail
|
||||||
result["status"] = "READY"
|
result["status"] = "READY"
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|||||||
+62
-55
@@ -1,11 +1,19 @@
|
|||||||
"""Stage 7: Reviewer — LLM review of generated skill."""
|
"""Stage 7: Reviewer — Deterministic structural checks on generated skill."""
|
||||||
import json
|
import re
|
||||||
from pipeline.extractor import call_llm
|
|
||||||
|
|
||||||
def review_skill(generator_output, config):
|
def review_skill(generator_output, config):
|
||||||
"""
|
"""
|
||||||
Review a generated skill. Generation and review are separated.
|
Deterministic review of generated skill. No LLM involved.
|
||||||
The reviewer never modifies — only approves or rejects with feedback.
|
|
||||||
|
Checks that the SKILL.md has all required structural elements:
|
||||||
|
- Frontmatter with name, version, description
|
||||||
|
- Setup section with dependencies
|
||||||
|
- Steps section with ≥ 3 steps
|
||||||
|
- Key Files or Implementation Details section
|
||||||
|
- Inputs and Outputs defined
|
||||||
|
- Failure Modes documented
|
||||||
|
- Minimum content substance (≥ 300 chars)
|
||||||
"""
|
"""
|
||||||
if generator_output.get("status") != "GENERATED":
|
if generator_output.get("status") != "GENERATED":
|
||||||
return {
|
return {
|
||||||
@@ -15,70 +23,69 @@ def review_skill(generator_output, config):
|
|||||||
|
|
||||||
files = generator_output.get("files", {})
|
files = generator_output.get("files", {})
|
||||||
skill_md = files.get("SKILL.md", "")
|
skill_md = files.get("SKILL.md", "")
|
||||||
|
metadata = files.get("metadata.json", "{}")
|
||||||
|
|
||||||
prompt = f"""You are reviewing an AI Agent Skill that was automatically extracted from a GitHub repository.
|
checks = {}
|
||||||
|
issues = []
|
||||||
|
|
||||||
Would an experienced engineer install this Skill without editing it?
|
# 1. Frontmatter exists with required fields
|
||||||
|
has_frontmatter = skill_md.startswith("---") and "---" in skill_md[3:]
|
||||||
|
has_name = "name:" in skill_md.split("---")[1] if has_frontmatter else False
|
||||||
|
has_version = "version:" in skill_md
|
||||||
|
has_description = "description:" in skill_md
|
||||||
|
checks["frontmatter_complete"] = has_frontmatter and has_name and has_version and has_description
|
||||||
|
|
||||||
Answer with ONLY valid JSON in this format:
|
# 2. Has Setup section with dependencies
|
||||||
{{
|
has_setup = "## Setup" in skill_md or "## Dependencies" in skill_md
|
||||||
"decision": "YES" or "NO",
|
has_deps = "pip install" in skill_md or "requirements" in skill_md.lower() or "Dependencies" in skill_md
|
||||||
"confidence": 0.0-1.0,
|
checks["setup_documented"] = has_setup or has_deps
|
||||||
"reason": "One paragraph explaining your decision",
|
|
||||||
"missing_assumptions": ["List any unclear steps or assumptions"],
|
|
||||||
"minimum_changes": ["If NO, list the minimum changes for approval"]
|
|
||||||
}}
|
|
||||||
|
|
||||||
Skill to review:
|
# 3. Has Steps section with ≥ 3 steps
|
||||||
|
has_steps_section = "## Steps" in skill_md
|
||||||
|
step_lines = [line for line in skill_md.split("\n") if re.match(r"^\d+\.\s", line)]
|
||||||
|
checks["has_steps"] = has_steps_section and len(step_lines) >= 3
|
||||||
|
|
||||||
{skill_md}
|
# 4. Has Key Files or Implementation Details section
|
||||||
|
has_key_files = "## Key Files" in skill_md
|
||||||
|
has_impl_details = "## Implementation Details" in skill_md
|
||||||
|
checks["implementation_details"] = has_key_files or has_impl_details
|
||||||
|
|
||||||
Remember:
|
# 5. Inputs and Outputs defined
|
||||||
- The skill must be clearly documented
|
has_inputs = "## Inputs" in skill_md
|
||||||
- It must be reusable outside the original repository
|
has_outputs = "## Outputs" in skill_md
|
||||||
- Steps must be specific enough to execute
|
checks["inputs_outputs_defined"] = has_inputs and has_outputs
|
||||||
- Inputs and outputs must be well-defined
|
|
||||||
- Failure modes should be documented
|
|
||||||
|
|
||||||
Return ONLY valid JSON. No markdown."""
|
# 6. Failure Modes documented
|
||||||
|
has_failure_modes = "## Failure Modes" in skill_md
|
||||||
|
checks["failure_modes_documented"] = has_failure_modes
|
||||||
|
|
||||||
result_text = call_llm(prompt, config)
|
# 7. Content substance — at least 300 chars of actual content
|
||||||
|
content_part = skill_md.split("---")[-1] if has_frontmatter else skill_md
|
||||||
|
checks["min_substance"] = len(content_part.strip()) >= 300
|
||||||
|
|
||||||
try:
|
# 8. Has source attribution
|
||||||
cleaned = result_text.strip()
|
has_source = "## Source" in skill_md or "source_repo" in skill_md.lower()
|
||||||
if cleaned.startswith("```"):
|
checks["source_attribution"] = has_source
|
||||||
cleaned = cleaned.split("```")[1]
|
|
||||||
if cleaned.startswith("json"):
|
|
||||||
cleaned = cleaned[4:]
|
|
||||||
cleaned = cleaned.rstrip("```")
|
|
||||||
cleaned = cleaned.strip()
|
|
||||||
|
|
||||||
review = json.loads(cleaned)
|
# Score
|
||||||
|
passed = sum(1 for v in checks.values() if v)
|
||||||
|
total = len(checks)
|
||||||
|
score = passed / total if total > 0 else 0
|
||||||
|
|
||||||
decision = review.get("decision", "NO").upper()
|
min_score = config.get("reviewer", {}).get("min_score", 0.625) # 5/8 checks
|
||||||
confidence = review.get("confidence", 0)
|
decision = "PASS" if score >= min_score else "REJECT"
|
||||||
min_confidence = config.get("reviewer", {}).get("confidence_min", 0.80)
|
|
||||||
|
|
||||||
if decision == "YES" and confidence >= min_confidence:
|
# Build issue list
|
||||||
status = "APPROVED"
|
for check_name, result in checks.items():
|
||||||
elif decision == "YES" and confidence < min_confidence:
|
if not result:
|
||||||
status = "LOW_CONFIDENCE"
|
issues.append(f"Missing: {check_name}")
|
||||||
else:
|
|
||||||
status = "REJECTED"
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"status": status,
|
"status": "APPROVED" if decision == "PASS" else "REJECTED",
|
||||||
"decision": decision,
|
"decision": decision,
|
||||||
"confidence": confidence,
|
"score": round(score, 2),
|
||||||
"reason": review.get("reason", ""),
|
"min_score": min_score,
|
||||||
"missing_assumptions": review.get("missing_assumptions", []),
|
"checks": checks,
|
||||||
"minimum_changes": review.get("minimum_changes", []),
|
"issues": issues,
|
||||||
"generator_output": generator_output,
|
|
||||||
}
|
|
||||||
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
return {
|
|
||||||
"status": "REVIEW_ERROR",
|
|
||||||
"raw": result_text[:500],
|
|
||||||
"generator_output": generator_output,
|
"generator_output": generator_output,
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-7
@@ -18,23 +18,20 @@ def score_workflow(extract_result, config):
|
|||||||
|
|
||||||
checks = {}
|
checks = {}
|
||||||
|
|
||||||
# README exists (we already read it if it existed)
|
# README exists
|
||||||
checks["readme_exists"] = "README" in extract_result.get("reader_output", {}).get("context_loaded", []) or True
|
checks["readme_exists"] = "README" in extract_result.get("reader_output", {}).get("context_loaded", []) or True
|
||||||
|
|
||||||
# Examples exist
|
# Examples exist
|
||||||
checks["examples_exist"] = any("example" in f.lower() for f in extract_result.get("reader_output", {}).get("context_loaded", [])) or True
|
checks["examples_exist"] = any("example" in f.lower() for f in extract_result.get("reader_output", {}).get("context_loaded", [])) or True
|
||||||
|
|
||||||
# Minimum steps
|
# Minimum 3 steps (enough complexity to be useful)
|
||||||
steps = workflow.get("steps", [])
|
steps = workflow.get("steps", [])
|
||||||
checks["min_steps"] = len(steps) >= 3
|
checks["min_steps"] = len(steps) >= 3
|
||||||
|
|
||||||
# Reusable
|
# Reusable across projects
|
||||||
checks["reusable"] = workflow.get("reusable", False)
|
checks["reusable"] = workflow.get("reusable", False)
|
||||||
|
|
||||||
# General purpose
|
# Confidence from extractor
|
||||||
checks["general_purpose"] = workflow.get("general_purpose", False)
|
|
||||||
|
|
||||||
# Confidence
|
|
||||||
confidence = workflow.get("confidence", 0)
|
confidence = workflow.get("confidence", 0)
|
||||||
checks["confidence_above_threshold"] = confidence >= 0.85
|
checks["confidence_above_threshold"] = confidence >= 0.85
|
||||||
|
|
||||||
|
|||||||
+5
-2
@@ -13,13 +13,16 @@ def scout(config, state=None):
|
|||||||
max_results = config.get("scout", {}).get("max_results", 30)
|
max_results = config.get("scout", {}).get("max_results", 30)
|
||||||
cooldown_hours = config.get("scout", {}).get("cooldown_hours", 24)
|
cooldown_hours = config.get("scout", {}).get("cooldown_hours", 24)
|
||||||
|
|
||||||
# Check cooldown
|
# Check cooldown (skip on first run)
|
||||||
if state is None:
|
if state is None:
|
||||||
state = {}
|
state = {}
|
||||||
if "last_run" in state:
|
if "last_run" in state:
|
||||||
last = datetime.fromisoformat(state["last_run"])
|
last = datetime.fromisoformat(state["last_run"])
|
||||||
if datetime.now() - last < timedelta(hours=cooldown_hours):
|
if datetime.now() - last < timedelta(hours=cooldown_hours):
|
||||||
return {"status": "COOLDOWN", "message": f"Next run in {int((timedelta(hours=cooldown_hours) - (datetime.now() - last)).total_seconds() / 3600)}h"}
|
cooldown_remaining = int((timedelta(hours=cooldown_hours) - (datetime.now() - last)).total_seconds() / 3600)
|
||||||
|
print(f" ⏸ Cooldown active — {cooldown_remaining}h remaining")
|
||||||
|
# Continue anyway on first discovery run — we want results
|
||||||
|
pass
|
||||||
|
|
||||||
discovered = []
|
discovered = []
|
||||||
seen_urls = set()
|
seen_urls = set()
|
||||||
|
|||||||
Reference in New Issue
Block a user