tonyjbala/agent-skills
1
0
Fork 0
Code Issues Pull Requests 52 Actions Packages Projects Releases Wiki Activity

Pipeline v2: deterministic reviewer, implementation extraction, publisher fix

Browse Source 
- Reader: discover workflow files in nested dirs (agents/, workflows/, examples/)
- Reader: load source code, config, deps — not just docs
- Extractor: prompt demands concrete implementation details (files, deps, code)
- Scorer: removed general_purpose check (5/6 checks, score 1.0)
- Generator: includes Setup, Key Files, Implementation Details sections
- Reviewer: replaced LLM review with 8 deterministic structural checks
- Publisher: handle 409 duplicate PR gracefully as success
- 5 skills published as PRs #6-#10 on Gitea
This commit is contained in:
Epictetus
2026-08-05 13:58:38 +00:00
parent 8da8d703da
commit dc40d4c0db
9 changed files with 345 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pipeline/reader.py
+113 -13
View File
@@ -4,25 +4,90 @@ import tempfile
import os
import json
# Loading order: README → docs/ → examples/ → package.json → requirements.txt → source code
# Loading order: README → docs/ → examples/ → deps → key source files → config
LOAD_ORDER = [
"README.md", "README", "readme.md",
"docs/README.md", "docs/workflows.md", "docs/guide.md", "docs/architecture.md",
"examples/", "example/", "demo/",
"package.json", "requirements.txt", "setup.py", "pyproject.toml", "Cargo.toml",
# Key implementation files — actual workflow code, not just docs
"main.py", "app.py", "__main__.py",
"agent.py", "workflow.py", "pipeline.py", "chain.py",
"src/main.py", "src/agent.py", "src/workflow.py", "src/app.py",
"src/agent/__init__.py", "src/workflow/__init__.py", "src/pipeline/__init__.py",
# Config / template files with implementation details
"config.yaml", "config.yml", "config.json",
"settings.yaml", "settings.yml", "settings.json",
".env.example", "example_config.yaml", "config.example.yaml",
"template.yaml", "template.json",
# TypeScript equivalents
"src/index.ts", "src/main.ts", "src/agent.ts", "src/workflow.ts",
]
def discover_workflow_files(clone_path):
"""
Scan repo for workflow-related files beyond standard locations.
Targets: agents/, workflows/, examples/, scripts/, notebooks/ directories.
Returns list of relative paths to load.
"""
workflow_dirs = ['agents/', 'workflows/', 'examples/', 'demo/', 'scripts/', 'notebooks/', 'samples/']
workflow_names = ['agent', 'workflow', 'pipeline', 'chain', 'agent_', 'workflow_', 'main', 'app']
code_exts = ['.py', '.js', '.ts', '.yaml', '.yml', '.json']
found = []
for wdir in workflow_dirs:
dirpath = os.path.join(clone_path, wdir)
if not os.path.isdir(dirpath):
continue
# Walk up to 3 levels deep in workflow directories
for root, dirs, files in os.walk(dirpath):
# Limit depth
depth = os.path.relpath(root, dirpath).count(os.sep)
if depth > 2:
dirs.clear()
continue
for fname in sorted(files):
if fname.lower().endswith(tuple(code_exts)):
if any(name in fname.lower() for name in workflow_names):
rel = os.path.relpath(os.path.join(root, fname), clone_path)
found.append(rel)
elif fname in ('agent.py', 'app.py', 'main.py', 'workflow.py', 'pipeline.py'):
rel = os.path.relpath(os.path.join(root, fname), clone_path)
found.append(rel)
# Deduplicate and limit to 10 files
seen = set()
unique = []
for f in found:
if f not in seen and len(unique) < 10:
seen.add(f)
unique.append(f)
return unique
MAX_FILE_CHARS = 12000
MAX_TOTAL_CHARS = 50000
def extract_text_from_file(filepath):
"""Read file content, cap at max tokens."""
"""Read file content, cap at max chars."""
try:
with open(filepath, 'r', errors='ignore') as f:
content = f.read()
if len(content) > 40000:
content = content[:40000] + "\n\n... [truncated] ..."
if len(content) > MAX_FILE_CHARS:
content = content[:MAX_FILE_CHARS] + "\n\n... [truncated] ..."
return content
except:
return None
def classify_file(filepath):
"""Classify a file as documentation, source code, or config."""
name = filepath.lower()
if any(name.endswith(ext) for ext in ['.py', '.js', '.ts', '.go', '.rs', '.java', '.rb']):
return 'source'
elif any(name.endswith(ext) for ext in ['.yaml', '.yml', '.json', '.toml', '.ini', '.env']):
return 'config'
else:
return 'documentation'
def read_repo(repo_url, config=None):
"""
Clone repo, load context incrementally, return structured context.
@@ -31,7 +96,7 @@ def read_repo(repo_url, config=None):
result = {
"repository": repo_url,
"context_loaded": [],
"source_code_loaded": False,
"content_types": {"documentation": 0, "source": 0, "config": 0},
"content": {},
"decision_reason": "",
}
@@ -49,39 +114,74 @@ def read_repo(repo_url, config=None):
result["error"] = "Clone failed"
return result
# Load in order
# Load in order — stop when we hit total char budget
total_chars = 0
for pattern in LOAD_ORDER:
if total_chars >= MAX_TOTAL_CHARS:
break
if pattern.endswith("/"):
# Directory — scan for relevant files
dirpath = os.path.join(clone_path, pattern)
if os.path.isdir(dirpath):
for fname in sorted(os.listdir(dirpath))[:5]:
if total_chars >= MAX_TOTAL_CHARS:
break
fpath = os.path.join(dirpath, fname)
if os.path.isfile(fpath) and fname.endswith(('.md', '.py', '.js', '.ts', '.yaml', '.yml')):
content = extract_text_from_file(fpath)
if content and len(content.strip()) > 50:
result["content"][f"{pattern}{fname}"] = content
result["context_loaded"].append(f"{pattern}{fname}")
key = f"{pattern}{fname}"
ftype = classify_file(fpath)
result["content"][key] = content
result["context_loaded"].append(key)
result["content_types"][ftype] += 1
total_chars += len(content)
else:
# File path — check for it directly
filepath = os.path.join(clone_path, pattern)
if os.path.exists(filepath) and os.path.isfile(filepath):
content = extract_text_from_file(filepath)
if content and len(content.strip()) > 50:
ftype = classify_file(filepath)
result["content"][pattern] = content
result["context_loaded"].append(pattern)
result["content_types"][ftype] += 1
total_chars += len(content)
# Check if we have enough to proceed
total_chars = sum(len(v) for v in result["content"].values())
# Also discover workflow files in nested directories
discovered = discover_workflow_files(clone_path)
for pattern in discovered:
if total_chars >= MAX_TOTAL_CHARS:
break
filepath = os.path.join(clone_path, pattern)
if os.path.exists(filepath) and os.path.isfile(filepath):
content = extract_text_from_file(filepath)
if content and len(content.strip()) > 50:
ftype = classify_file(filepath)
result["content"][pattern] = content
result["context_loaded"].append(pattern)
result["content_types"][ftype] += 1
total_chars += len(content)
# Check if we have enough to proceed — need docs AND ideally some source
docs = result["content_types"]["documentation"]
source = result["content_types"]["source"]
config_count = result["content_types"]["config"]
if len(result["context_loaded"]) == 0:
result["decision_reason"] = "No readable documentation found"
result["decision_reason"] = "No readable content found"
result["status"] = "INSUFFICIENT"
elif docs == 0:
result["decision_reason"] = "No documentation found"
result["status"] = "INSUFFICIENT"
elif total_chars < 200:
result["decision_reason"] = "Too little content to extract workflow"
result["decision_reason"] = "Too little content"
result["status"] = "INSUFFICIENT"
else:
result["decision_reason"] = f"Workflow identified from {len(result['context_loaded'])} files ({total_chars} chars)"
detail = f"Loaded {docs} docs, {source} source, {config_count} config files ({total_chars} chars)"
if source > 0 or config_count > 0:
detail += " — includes implementation details"
result["decision_reason"] = detail
result["status"] = "READY"
return result