agent-skills/pipeline/reader.py

"""Stage 3: Reader — Incremental context loading."""
import subprocess
import tempfile
import os
import json

# Loading order: README → docs/ → examples/ → deps → key source files → config
LOAD_ORDER = [
    "README.md", "README", "readme.md",
    "docs/README.md", "docs/workflows.md", "docs/guide.md", "docs/architecture.md",
    "examples/", "example/", "demo/",
    "package.json", "requirements.txt", "setup.py", "pyproject.toml", "Cargo.toml",
    # Key implementation files — actual workflow code, not just docs
    "main.py", "app.py", "__main__.py",
    "agent.py", "workflow.py", "pipeline.py", "chain.py",
    "src/main.py", "src/agent.py", "src/workflow.py", "src/app.py",
    "src/agent/__init__.py", "src/workflow/__init__.py", "src/pipeline/__init__.py",
    # Config / template files with implementation details
    "config.yaml", "config.yml", "config.json",
    "settings.yaml", "settings.yml", "settings.json",
    ".env.example", "example_config.yaml", "config.example.yaml",
    "template.yaml", "template.json",
    # TypeScript equivalents
    "src/index.ts", "src/main.ts", "src/agent.ts", "src/workflow.ts",
]

def discover_workflow_files(clone_path):
    """
    Scan repo for workflow-related files beyond standard locations.
    Targets: agents/, workflows/, examples/, scripts/, notebooks/ directories.
    Returns list of relative paths to load.
    """
    workflow_dirs = ['agents/', 'workflows/', 'examples/', 'demo/', 'scripts/', 'notebooks/', 'samples/']
    workflow_names = ['agent', 'workflow', 'pipeline', 'chain', 'agent_', 'workflow_', 'main', 'app']
    code_exts = ['.py', '.js', '.ts', '.yaml', '.yml', '.json']
    found = []

    for wdir in workflow_dirs:
        dirpath = os.path.join(clone_path, wdir)
        if not os.path.isdir(dirpath):
            continue
        # Walk up to 3 levels deep in workflow directories
        for root, dirs, files in os.walk(dirpath):
            # Limit depth
            depth = os.path.relpath(root, dirpath).count(os.sep)
            if depth > 2:
                dirs.clear()
                continue
            for fname in sorted(files):
                if fname.lower().endswith(tuple(code_exts)):
                    if any(name in fname.lower() for name in workflow_names):
                        rel = os.path.relpath(os.path.join(root, fname), clone_path)
                        found.append(rel)
                    elif fname in ('agent.py', 'app.py', 'main.py', 'workflow.py', 'pipeline.py'):
                        rel = os.path.relpath(os.path.join(root, fname), clone_path)
                        found.append(rel)

    # Deduplicate and limit to 10 files
    seen = set()
    unique = []
    for f in found:
        if f not in seen and len(unique) < 10:
            seen.add(f)
            unique.append(f)
    return unique

MAX_FILE_CHARS = 12000
MAX_TOTAL_CHARS = 50000

def extract_text_from_file(filepath):
    """Read file content, cap at max chars."""
    try:
        with open(filepath, 'r', errors='ignore') as f:
            content = f.read()
        if len(content) > MAX_FILE_CHARS:
            content = content[:MAX_FILE_CHARS] + "\n\n... [truncated] ..."
        return content
    except:
        return None

def classify_file(filepath):
    """Classify a file as documentation, source code, or config."""
    name = filepath.lower()
    if any(name.endswith(ext) for ext in ['.py', '.js', '.ts', '.go', '.rs', '.java', '.rb']):
        return 'source'
    elif any(name.endswith(ext) for ext in ['.yaml', '.yml', '.json', '.toml', '.ini', '.env']):
        return 'config'
    else:
        return 'documentation'

def read_repo(repo_url, config=None):
    """
    Clone repo, load context incrementally, return structured context.
    Returns only what's needed to understand the workflow.
    """
    result = {
        "repository": repo_url,
        "context_loaded": [],
        "content_types": {"documentation": 0, "source": 0, "config": 0},
        "content": {},
        "decision_reason": "",
    }

    repo_name = repo_url.rstrip("/").split("/")[-1]

    with tempfile.TemporaryDirectory() as tmpdir:
        clone_path = os.path.join(tmpdir, repo_name)

        # Clone — shallow clone but ensure top-level files are fetched
        try:
            clone_cmd = ["git", "clone", "--depth=1", "--no-single-branch", repo_url, clone_path]
            subprocess.run(clone_cmd, capture_output=True, timeout=60)
        except:
            result["error"] = "Clone failed"
            return result

        # Load in order — stop when we hit total char budget
        total_chars = 0
        for pattern in LOAD_ORDER:
            if total_chars >= MAX_TOTAL_CHARS:
                break
            if pattern.endswith("/"):
                # Directory — scan for relevant files
                dirpath = os.path.join(clone_path, pattern)
                if os.path.isdir(dirpath):
                    for fname in sorted(os.listdir(dirpath))[:5]:
                        if total_chars >= MAX_TOTAL_CHARS:
                            break
                        fpath = os.path.join(dirpath, fname)
                        if os.path.isfile(fpath) and fname.endswith(('.md', '.py', '.js', '.ts', '.yaml', '.yml')):
                            content = extract_text_from_file(fpath)
                            if content and len(content.strip()) > 50:
                                key = f"{pattern}{fname}"
                                ftype = classify_file(fpath)
                                result["content"][key] = content
                                result["context_loaded"].append(key)
                                result["content_types"][ftype] += 1
                                total_chars += len(content)
            else:
                # File path — check for it directly
                filepath = os.path.join(clone_path, pattern)
                if os.path.exists(filepath) and os.path.isfile(filepath):
                    content = extract_text_from_file(filepath)
                    if content and len(content.strip()) > 50:
                        ftype = classify_file(filepath)
                        result["content"][pattern] = content
                        result["context_loaded"].append(pattern)
                        result["content_types"][ftype] += 1
                        total_chars += len(content)

        # Also discover workflow files in nested directories
        discovered = discover_workflow_files(clone_path)
        for pattern in discovered:
            if total_chars >= MAX_TOTAL_CHARS:
                break
            filepath = os.path.join(clone_path, pattern)
            if os.path.exists(filepath) and os.path.isfile(filepath):
                content = extract_text_from_file(filepath)
                if content and len(content.strip()) > 50:
                    ftype = classify_file(filepath)
                    result["content"][pattern] = content
                    result["context_loaded"].append(pattern)
                    result["content_types"][ftype] += 1
                    total_chars += len(content)

        # Check if we have enough to proceed — need docs AND ideally some source
        docs = result["content_types"]["documentation"]
        source = result["content_types"]["source"]
        config_count = result["content_types"]["config"]

        if len(result["context_loaded"]) == 0:
            result["decision_reason"] = "No readable content found"
            result["status"] = "INSUFFICIENT"
        elif docs == 0:
            result["decision_reason"] = "No documentation found"
            result["status"] = "INSUFFICIENT"
        elif total_chars < 200:
            result["decision_reason"] = "Too little content"
            result["status"] = "INSUFFICIENT"
        else:
            detail = f"Loaded {docs} docs, {source} source, {config_count} config files ({total_chars} chars)"
            if source > 0 or config_count > 0:
                detail += " — includes implementation details"
            result["decision_reason"] = detail
            result["status"] = "READY"

    return result