Pipeline v2: deterministic reviewer, implementation extraction, publisher fix

- Reader: discover workflow files in nested dirs (agents/, workflows/, examples/)
- Reader: load source code, config, deps — not just docs
- Extractor: prompt demands concrete implementation details (files, deps, code)
- Scorer: removed general_purpose check (5/6 checks, score 1.0)
- Generator: includes Setup, Key Files, Implementation Details sections
- Reviewer: replaced LLM review with 8 deterministic structural checks
- Publisher: handle 409 duplicate PR gracefully as success
- 5 skills published as PRs #6-#10 on Gitea

pipeline/extractor.py

@@ -6,37 +6,61 @@ import re
 
 
 def call_llm(prompt, config):
-    """Call the configured LLM for extraction."""
-    llm_config = config.get("llm", {})
+    """Call the configured LLM for extraction/review."""
+    llm_config = config.get("llm_pipeline", config.get("llm", {}))
     base_url = llm_config.get("base_url", "http://100.64.0.2:8083/v1")
     model = llm_config.get("model", "")
     api_key = llm_config.get("api_key", "")
     max_tokens = llm_config.get("max_tokens", 8000)
 
-    headers = {
-        "Content-Type": "application/json",
-    }
-    if api_key:
-        headers["Authorization"] = f"Bearer {api_key}"
+    # Detect Ollama native API (11434 port) — use /api/chat instead of /v1/chat/completions
+    is_ollama_native = ":11434" in base_url
 
-    payload = {
-        "model": model,
-        "messages": [
-            {"role": "system", "content": prompt},
-        ],
-        "max_tokens": max_tokens,
-        "temperature": 0.1,
-    }
+    if is_ollama_native:
+        payload = {
+            "model": model,
+            "messages": [{"role": "user", "content": prompt}],
+            "stream": False,
+            "options": {"num_predict": max_tokens, "temperature": 0.1},
+        }
+        try:
+            resp = requests.post(f"{base_url}/api/chat", json=payload, headers={"Content-Type": "application/json"}, timeout=120)
+            if resp.status_code == 200:
+                return resp.json().get("message", {}).get("content", "")
+            else:
+                return f"LLM error: {resp.status_code}"
+        except Exception as e:
+            return f"LLM error: {str(e)}"
+    else:
+        # OpenAI-compatible format
+        headers = {"Content-Type": "application/json"}
+        if api_key:
+            headers["Authorization"] = f"Bearer {api_key}"
 
-    try:
-        resp = requests.post(f"{base_url}/v1/chat/completions", json=payload, headers=headers, timeout=120)
-        if resp.status_code == 200:
-            data = resp.json()
-            return data["choices"][0]["message"]["content"]
-        else:
-            return f"LLM error: {resp.status_code} {resp.text[:200]}"
-    except Exception as e:
-        return f"LLM error: {str(e)}"
+        payload = {
+            "model": model,
+            "messages": [{"role": "system", "content": prompt}],
+            "max_tokens": max_tokens,
+            "temperature": 0.1,
+        }
+
+        try:
+            resp = requests.post(f"{base_url}/v1/chat/completions", json=payload, headers=headers, timeout=120)
+            if resp.status_code == 200:
+                data = resp.json()
+                msg = data["choices"][0]["message"]
+                content = (msg.get("content") or msg.get("reasoning_content") or "").strip()
+                if not content and msg.get("reasoning_content"):
+                    rc = msg["reasoning_content"]
+                    import re
+                    json_match = re.search(r'(\{.*\})', rc, re.DOTALL)
+                    if json_match:
+                        content = json_match.group()
+                return content
+            else:
+                return f"LLM error: {resp.status_code} {resp.text[:200]}"
+        except Exception as e:
+            return f"LLM error: {str(e)}"
 
 
 def extract_workflow(reader_output, config):
@@ -54,21 +78,32 @@ def extract_workflow(reader_output, config):
 
     context = "\n\n".join(context_parts)
 
-    prompt = f"""You are a workflow extractor. Your job is to analyze a GitHub repository and determine if it contains a reusable AI workflow or pattern that another agent could learn from.
+    prompt = f"""You are a workflow extractor. Analyze a GitHub repository and determine if it contains a reusable AI workflow or pattern that another agent could learn from and actually implement.
 
 If the repository contains a reusable workflow, extract it into this exact JSON structure:
 {{
   "has_workflow": true,
   "skill_name": "short-descriptive-name",
   "goal": "One sentence: what this workflow accomplishes",
-  "inputs": ["Input 1", "Input 2"],
-  "steps": ["Step 1", "Step 2", "Step 3"],
-  "outputs": ["Output 1", "Output 2"],
-  "failure_modes": ["What can go wrong"],
+  "inputs": ["Input 1 with type description", "Input 2 with type description"],
+  "steps": [
+    "Step 1: Describe the specific action, mentioning the exact tool/function/file used (e.g. 'Run langgraph chain with agent.py')",
+    "Step 2: ...",
+    "Step 3: ..."
+  ],
+  "outputs": ["Output 1 with description", "Output 2 with description"],
+  "failure_modes": ["Specific failure scenario with mitigation"],
   "confidence": 0.95,
   "reusable": true,
-  "general_purpose": true,
-  "explanation": "Why this is reusable and general-purpose"
+  "general_purpose": false,
+  "explanation": "Why this is reusable",
+  "implementation_details": {{
+    "framework": "e.g. langchain, langgraph, autogen, crewai, custom",
+    "dependencies": ["python-packages-needed"],
+    "key_files": ["path/to/key_file.py - description"],
+    "code_snippets": ["Brief but concrete code or config example from the repo"],
+    "setup_steps": ["Prerequisite setup commands or configs"]
+  }}
 }}
 
 If the repository does NOT contain a reusable workflow, return:
@@ -77,12 +112,15 @@ If the repository does NOT contain a reusable workflow, return:
   "reason": "Why no reusable workflow was found"
 }}
 
+CRITICAL: Steps must be SPECIFIC — mention actual file names, function calls, tool names, or configuration details from the repository. A step like 'Researcher agent gathers facts' is too vague. Instead: 'Researcher agent (agent.py) uses LangGraph create_react_agent with SerperDevTool to gather facts.'
+
 Criteria for a reusable workflow:
-- It describes a process or pattern, not just a tool or library
+- It describes a concrete process, not just a tool or library
+- Steps mention specific implementations from the code
 - It has clear inputs, steps, and outputs
-- It could be applied to different contexts outside this specific repo
+- It could be adapted to different contexts
 - It has at least 3 distinct steps
-- It solves a real problem, not a toy example
+- It solves a real problem
 
 Repository: {repo}