From 8da8d703da298047a53025b276783f93c1855fab Mon Sep 17 00:00:00 2001 From: VPS admin Date: Wed, 5 Aug 2026 05:51:06 +0000 Subject: [PATCH] Agent Skills Pipeline v1.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 8-stage pipeline: Scout → Filter → Reader → Extractor → Score → Generator → Reviewer → Publisher - Scout: GitHub search with token auth + rate limit retry - Filter: Deterministic rules (language, stars, age, keywords) - Reader: Incremental context loading (README → docs → examples → code) - Extractor: LLM workflow extraction with JSON retry - Score: Rule-based evaluation (no LLM) - Generator: Standardized Hermes Skill format - Reviewer: Independent LLM review (separate from generator) - Publisher: Branch + PR to Gitea First run: 5 repos discovered, 0 extracted (correct — all frameworks, no workflows) --- config/settings.yaml | 79 ++++++++ pipeline/__init__.py | 2 + pipeline/__pycache__/__init__.cpython-311.pyc | Bin 0 -> 166 bytes .../__pycache__/extractor.cpython-311.pyc | Bin 0 -> 6675 bytes pipeline/__pycache__/filter.cpython-311.pyc | Bin 0 -> 3558 bytes .../__pycache__/generator.cpython-311.pyc | Bin 0 -> 5229 bytes .../__pycache__/publisher.cpython-311.pyc | Bin 0 -> 7640 bytes pipeline/__pycache__/reader.cpython-311.pyc | Bin 0 -> 5096 bytes pipeline/__pycache__/reviewer.cpython-311.pyc | Bin 0 -> 3476 bytes pipeline/__pycache__/scorer.cpython-311.pyc | Bin 0 -> 3011 bytes pipeline/__pycache__/scout.cpython-311.pyc | Bin 0 -> 5267 bytes pipeline/extractor.py | 168 ++++++++++++++++++ pipeline/filter.py | 80 +++++++++ pipeline/generator.py | 103 +++++++++++ pipeline/publisher.py | 136 ++++++++++++++ pipeline/reader.py | 87 +++++++++ pipeline/reviewer.py | 84 +++++++++ pipeline/scorer.py | 56 ++++++ pipeline/scout.py | 95 ++++++++++ run.py | 164 +++++++++++++++++ runs/20260805-053642.json | 21 +++ runs/20260805-053718.json | 21 +++ runs/20260805-054839.json | 21 +++ runs/20260805-054930.json | 21 +++ runs/20260805-055041.json | 21 +++ 25 files changed, 1159 insertions(+) create mode 100644 config/settings.yaml create mode 100644 pipeline/__init__.py create mode 100644 pipeline/__pycache__/__init__.cpython-311.pyc create mode 100644 pipeline/__pycache__/extractor.cpython-311.pyc create mode 100644 pipeline/__pycache__/filter.cpython-311.pyc create mode 100644 pipeline/__pycache__/generator.cpython-311.pyc create mode 100644 pipeline/__pycache__/publisher.cpython-311.pyc create mode 100644 pipeline/__pycache__/reader.cpython-311.pyc create mode 100644 pipeline/__pycache__/reviewer.cpython-311.pyc create mode 100644 pipeline/__pycache__/scorer.cpython-311.pyc create mode 100644 pipeline/__pycache__/scout.cpython-311.pyc create mode 100644 pipeline/extractor.py create mode 100644 pipeline/filter.py create mode 100644 pipeline/generator.py create mode 100644 pipeline/publisher.py create mode 100644 pipeline/reader.py create mode 100644 pipeline/reviewer.py create mode 100644 pipeline/scorer.py create mode 100644 pipeline/scout.py create mode 100644 run.py create mode 100644 runs/20260805-053642.json create mode 100644 runs/20260805-053718.json create mode 100644 runs/20260805-054839.json create mode 100644 runs/20260805-054930.json create mode 100644 runs/20260805-055041.json diff --git a/config/settings.yaml b/config/settings.yaml new file mode 100644 index 0000000..cb54326 --- /dev/null +++ b/config/settings.yaml @@ -0,0 +1,79 @@ +# Agent Skills Pipeline Configuration + +gitea: + base_url: http://localhost:3000 + token: "b721b5f288d227e9336241f95319437ab40256d6" + owner: tonyjbala + repo: agent-skills + clone_url: http://localhost:3000/tonyjbala/agent-skills.git + +github: + token: "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" + +llm: + base_url: http://100.64.0.2:8083/v1 + model: /home/ty/models/qwen36-27b-mtp-gguf/Qwen3.6-27B-UD-Q4_K_XL.gguf + api_key: "" + max_tokens: 8000 + +scout: + queries: + - 'agent framework langgraph mcp multi-agent' + - 'ai workflow agent pipeline rag pipeline' + - 'llm orchestration tool-use tool calling' + filters: + stars_min: 10 + pushed_after: 2026-06-01 + language: Python + archived: false + max_results: 30 + cooldown_hours: 24 + +filter: + categories: + keep: + - "AI Agent" + - "Machine Learning" + - "NLP" + - "Data Processing" + - "DevOps" + - "Web Framework" + reject: + - "CSS" + - "JavaScript" + - "HTML" + - "Game" + - "Dataset" + min_stars: 10 + max_age_days: 365 + +reader: + max_files: 5 + max_tokens_per_file: 40000 + +extractor: + min_steps: 3 + required_fields: + - goal + - inputs + - steps + - outputs + +scoring: + min_score: 0.85 + checks: + - readme_exists + - examples_exist + - min_steps + - reusable + - general_purpose + +generator: + output_format: hermes_skill + +reviewer: + confidence_min: 0.80 + +publisher: + branch_prefix: "skill/" + assign_reviewer: "" diff --git a/pipeline/__init__.py b/pipeline/__init__.py new file mode 100644 index 0000000..26c476e --- /dev/null +++ b/pipeline/__init__.py @@ -0,0 +1,2 @@ +# Agent Skills Pipeline +# 8-stage pipeline: Scout → Filter → Reader → Extractor → Score → Generator → Reviewer → Publisher diff --git a/pipeline/__pycache__/__init__.cpython-311.pyc b/pipeline/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..42e7b1144e8f6aa80c6203bc7615fc37573f1096 GIT binary patch literal 166 zcmZ3^%ge<81p23ovO0nEV-N=h7@>^MY(U0zh7^Wi22Do4l?+8pK>lZtA{H)aEl4AYD^whi(-Qw)boSb6)g3N-{oXosb{rLFIyv&mLc)fzk lUmP~M`6;D2sdh!IK$Adr74rj$56p~=j2{?aL=iJk3;C|Cdh literal 0 HcmV?d00001 diff --git a/pipeline/__pycache__/extractor.cpython-311.pyc b/pipeline/__pycache__/extractor.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..58ede8c62707fa722baf9cd37e87ec41382bff27 GIT binary patch literal 6675 zcma(#TWlLglEXJSBt=P-NRyW25v>;$iIi-|b`(%>$#Se+5TiebP?*2d14Plwz`K5~HX@36PnyRQQ&5HK-70O1a|fd9D223YLZ zRnPFHhjZH;_H_4DS65Y6Rae(Py4^GdVHV5R*cogcNd}YghJFE3K@`5 zFx)X{Z{r=K_BP!yX>ao#GctWj-LW7u)K+9%q{5c;EKFXM*%gktFu~js@%pkLZZI>O zGG^nlh?)P+Kd$u{%%vrEN#K}`8r(7#6EK_-nYfsgxlK9picmEduMtOjR`{FX|J_eu z^a+(UD9|}xkb$S31H@PLxjAFjs2m4`5oAQBBe15|!kUrg2(0&3H{@(t3$`O?*795GnK5fat}8%u){NX)3bi~nt{Adb59H`9jXDGM zKJqoS{Y~vtjrJmi0?lgzbwTJ`YDmy&)bVR$10FaxFH&bI*;Rv}s6GmJ<(w!Wdm3X| z=W_$l27kQ}iN%e#DResLyzhinwfBZ$gW5FRrZx=WuFT*RD+r9d$`Peo=2y~~L(J^# z`^+-vE9fB6TwY8@RFfbiRQnPuaj`TOGJ#cDPEACk<6~oyix(nek#iH5$1b0b-Wyjf z2@!FEN~tzB#mCmUEtO8Nn=x5j=aLftlQR^RaZM2oPs*b=w^CfDl})7tKF*SbqHB_v z%-F7SEaotyx+l}}s)+fF_EEJ;TawHr!ba7MMS)Xonx!P=l`p>_+EzIhaadAqGN0hY zwEXHn2^w2`RpjGHhR z3Q~iOk=A)ZkfJF*#R+_pi&j-Z#F5mNYLA1`#DE+;2xtif(grA9o;qmrKM6hx?#?LZ zifsd>wt@V6Wl!MY{Rj7V*9yZkMbEX8=UV>Vvdi~y?!nydCFN?-HCS>D=4Z++ryeFB zBzJG`8H+7%lv>`%UoW?ve$w-(M{(_;V%zyr+xh%^6)WZN7fuaVOh%72e;vAZPlYmA zE|t$-+`Cmcdl8;(+m5GbXW(1A|DdybZ=m2BhXjtf@i(xSuc6kx6Ngzr{p}nEA2Z&1!t)2^*?#*(OaLt z{rK(eY3S@a9>1fEm%RN7U-FK99VmI zP&sqfq||*<^9qMO=`?kB=p(SKoF!+?nw7fuZk~b6_8snMkZ!;3P2|k%|LZbF}QNLBP4{KD@>> zwPUyo9@s&_$wIu;v2s`*Iak)D*p30g0BSjm8P5YGF~ei!C?xVUPL&_P-+BZ;opqr$ zWNns;HoHk8jirySsX_i-{K^oP)yi63lBhlFk~?bS6tz}A1)sy-epTP!oIC5r9a*#(6p#i(;{E9 zyg;}7e~%U=;bew2ZMQ*NJ`2=N>MWgaL8r2yV<^y!HETq{r=~?JODi>^4!t3g=sK*Y zQV?~&7P<$WISdW>nIFX8yK*$nY?044NXQ=1ROGHo%8$BH=!YOdUT2>{9QyyP--r4S zQ;Y_xD+JKsYx(Ruyvwl45aAMqvqp&KhDkg8MkY;@19MU)!Gp}mVl5tG*w;nxB#t^1 z!D|>6IEXuv5(CG2I!;0m)Cl?^%EBBucuA0@L8~;uAn|Zv*uup8_4vX(Iy%Z^EDXHp zIevEnLJlKMvjT*ux5YFA;S!Klze_ZZYX-hzW=&jTc!}`9CRt%CLx8XI^1JCJjb42< z_=Xt7$sC4&fMfV&hL^Rw2+ITYO-xVTXsiSbq*xFUPC^vP%78-zDPTqmBVcw~K#Ty< zT#_O8n+Q$cSk}VL29qZ)QR8H!E_9%UPXYt-Dlo!rLR`wcw>Ur7km>~e2?#K>%1W`? z?9c=QGUrCfxE9gGl5B#5(U1gDr96r_DUNxFy6$nK+7}ISMPvm6GY>+LAQg+EhN7O3M;V{(0yIX)@!X5vJNa7rLtxB6BGnWD#0Agw{Go zV71oyDkwyrb$s($TL-VUYZ#YVUI4XCC z-g;A`sCE~HvAVb*3WR`Jh^5jPw2^?W z&YiUTNaQ4DZvJLX4Vo3LCg~ACOl;`nEna2?N#A44u@Ju3h)RkLdTN-M4OW7?O*)A} zgq$hNgX;3EcHL>x@EZ68J<8mW!6=BHECHb`1A#JJ0VJlO_#+JHJhPUTK>NYOM1g!0 z_$AEZEqy-dBoilA04hv2bOf{(lwd>?Q+1PdNZnwTz&^>H2s{O7JmQy^ISh)Zi3m&^ z76&9e;&i!7DK5@0^Ks3T8w-L4z>+}JdB7mad=e~1TUG~=MB$#s5C}~mTtPA-p|e7` z*djJYu17SzQ1?d@OvcyTuMsps^J8i7NY}|nvQfo|Nh+hc z8(lt{M}{3B?jpjxhCrPyV6cat($g=@&@zufd~n4b8)M+Yh^6oZ!?(hnhy4)9qUmcQ zPOvg49sM=cIX54>HNWuFYqRsWRI9#ZCR|kzu*dqj zhKTmnBvL=e!X_Lgns%O48h(qm1sThHQrKcLLEUt8*D=Dt0m&!Q338^MSLuexwamcE zvR3}|j@kmm#HQxwuHBfPnVXuaE%Ii45y#AjHy0+SZq7{CzG-XQYyKzPj!zM0E#RgS zl4=F7z$M{wzb5iY?eeW!Fq>TARG=0PGL;4~VOiSX`y`nYlk_k~Qr+6frW{KVVj%Is%BxxwQ%%@_)j}2s(lU^vNDFd|BtYh`UU3NTo?aCI)5}U&R*yL?W?zQzC;j@c^0IO0Ym!P(S_g0t^q6Iq!o%$x{`Y zas^&GyY`)3yC3bHD>@@3XCyye_V>N?5AFMh_5xpDE&4B&{Fe&m4oDSr`t$FX1L2o} z=zbtt42+iorWSikY;{i6M9 z$$qtIaNjpr^o2{l@b>kx$M@3HyYJ~$`iq{Sl4qzwncZXMVDIlnzZ~7O6oZjc zFtYcHQgD37R`v#8di(bw$FW@Wj+VTmAcwZ$a!(pmJ~TUa{w5spsO3bH{nm+PP!du~aNn zTi55#N;fP-vMqg-y|rNTANbCey*<0KV%n8U-oA3Mr(!d=(jYdia?lkQ!OmCsQJ!N08+|dP0x*%hwR~D<}1+)77r77PvkMWP5 zOV{0&|8NfxtB_a6iVk zV2m92(p{(^zhy9#sf7ZyP^O%D?N_E8dF@xGJbCR`AE{W+7$AvOuci(64F(|j$x2nCZmq#Hn2E<8>&`%e zHI`7-2Np#NMJwTIvy!zBS*pCCzBG@0X;rB&GiW5Nks?LyQ{KEvAEKxaJ=ca1L#*cN z`pms|&fj;=x%ZBL^80-V+Q>iggE1KWn@si^wL5qdguw>l(S5{Izeo2e-gXzo?F$!R z`L4=k1ord*C!9cwM}Q_e!>$t>n(xOCAbkP{|8FpXRPOnt<+3p273^ z`xhlzLz8HY`sQ7U(U{8n^$!QUuQ?(-ovp_hQ;%a{$=_0r;r-3t;;81YKo$=-o@us0 zp_W?Pcu%ud&6%w$#3L=W2R>cf_lOT_&hOC=j!*Weoewp8RBF@OFy}>(1J_}!2x)B< z_s7=xFdu2w)a*kkpwU{O2)pqC*%kGsxw7><<6|w+M-NK@%8noG&DtrHcOOR-x`*=i zNA`Q@A=NT!ore$ywV-BSM7XP^7hPHqqG~&6v!#tA2lHHN*V_5+-CQ4OX{SeP2XwuM z(Y3S`Yxb!?@u7-$FTNi>5A2;#2JuIKq)K5eJc+(xOA#%ig&+pqEq&|LA|HzZM4oyK z`NKV$@8kPJV|Z5c^T(bZ%GnzA_!a*dy!R!*Rn6NvbD;mJ6Hfcji@)=+&@L4U(h|!zo$z=;5)@XI*}29YATOw_h_N6Ei(FoXqrV_4 zr|QzzWKNo31xbY4fl~!mo?R5>1*LIE48cQX3j&@K(kh#lI7JcX#5AXha=syMxf&KM z7Zx4>ljXj-#Pq=~$e z<0J_**l%5qTQn~yX)G2<0OEejo>r7%JC_%8^^mZVIEy*FRAkb!BxAN9rn3Sr24-bh zRaDFsSan_iq>clE(Wm6OtiaZJ-2jkN1mMha znT#Y9J!xf;mGjb4(Mw*j{6el!?D)VtHZA8>PRuKS$~ANK=G7}-$L*GbWFBHJq(ucF zSZ*wEikw$0&(w|EUtgWMI%(OGx8hFBkgP#23##SW3zX#}^Hf@1$g5V{-fCkWr!9L1 z{Ii3Tn0&urnrN`+<%gCFFbR3y^3P#8Cn_?0Za}<t-yM9MO#2Cm6LORg z7Kr8EEusBd;-QpDVPH}TmXAa>)$q`-WW_=%D=a;fF|Sy*>_dfYWU>_xwRdA9^KwoY zSu7|V5BVM`V3}+aWduqvuMUGMNm51%VgYtwUKnY>JXKh-m|ev6IAC(bkZ&p_33M6N zd??zpadZ7<`O#*`hzyyLq2)U-yL!rZc6x`mdxtmYU)YS^3uf;Hqbp%{C3HvhWyg{7 z@J{#OcK6`s)YfIAd(7+}Gdjl2j`8JWH5l6o9@`Ecs|;>pBY4^jo?gCD4YNDp{_Svo z<+odRjqrIhe17@%E60%jvHptoFHcqj?aSAzzR;7Q)uC@sJwCO312b`0wsXSPB+(TVNogb_`c(Zq_cX0si+M%BgSWqifEZ69w6t2QlFq@_3Iz z4LnJ%CaYb&)!zPU&u2BaBi2?!Kz>FJf8alaz|aDWF zU}7c{)0r3y>cN1hRwKQn@caKDekZ={j+N*CIQ(?DM!OgnWVcS7F2xTU>u;Txx~4{L ze;>6?opnybz2r`%csZR))wkZC&>P@EnEbtZ=Sk#b9`4s);ICjx{;w$HG^yDsimIYZ TI=WPI+@UD2@NtTN3Eck!oH<`t literal 0 HcmV?d00001 diff --git a/pipeline/__pycache__/generator.cpython-311.pyc b/pipeline/__pycache__/generator.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ae7d462f37b939abdbefe74ca8169b7816934248 GIT binary patch literal 5229 zcmdTIOKcn0ak*UnmXydPZPEH!E7ML)SrjchmR(u#PoGG!;@DB#IR1oQ@vS6EB$wS? z+7huub!ZDW2oOU^A;Ohmo?I2q!RX1ypqB>ip9@TB&TRpccL$U1?D>jbhOGHJPPF-Gz_ ziLAdOu3Hf~Lxid9`@l9MbMrhqG|bK{C6Y<@9X`#AoGgg!x2s>UABtRBniIqndsh&b z=90o)Hj$PER+70i;zX3l@ra$|#S|}@`v+cN6E4MTmf%?{e5~+!`6maqN+F6cX?&37 zv4tR>z{h+LxrBESq77LQg=`yObJM-{1o0SjKB?=?lL}ch1?_^!ft)*ei*=o^vp$B& zlRBYTms}?Sf?SB+CF}e7+`IYm)~3FAP|Hr4c}lUO)~5b=ck?&p^v&-&rKX(BkLL96 z=A0)K3bizq96)<^dWZIISW(-CGjBt|yj`)&5GA!|k*Z;e4edkwzHEOCG4aHR3%rtR zAqcz&b=0v=#rk^!bv?A!C`~PCI5%S-S=)!YO-y#zY#@lHvKr0{?ceF;7;(#Y6JjIp z$U7Cg;s9MeiW4CFkTUgr#(*EaRqf4Ux8Vjst~>97od*<`A+757XC9!1en48o-5Jto zLmJ&6?cfhcJG46z3~6pdn!7>T8&zp`^rk|=>R|(hAKL2lzNV+$5JRs^M@MQAkoPDa z)NjbcSOg6e?XT|g71mTw!)foVP@px6j{ZcMjgc~lj{TZ!ubb1LauyWw6NWWM## zVV(;*Q9rE~r3DQcRE(W8=%=cAgegBWD8qHi|4~h9dczGd+my2*=9+SzZm7pp(^@~$ zPz~SSaOYdmnfk4G6)!q#+zQ6d)v@z+>_QzItz%;*hQ<}ELP5RpC@nH{3$>QU!@+eH|cIY_d ztfbpTo|9mcvnEcbCET~f(p-wy9oh6!TDY6mEv#;d=$4pHi+o0q641}A=tRyoIB;~} zs7}obTvE3u(wVF*={8B`Gm`EQva&I!<3f5af%tTs7x6#yuvK?ULRO6PF)U2?@XNBu z#bqAFI9c3_Ih;CW>c7K_hEHz9OK~xgk-1C|s)U1~N(52;w{PFhtjG&Onhvp( z&`v@#nU#1}UPwr+ag#tdtp^EYKga~aWl@Op63EQjz*F|y->k8hUf?FcSgpDS{5BfI zgHa)NR7jZM3}80*78VC+Ea$Vm8fY zgLS6yWj>yj`Fc>(x|?6lB)K$)ZyZ7j$8gv!?t>j7F6|2BP52R2I&e04Y zV-l}kBc8O>GG<^ADnQOD;}e2~uua4rqlmv2iMYn;wmGO8lGp`!*ri+Nd0DqXO7WuZ zfDBG@abBlZxKvW7P&Sp(U3?n+fiFEkUKA2gk+1^Yh8LxHEZ-ICBLd>^_M?m)M~f#~F13Wtm7W(;`TH8){mSgo7R z!Bp~r8eDu|f4&=zE(j?;dM6`sD3wS@p~NooaaoE&vZdvSp0$ZdZU^*x;j~A+Sj|*x(dn7DU})0n4xWEWQ!S5nKK%5 zW}BJVVkW+lz8+PX%Nlceo4K~dTq}Kiv-An4GP4>pyZUa~9o%+zZn--P)0+oWcSLhX zR>#0Nrn4OAC?wEcFLxYx)c>Hrs64%*cAV5YPF5V$UhohDrX^sn5CB#$ zR`wIVAUH;Oen)-bDja!s=<5?D-xbw&MFU7(F_!MTRI1&;O1?w)uLCe&l{#dupE8WF zZ<1=@h!!|f@(r4=N)4I}UX|)uo%|-yRk-%(=7XD?Z$3Syb`NRYL(eQ~U|0(bR|t#k zkO6+{xM#WdR@v#h|BKR|gYXu{%n_jKJb2G}&k5F+d$ot5g{Fwzhqq{4`lO3wPBT z)mo!?FKd8%B-VSe;J<$n|EbQNy56eI*!&a4F#zyxR!2Zp>V$DzCEvNi@+0MevN`>% zSM3?mdPaa`zAANYb+Te70>_FSCI6uLu3jut-cq1XrTPrCLJ)gqtzV7(W$MpUUoZSU z`PXFWM!dw&tCtqEOAG2)LK{n%JSHqH|MEWsv0@n~|78w9o**xgK%5{yz(hO%aRz>C zf#D|k3HeVIx67%FWD@eox%p~$-Ji|j>a$O6o+h(m0>)(HBQI(2T z4{)?dmja{at5Ty?>V?8$DR9AjRq6r=swJ#%-@0ebQ}}uD1J!p}^Bw+Q61KhfQP+d6 zqVQBy+lRIG;d0yl_2gQzIP#QH+m35($7^NbJ;<5hkQN*&2RqA5_cqhJ#q<{6+#FPy zh{i-JL5~lz8UVxxWOaXGxa2)zzSXEYtWt-o=qZ&tRYeDDSz7WPQ>kNB>PeM4Sw-Jg zskhCT+h1;D%WWOc_p#;9H{kuIyLaRIrt=w7>OQS@pVqohm-n$1x7`ng27^{&Z#UTG z!FCzIW`7Aq8GfYG9kCb^;<1?64|DM-2Im0$kr2;g{49X)oFnk>Ih*9qh+{Ct?VxlT gK*egYSjxoJ5^=RmxL1w8I#Qu#Ef&!C{m>BpUmkV_`~Uy| literal 0 HcmV?d00001 diff --git a/pipeline/__pycache__/publisher.cpython-311.pyc b/pipeline/__pycache__/publisher.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d5ca4188d2db1effda5f4677b273b23843c36007 GIT binary patch literal 7640 zcmbt2TTB~SmgRT(0X8oiFoXg_cqCxxBoo?8GvSp$LP%ml(jnbLjBf=S{7O|hfTz5} z$BcwVk)}6_1ezh5+Gvz`wp&_uwbJG@KdX(pf67*Au}X>*?W(2iKW96wZRKmvErS6E zOBD&r^?k=Iw$>2Df8oa7rQWUlcn&HX1R`b$q(bVDYF4F` z>RGi?YGyS`sh!m-rEXTIl=@jc()^Z~H6Zmg(W#C964*?0z6F}<_fS{k^PwOcpc(2v z*B((L4DI7+YM${${DG5{KN=1Pxsy~hMn|ZtUMd=)E(AH+*Yg8bY0AH+8E#d;tAY2& zZ_QXso)Dn%{)JReRRr-hy!i{MmoOE@&cLSAqYM5Z8dm?o*_)g^Gv#r%_+lDHOBR>W6Z61N;#i*R)%ac!ug2v=Vcx02U_ zy{dSf0`&l^AM1;#4JEBv^PXi$o-AP*d+|IajU{m%@9{Nph503j#e$Cr2YUCX?I~2h zE;I$jiwe!%3aUNqZAmk#dfxme;;}Yq;f(tRIajrhv8cp4ZX+jZK#k}aYT^xi`GNsaa5sO`@>C0BgExOAt-N&+qAGtO z*KWj&mFB5%(|!=QW2hCR6h?{K(DC1?>-K~asC`TAS3@t{hdoC{OcO_IvOu7YCn~=D zo6cg2lK5RE@C$nVKdfJlPVmO3;16FHd?IP%$_hgv77KOWhNy2&7Hh|wgz|mt{n}x# z7$v@({{lQSS&^*#An&m8#gSXZHSJ5`t5El0jlln^`D($okF#InD-V0Pia2_XB4!uS zN3hyu^@`>ev8+-Nw+I#75Bw$Pq6g0f*vEa?mZ}o74Zhrq`p_pze4Q#8^{0!YUf3VZ zff_nfoJk;RfAP^=&mz}h+d>x5FKCY*CvOn&s`lQ6`q8Jn2K4bf)*R-RtU216V4J@| z1AGnojJJS9^@6TMY}$GI5s}_slItM)C2uLEe@QAu7ohtvEsmjcpj{ynKP*EiE6~t^ z6=?XeKIzDN%VRYfQM_jKqvv<5cx8rJI{(pJ&m6(^!bfvG!WYr_M{}Kc-^03sgw15olt$0B;LW-^7ZK2H{POk(#bm)VfUQB(DahBZ(stvQ?T##9xM-Qm~24T zc_VM(8-#;Mg*uuk_JWd_H;OUQ&62q9aaAnv`If%)YKv|i94FKrdrB6^B-nU1pK;Lb zEnR7jN3RX24bBT+#D5LkO%z0pJr(5w!N>v?WT`OCEC9JL&PBsMF6j4#LaWO8BM=Y!B76MVJe{m(IUg5i zWpe21RqvG>W23TRc=F1~rLob3)l1(G(#s&3jfXf7B^%AGYuP! z$V8`_!CNW@@7ZJ>=t%oy9T#1qBQiPfW9hj#6H3$sI4C( z?-lP<@GWe%ckLtVXoiV0omvJ5jjZ=aBX@%f2~(VWt(QnBx_Kt|FlL6)0~F3GCpPEB5@#)r^5 zRS#%H0VAeJKc$j-AZACJ?~WKWNj?Y20&swp~DvU$fQ3&`cl%;N_x(OM7R0&}dUq@i15B7)u7ai(lc}rgLev-Xbahd(b^&UM zGU%FCRK%SL3vMbp;#LK;MzH-69r4o&l}VWorLEBBBVd;ngt@5m*riPV!tn`=hyDH( zyg#-8xCGlid_e!wlvE)&5W^CwdaSuYY`~`Qfm!)Uph;-DsXGbv9n;ib!7pTWPp_=H z%VNi&sCOQBAR4CK_hYONg@X|{WP?T8&#~@XwDPSytIJOJGf5W{-h>b(=r0#8ogrUt4zVkf?;g5g~|2QD7gItJ~ zwewN5Dr*C@4?!JbJfT+to=VstiG?8ZaKUKAy$JaMMp-r#=K@hCm{58X7O*^rqx<@5 zjQ+uj8Ea%M#+412Bh9c(1N8pz5`b*HIx{>uF+C3Hi41Vz5Y8sDE*L@dO2Ul)oxnd7 z8hNNreLf4}^PsHXjc0v~QSgmWM&jXlNHqosFcu6dlUKdFE_P;E`7Z-@gC91CnvZk% zr;H!}vw=}icOf)s)B|Zc|5-r>fjUK?Y{VWu=Z}KkI887k7@P;tSqoW)=42A$g~mBT z)<6oCNtR>oDov83!!ewwj zzN}&5(817wHasaW8mjfjU$rdHi11`>D3_e9OcnizmGoWn5 z!K!3p2A{659U=^#83u1WWHpF1=cg#2uJv3v23li{fDYGSb-}V^qvFI$7l}=!WOoBd z#U3OpUD!qpjzXD)l`o)7xEasop1z~_n+aSX=35O#eTex2Ychy?VYOjnvH`x~q{cQv zA1uhFFXl9+DmxTm%suGCzdP6&SOV9GoQWW7b_}%{L+xg5+R!2yT7<=np;t8YzP2}I z?e|kGv$ap zFq%L_b^V6tp=WbJVA54xQdQU5WY$uI@YRJ)Z`wjh7HVzmCp}@Rl#KN|#$y@dv9ytrjMR>?Gh^&bxzk3EWb~{JXDd%` zdc?|;@TAoFa_!Rh#tO;Unl>JnjK^~XtvJ^_+RK5^UJiuz^7ZpTXtPUJN*Kym+eB;I>tiRf zjjpXPQ>RlSsnb$pFHqE0!gK;?#0uxtJ_z3^19_{!gZ)2s5XPzp<)UM9_wmk(;aT^={xpm_!()7Um z)kI1wlHF;tTOzxmlG76{?a!P~oPXc=mqt+4*sOrGrEz`ytA$sNx<|_!{6k)tNUfwD z{gR`9ef(8L%~t_Ildfo$Dq3@ds;oJs%GxS63=a*PhO~{6Y?NrDavJ!cd{w|3jQ19U z{l24WKmYvmk=;L$y9VhJ9r)QMUsj9cRGOTU$SJ5O7QvG~zKL=K(Z;K{yYgioir$$W z@2!mYmN<*VyMeSfD0zcB-f+en79+m~oWE6FP``mc`>JvE+kA159O~2R8|weX#jN^& z@&$C>R?})I7Sut7SiPtaL!n?s@|Y2f5>?;F#fo}0kD170CU7TcTYzOrb=Fp!ss`UN zVmCEnH-%K701HvwxKq`dscIE&iM^9A#sK7>bk$|4>M}ULoq80(b~wK|Ds+e^hqf;P zz>j5kKL8wnAXjZW-JcL+$>znn2(F*4B%coMyr#mg5Wm-OazVPB+x@AIYnaCLm z$RyEm;@R?(;heefSs7V5;i%hKc({&1V*_)nAnG70T;XK( zsG+je^*M*d4ju+z-IQw}jJ4}le7|n&7HF}t8=h=K56n{)cEc*ux&ddt+CLUzHLl37ottzOCMUDv)gTP3q|lg*f0L~~2lUcV8182r1XM@#D$Ue~u}k9A}L);It8 z#2-%xi`y;f`ZH4fnQUY0pKt&1cFF-Ajc26BGg;^Pl;aV(t6h;VwV32WrMBToNH1n9g2CdDE!_|1ok zcrTIXle&`Cj@I^I)@LVSd+~1_!(~ITZEJ%`wVO&kF*WD;Qh`0Q<>6 zoC}7Si?|2xZ_II5m}M6r8ce&_C5V=u(*q@wBtBR5OE^8 z#cs0(q1&L|%{?Y`v;m*DA&qx8Q;7(~*&9(i5)D(jY5iq1MG)vRloCiX=9;3o?lyS1 zoAW{&Ck?n(aK>@3#-He_bH=nO26{plha@*bKbH}0xPY>Kd(5$eIML)N8)CA#<)jue zNJU6WBE!@)De;pGCwf&vP)+O@#|w;yP)&>=@PatVcHKfKR$;Swym2Spa_C_+i&Dlw{>H863Kh)0hk$)_E@aX!rWu0=$^4a@m9 zk|g>dtDMv-PO_np=!>usCd6`#4|mK6kWjS}ASqBmDS)vV7!~+%06P+jp<{fDpMsc0 z8H3rlXWdyk?_TOzwxpcxva?-rwr3DwtzUU1UAcF0^ycW##uvut&SdmZrd4R-5jOb- zi+tHm+LI?wb}IhbUNkrLw*Q8I>EN2ZK4q^@SMFaLU5YM^F7PRLw_MphcP3q2Ial-~ zqa%!a(#7R-(wqdHi7p8I$7S6I&_cy={T&A=pBbJv|3(kW?}_Kle`o^b%LBUZW_>V+ zA!uTTTSM*wV0aP55lno7MYN9A-!cLgP!bHDJphdbxJu>_w$*mK6>ZF;Eur6v(}~=? zD8xR@5`$#P_c@Wnb`s)!_J|we`oejVFj^5TUuN1uTjMBh95W?M5>EBTF_G&%ZlZ0p z{hp(6p29qU_%?lKDM=wf^pdLz*0jFHMVJs{MJ=83CM66Ux$sV&m^ znyj{coaPDP;Q#d2#<;mKj)Y~G_V(?*@pIf#m`&lIQa({gvgX@Nz^5g9Y;RN81B^_` z*0rSY1`6MrfcbvospQFz3tjORtQ%YNoUF*#=Cwl1^loDUgg!!Z*TtP6>)$ixdLlP{?Yw60 z#$+%P@ifGKYJ+b+*%RV9h7_0x-=dMq9K&SutTQyz0v5mlIZ_0cp|L@F)6lDRGo)3E zB+rFr$fl*^a1;>XU;$(Sr;FOABnj7GwQY0#=W{C|mpvN59AsVHC z(a@?<<6eUg@fXu$is>;ig=DdSPN zA0GqpB76)nQMg*3APQ(MQ3$$M(INp$&>k8-iFb%W4*+m~aGhQ?+auG+v;DnO@kY1U2C4cu_TI^w==L zvz%%Zqi+BS4l<&s>VaOXwwKO#_Xf@n_4W;^7KWq6DHgc2krinc2)f7%K)+RUh!rKR zfE9BzSW6Q`p#@{2B0$Yyr^ZZ!DfBHy!>Z{T6^b&Vr%1KnJO!dcNHxJs7-h%cEf_Hx zl2q8zK@CTR8Pz0(BiM;eLwP`p+M3maun9Cnfy4-$Y6c^%(m1NMGmhe?EF>{)QSA~B zJIjDJa3aPHtO{EW4NBPY27ZJWz@!Fl@sGeRfz&(D-uiUSzI0tgAFqrKIUW$)UB9L(50y5}#7yThS>cM`s7Vww5Z^CfVvztgZ}d zF*Yx|)+_cdTw1(x^GedS?2#*uDHX@&Y#Ec$T9dA>OPZH!Q_sC{|7_~j*Hcy0+`zh{ z^2XrO^DDz&97#F)Wku-)^6gFCbgZygW~$N;(B%yx;jw=kh_hwoR#R%OFGb zMPlXR=dXS8TB`R=`M97Q7qW%xF8}I9+0~=CdX~tTp>`!5n7A6=gMb>dfvJJ;`B zmk*y(4xjqsz}n&7)ZyOrz{^Qfaz-KBS1zTF^sO==;F8JyH0eq;cPQld9+0Qj$Wvc* zq`EHL?*jpsOb#pLaEcs$^6zg=XkQ(K=YPk)Y+E@a*ZP%O|GXtT7M(XclWw{Euu^^) z!Yl12KGS__{=DduqSa}+?X1#vHe0%0cWmVca$ToV*ExSCy|)g6uIMIyc{DloQR0Kd zFRp)hU2g198aq}GtTp;mjsEm;|B`8GMyc{FH_KH=(p7b-dY@9&_Moa`t*T?yELU|Y zRb8p7E*#z7n(S(}>{_W-4xM;#==9p5)AFHS^ZR=X7iV}hL=w2zig>L?a+N?CqVzoVLjbw{Hj9J8(%1bqCcNBoFmMCF%TdH z%+);>)f@=GvL6TtxEG-X3oi)Ofd2*(@b^vo8$eq&?jYeHF3`SC7JrzhqamhK7=$vs z(-k8iVCPQ|Y4ltQJ(otMsm&{mif6SmjjXfUnMU?m?Sy5aNS8sIV%C-^vJ!hTC?_gU M5@i{*i_jwaFO-ti%K!iX literal 0 HcmV?d00001 diff --git a/pipeline/__pycache__/reviewer.cpython-311.pyc b/pipeline/__pycache__/reviewer.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ca25fa3299112d1e65f7bceb39933e7e4bdec3b8 GIT binary patch literal 3476 zcma)8O-viv9e@5dV}k()2S^AHW43V;Y|_ncTDP0sG;x50;+5HM6snao@q31W@r?Cm zCWO>f99k9arCSsUnxmvHl3p%={EH>j74Cc!4UEi8>`%Tb5)w z_JaGGjoM6aR+Qz8Ea$T=_ngxK9|wG1{u}5PDxwk`USGua)P~SA_%s%=FLN!#T*N_~ z#PtM%JXY5&p{L;U*}mU3d(mEM7;2-$L%eOW4$@I{`~eZ+kB&8`Zh37e6`e&}o2;t? z6^n(dPJh(PnumS^v= z_IgS1sJ*q0qOJ5MlZc#X@1SGk`benFrcm^flgn0ySVys=C@fofWe(E6=2LoJ<68cK zI{;ie8C>%feMK+m9ha3;?frVJgY64fktQWbd>i-iJH@52+7f$$Q|>Ka~=^hKPb zl-v`u%MDwmThm3|(9Dj>koSlg_IK|nUbiT@oJiGMnz3k zOjiz`Y7f10x29)iVtHaB@$UdxcT@xOz#AYbDGP8Te03UM1+3KZ9EVA)FN!)|5j89t zx|$btiKFEzrVn%`W;vSXn3{(?O-lNdaSu!tR87)Vwi*+J+o~ZGZj(MJP$p3&3-VN1 zkQ55ZkQBI+<_^9u$`Zjh=Tfs^0|Er~ zoXQ}!gzyW6`Ga(3~7iugBX_}|`dz2-Fam;(tdi~+N0haGkY()nJC2H2lL)j?-v1xsu�k^ z_hgEpE*d1xibiD;6j789l^e9?tShz*YXvAnPRc^TRbych>nfy@W(}Qk2TB3VSpx)i zt$j;xh05|cp*d(09?A`9Xajl_MhB)ugZ6-yg18UG3pdmarr@^bh~Zfk=S8+m)D)tPJIkh*)@~8ow=5py)k_~IeRT>deh09$!jpLjK3Ak;2Rpo3^c3T>VT-n z8ei508myZWuFFnx(htP!ZvL22S*DRb)1OX$Je|CqNv6}OwDcdiXVlTa00$FS7=MLE z1=F!Wb;jTBrc2YAR4@e?L8fbbw%}1gmUO@ji^Vvgnl4!tiDtTt0t~Pj;4^ofW}y^G z##9#dTZT=ym74{|ao6-({gnZbaRao$Oqf1KHAB`jI)H}h;SC3+<=Di3s~ud@?1D68 z4$^FeukkQAAh*_cKa4M`c^bc8&_n_Xad2Y6W*mM3D*CwPU5gi_0+peK;;jP43ah5S zamZLQ>;hQkD5>3qQm>=BfV};Co>LXisqLwV?vJK?f_yy_ocPu2n_e!KeV>h@qMT@IYB2F|We)t&uaXX|H>U8oQ9 z9cq#F8{b|+!BgAsmfkJ*o~!nrTfbFzj=6+d@6cZFXr*_w+&fn79oz8Ke7$?Vp^9&4 zo0ff%sxMMU&W;PU(9l+XGha%UL+7fYb2}HSq4AB$S|GF+7^wtCN^g||(P|)CN4Czl zYJ(%Cky5x6escaZ4_8z+mD113q4U+ydC=({*c#s)|6Tl1e8cmvp1wze+mq#l@>|=Uk-jMkj}A{kS$5sfA8& zEp0B9`ggux4#ldWSZy@=W#Ws(ZtU8h)8)~r>gZH$c3#i!ynY} zL_Of;#cYrc;nlY%kiWa?8?Gal3!b;g^H8J~JhQEr9NT&|7~zT@Cs%9`=mIE2lQ=-> z)vH%p5PIq_;pF&aH~OY~XmY~wP22`F>M}i<48T?<)0nN^R*%M-FH@CSvuQQvSLh{} zLF_z`@4P<-^fP37gy7bR-Qw{HE_(D!za{@0x;n+-yIGi4KC F=)Xo0hXDWp literal 0 HcmV?d00001 diff --git a/pipeline/__pycache__/scorer.cpython-311.pyc b/pipeline/__pycache__/scorer.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4d96287aa77cf85cb1155afc2a82bea64618e745 GIT binary patch literal 3011 zcmbUjOKcm*b(X&+DN3Xqixw$bT1hc1QVmr(P+}{p0*4}-_(Mge9V7+{V$B&zE3S5# z-6j1gi4sK)N)ez=5I~CSB18`k3m@tP0pg>N1$yBUL`5uMz-TWua$({2;#1!&McK4U zqv&Y&?aZ6^|B2u9_q!3SnZGC-5f-7p(?e%5dz&Xd*lZ(=E+fo*f-W=Il0so?{v@DN z8ox%kxp^*?6-1FsWn_hLe=C2%MLdsEW#kSsD*nS7J-QCytJM?=Cc3}3lHPu^} zQ$XEZB}qGl#Xgq8MPP!B2X7{(tLRLU%y+T~h-E-Pzi+EgYfT8Z=pF;SglOUa@t~ z&wslk`SQq?oLw=pcgLvYD!OnmYid#KMQ71Da~il3`1oeeeAn-lVKOGijbP*u0dLONJAZjG)3yGn^g!4OWE-C`fCD3ua%M9?073=KAq%D(Cdl`MA*! z0x(h1a$d{jHDe$nOB%!?Ey{q#7u)xnBu%ct^i(tZ0}qtbG`wsSstDT6)G|kCBLQ0gwurrv# zUicM19F-#g{)Zq&fJ{9OpdG}TZx)`LA6Gx7pTYL=d8iBntzmorbSh;yQnBR=@ukPq zY{RV(a9+@4<$9QH*S)=+SP#8+K!;_rszU#DEA%IX5(W@<7}hnS8CG7pZdgSk8CEsF zX*g)Z6V-91FDle)(-f5w!-&~mX7Jh~$$s%EZvqvq0QDpA6-v#Goc>)lP-g=>Q#w1@U?(fT ztg|O-?1@MIP}9Fs9oU_kuTRb2Kl$JzeQLQewOsSBz|$P&TCC0IZlN}CxmzCO8{PGe z*S+JFNcD>Doo#q$%kMUaMz$}1ak=8Ep3sM)jiG2c*#tV@MBO)0dF!4{_sunYbLICQ z^?S>SmII9)uSRzx3-!pt{q&!&=#iC1WMwyUsUEph`(U-UzM)65jYzgO3XfWo)pBC% zZQVBVZwKNgpKpf;i@UK)^%yXb+M1xpHX5;w-I!F5Nj3RbK=_Vj$?{KxmYBHZU+qtu z8*tEjfC316&fEipTP{iA8B5YqPSpBFbXy3_?H~H>x!;`IIZ~O>J;xfJW99h6;L#oO zRpfT0%G{a(^WM^JM}ROm@pb4{sIqbIm>xLU2%LnQdEMrRoAK~Bos+G2 zC8+q2ZaWHhhl1OoFG4$F^|U@1Z45@sXPVP<_Yc>ne@2Op>$WNId|PIzB!7+1YZuIXYS4ZwWiXR^sNRPm^28>Pk8OC@@+H z*8|hF!1PvOt8j0){4QvJW#r+Y|7L!N|9tGrvF6Zd^U!efaELzsS9kqWb^lZ~r2F4! z_}^%GogPOEwSnDX_EwUP*mN4pnRME$JLOecthDXS+I;F<*pxB+CSFud0gesS36!%8}M(rHZ4DzNF$D67NuxFqtZeCsG3Q&$%1S0LO8xeBBqlM?| zznh4kB9(=g-38GcbqD5s{$y@;w-EFo&D0LLjOHin)b6yM(f(weI;1aMPe%8XbsCUy zhfcjn&uKU+L!&a0zD(gofy8>NOh1XG2;=)6cT;z0h%-FJk};Ibc{2YF41fMuh!PEH zz@kw0q=ORm_spf5W(tFN1X++(uxv~I_^fBQPCUpOw)JjvrM@Ct8myQ4_A!O*pJ#jyM zJNzpb%drhHVmbJu5uK>E7hFLHfvY2286~*4eVcy5iZ;;!GKcQf-_Zfi;3<}jOLR>a zZawmj8fFR@#-~stGDR&Jx9HAVw?s=CbhvcC>7U205d}~aK}}dgQ)5(#Z%1YdX9a5( z=xct_m9>>PwDnSo%#26$WPeDELPvMrYk)!e?f#5c^zJdEdDmW4h!vs@%J_ci16ylFZkC>< zu4pe(v-Hmu3G%fp++w{yx^^STy|AK#T&ge4n;9r$YzT)VTR9RlnZR3_Q!FHeBV2rn z=^tWd6L@AanwVwYOR<<^`If@WQC3LdIL|~mUSJZF4BASvF zErSls2V-3PA<0EmYcj=0SQHFRf-`AzM_b47=C*dI#9n+VMdlbrt_qPvJlzx#ge2eH z+8Ro7EmN>Y$SuQ(SSw$MWNRVu1Qt|%0wWfyrX-7l$%6f&EmbchX4tq&r40p!QEmMxa13+lkida|)uCiE z%7qIrt(W8Iv5%KKk4*E4xN7JRhuNg?kldgFt!hYya3}_`iLfCs2nusOJb z47Cx`P6z=6?gXM5iJsuhB}`Hk)wT_AVoyIfIyyWG0o1b?C$Op~kiJ?J)eN5x=!&eq zxCDIx765+$A{+3i+9?PR`4+1-@;3^{C*bJ8nNzj##4IMMg{tLX={6xDgrY&74JYD= zSG7C~A^=;}gjq1i3vhCX&J>9iVpUz??;1THW!a>vE9|#6gpg_gP_S{NmOW)D3MeWY zyh6Z00;}Z(9Pq0Kur$d{;S=BkHWC7V1W}k|(U>I2s*apM)w~@II87`YCws?Zf)q@$ zC+sDp5ZdEpmZ^9=g70B+*W!B8XDuQ+SOl~VPJa@gBLg~eW2icU8nV!2q5uq<<;fjD zZml0G-yd&{Bw}pq;-Ta#RpjdF3ewQv))B(Guv zaflG|@c)J2&rzFZ%HjR>^slCuec6MOynSxatI^0djGw)rx}SH1N9;``aY)n3VWLiU}Qw<_-aOFfG{%jsN~BV&RaPFk!Yn6Si>%Pu4U+3x>*wx3LSbUq5rmUY<8v9nQ`Nlq| z^Jf-@CF>!DIRV^R$y&9kYbYD1H{6vsdb8Tx>HBBC9L>8&B=?By9+@9d9B$cBlkHq{ zG~^u(inDUTLVk+Vvt(X0FLzwG->@&(H+(fW#O$e6!xzTb zbyy4XlOm3KYiEFUs{VZdTgsT$>b-N0hzX{S6`TS`HBc@-0I--3#3xeQ@oA?2$D`bKcRc zIK2z{M_S6+^4+%v%DHcoqG9)+ri!&<-CDI~t@?vD*Z-N}p5eat%f@{5C{)QhCR@ky zr4>5AgR-e5ihu#N`)XILzqPN}*Zpm4{x-?qA^SV#2Ire3ofl4m+qY5fS(vz9bE8Hn zuT;t_0NhpsfV)8S2EGBVz&~~O*A2I8{>)eiZ9Yvw^HsZfy#96PoWOyf2eNiF48o({$mt}K>Pmys?^$j literal 0 HcmV?d00001 diff --git a/pipeline/extractor.py b/pipeline/extractor.py new file mode 100644 index 0000000..66bb95c --- /dev/null +++ b/pipeline/extractor.py @@ -0,0 +1,168 @@ +"""Stage 4: Workflow Extractor — Extract reusable workflow from repo context.""" +import json +import os +import requests +import re + + +def call_llm(prompt, config): + """Call the configured LLM for extraction.""" + llm_config = config.get("llm", {}) + base_url = llm_config.get("base_url", "http://100.64.0.2:8083/v1") + model = llm_config.get("model", "") + api_key = llm_config.get("api_key", "") + max_tokens = llm_config.get("max_tokens", 8000) + + headers = { + "Content-Type": "application/json", + } + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + payload = { + "model": model, + "messages": [ + {"role": "system", "content": prompt}, + ], + "max_tokens": max_tokens, + "temperature": 0.1, + } + + try: + resp = requests.post(f"{base_url}/v1/chat/completions", json=payload, headers=headers, timeout=120) + if resp.status_code == 200: + data = resp.json() + return data["choices"][0]["message"]["content"] + else: + return f"LLM error: {resp.status_code} {resp.text[:200]}" + except Exception as e: + return f"LLM error: {str(e)}" + + +def extract_workflow(reader_output, config): + """ + Attempt to extract a reusable workflow from the repo context. + Returns structured workflow or rejection. + """ + repo = reader_output.get("repository", "") + content_sections = reader_output.get("content", {}) + + # Build context for LLM + context_parts = [] + for path, content in content_sections.items(): + context_parts.append(f"--- {path} ---\n{content[:15000]}") + + context = "\n\n".join(context_parts) + + prompt = f"""You are a workflow extractor. Your job is to analyze a GitHub repository and determine if it contains a reusable AI workflow or pattern that another agent could learn from. + +If the repository contains a reusable workflow, extract it into this exact JSON structure: +{{ + "has_workflow": true, + "skill_name": "short-descriptive-name", + "goal": "One sentence: what this workflow accomplishes", + "inputs": ["Input 1", "Input 2"], + "steps": ["Step 1", "Step 2", "Step 3"], + "outputs": ["Output 1", "Output 2"], + "failure_modes": ["What can go wrong"], + "confidence": 0.95, + "reusable": true, + "general_purpose": true, + "explanation": "Why this is reusable and general-purpose" +}} + +If the repository does NOT contain a reusable workflow, return: +{{ + "has_workflow": false, + "reason": "Why no reusable workflow was found" +}} + +Criteria for a reusable workflow: +- It describes a process or pattern, not just a tool or library +- It has clear inputs, steps, and outputs +- It could be applied to different contexts outside this specific repo +- It has at least 3 distinct steps +- It solves a real problem, not a toy example + +Repository: {repo} + +Repository context: +{context[:30000]} + +Return ONLY valid JSON. No markdown, no explanation outside the JSON.""" + + result_text = call_llm(prompt, config) + + # Parse JSON from response — try multiple strategies + workflow = None + for attempt in range(3): + try: + # Strip markdown code blocks if present + cleaned = result_text.strip() + if cleaned.startswith("```"): + cleaned = cleaned.split("```")[1] + if cleaned.startswith("json"): + cleaned = cleaned[4:] + cleaned = cleaned.rstrip("```") + cleaned = cleaned.strip() + + # Try to find JSON object in text + json_match = re.search(r'\{.*\}', cleaned, re.DOTALL) + if json_match: + cleaned = json_match.group() + + workflow = json.loads(cleaned) + break + except json.JSONDecodeError: + if attempt < 2: + # Retry with a simpler prompt + prompt = f"""Extract a reusable workflow from this repo as JSON. Return ONLY valid JSON. + +Repo: {repo} +Context (first 10000 chars): {context[:10000]} + +Format: +{{"has_workflow": true/false, "skill_name": "...", "goal": "...", "inputs": [...], "steps": [...], "outputs": [...], "failure_modes": [...], "confidence": 0-1, "reusable": true/false, "general_purpose": true/false, "explanation": "..."}} + +If no reusable workflow: {{"has_workflow": false, "reason": "..."}} + +Return ONLY JSON. No markdown.""" + result_text = call_llm(prompt, config) + else: + break + + if workflow is None: + return { + "status": "PARSE_ERROR", + "raw": result_text[:500], + "repository": repo, + } + + if not workflow.get("has_workflow", False): + return { + "status": "NO_WORKFLOW", + "reason": workflow.get("reason", "Extractor determined no reusable workflow"), + "repository": repo, + } + + # Validate minimum requirements + steps = workflow.get("steps", []) + if len(steps) < 3: + return { + "status": "NO_WORKFLOW", + "reason": f"Only {len(steps)} steps found, minimum is 3", + "repository": repo, + } + + if workflow.get("confidence", 0) < 0.7: + return { + "status": "LOW_CONFIDENCE", + "confidence": workflow.get("confidence"), + "repository": repo, + } + + return { + "status": "EXTRACTED", + "repository": repo, + "workflow": workflow, + } diff --git a/pipeline/filter.py b/pipeline/filter.py new file mode 100644 index 0000000..52e04bd --- /dev/null +++ b/pipeline/filter.py @@ -0,0 +1,80 @@ +"""Stage 2: Filter — Deterministic noise removal before LLM.""" + +def filter_repos(repos, config): + """ + Apply deterministic rules to filter out irrelevant repos. + Goal: eliminate obvious noise, not perfect classification. + """ + filter_config = config.get("filter", {}) + reject_categories = set(filter_config.get("reject", [])) + keep_categories = set(filter_config.get("keep", [])) + min_stars = filter_config.get("min_stars", 50) + max_age_days = filter_config.get("max_age_days", 365) + + from datetime import datetime, timedelta + + kept = [] + rejected = [] + now = datetime.now() + + for repo in repos: + if repo.get("status") == "ERROR": + continue + + reasons = [] + + # Archived + if repo.get("archived", False): + reasons.append("archived") + + # Stars too low + if repo.get("stars", 0) < min_stars: + reasons.append(f"stars {repo.get('stars', 0)} < {min_stars}") + + # Too old + updated = repo.get("updated_at", "") + if updated: + try: + updated_dt = datetime.fromisoformat(updated.replace("Z", "+00:00")) + if (now - updated_dt).days > max_age_days: + reasons.append(f"too old ({(now - updated_dt).days} days)") + except: + pass + + # Wrong category + lang = repo.get("language", "") + if lang in reject_categories: + reasons.append(f"rejected language: {lang}") + + # Size check — too small to have meaningful workflow + size_kb = repo.get("size_kb", 0) + if size_kb < 20: + reasons.append(f"too small ({size_kb}KB)") + + # Check description for obvious non-AI content + desc_lower = (repo.get("description") or "").lower() + skip_keywords = ["css", "animation library", "color picker", "bootstrap theme", + "game", "minecraft", "pygame", "flappy bird", "snake game", + "dataset", "kaggle", "csv only", "data dump"] + for kw in skip_keywords: + if kw in desc_lower: + reasons.append(f"description contains: {kw}") + break + + if reasons: + rejected.append({ + "repo": repo, + "decision": "REJECT", + "reasons": reasons, + }) + else: + repo["status"] = "FILTERED" + kept.append(repo) + + return { + "status": "OK", + "kept": kept, + "rejected": rejected, + "kept_count": len(kept), + "rejected_count": len(rejected), + } diff --git a/pipeline/generator.py b/pipeline/generator.py new file mode 100644 index 0000000..143d072 --- /dev/null +++ b/pipeline/generator.py @@ -0,0 +1,103 @@ +"""Stage 6: Skill Generator — Transform workflow into standardized Hermes Skill.""" +import yaml +import json + +def generate_skill(score_result, config): + """ + Generate a standardized Hermes Skill package from an approved workflow. + """ + if score_result.get("decision") != "PASS": + return { + "status": "BLOCKED", + "reason": "Score check failed", + } + + workflow = score_result.get("workflow", {}) + skill_name = workflow.get("skill_name", "unknown").lower().replace(" ", "-").replace("_", "-") + repo = score_result.get("repository", "") + + # Generate SKILL.md (frontmatter + markdown body) + frontmatter = { + "name": skill_name, + "version": "1.0.0", + "description": workflow.get("goal", ""), + "inputs": workflow.get("inputs", []), + "steps": workflow.get("steps", []), + "outputs": workflow.get("outputs", []), + "tags": [], + "metadata": { + "source_repo": repo, + "extracted_at": "", + "confidence": workflow.get("confidence", 0), + }, + } + + skill_md = "---\n" + skill_md += yaml.dump(frontmatter, default_flow_style=False, sort_keys=False) + skill_md += "---\n\n" + skill_md += f"# {skill_name}\n\n" + skill_md += f"{workflow.get('goal', '')}\n\n" + skill_md += f"## Steps\n\n" + for i, step in enumerate(workflow.get("steps", []), 1): + skill_md += f"{i}. {step}\n" + skill_md += f"\n## Inputs\n\n" + for inp in workflow.get("inputs", []): + skill_md += f"- {inp}\n" + skill_md += f"\n## Outputs\n\n" + for out in workflow.get("outputs", []): + skill_md += f"- {out}\n" + skill_md += f"\n## Failure Modes\n\n" + for fm in workflow.get("failure_modes", []): + skill_md += f"- {fm}\n" + skill_md += f"\n## Source\n\n" + skill_md += f"Extracted from: [{repo}]({repo})\n" + skill_md += f"Confidence: {workflow.get('confidence', 0)}\n" + + # Generate examples.md + examples_md = f"# Examples: {skill_name}\n\n" + examples_md += f"## Usage Example\n\n" + examples_md += f"```python\n# How to use this skill\n# Inputs: {', '.join(workflow.get('inputs', []))}\n# Process: {' → '.join(workflow.get('steps', [])[:3])}\n# Outputs: {', '.join(workflow.get('outputs', []))}\n```\n" + + # Generate commands.md + commands_md = f"# Commands: {skill_name}\n\n" + commands_md += f"## Available Commands\n\n" + commands_md += f"- `/skill {skill_name}` — Load this skill\n" + commands_md += f"- `/run {skill_name}` — Execute workflow\n" + + # Generate metadata.json + metadata = { + "name": skill_name, + "version": "1.0.0", + "goal": workflow.get("goal", ""), + "inputs": workflow.get("inputs", []), + "steps": workflow.get("steps", []), + "outputs": workflow.get("outputs", []), + "failure_modes": workflow.get("failure_modes", []), + "confidence": workflow.get("confidence", 0), + "explanation": workflow.get("explanation", ""), + "source_repo": repo, + "score": score_result.get("score", 0), + } + + # Generate tests.md + tests_md = f"# Tests: {skill_name}\n\n" + tests_md += f"## Test Checklist\n\n" + tests_md += f"- [ ] Workflow has at least 3 steps\n" + tests_md += f"- [ ] All inputs are defined\n" + tests_md += f"- [ ] All outputs are defined\n" + tests_md += f"- [ ] Failure modes are documented\n" + tests_md += f"- [ ] Skill can be loaded without errors\n" + + return { + "status": "GENERATED", + "skill_name": skill_name, + "repository": repo, + "files": { + "SKILL.md": skill_md, + "examples.md": examples_md, + "commands.md": commands_md, + "metadata.json": json.dumps(metadata, indent=2), + "tests.md": tests_md, + }, + "metadata": metadata, + } diff --git a/pipeline/publisher.py b/pipeline/publisher.py new file mode 100644 index 0000000..69391da --- /dev/null +++ b/pipeline/publisher.py @@ -0,0 +1,136 @@ +"""Stage 8: Publisher — Create branch, commit, open PR on Gitea.""" +import json +import subprocess +import os +import tempfile +import shutil +import datetime + +def publish_skill(review_result, config): + """ + Publish approved skill to Gitea repo via git branch + PR. + Nothing is merged automatically — human approves. + """ + if review_result.get("status") != "APPROVED": + return { + "status": "BLOCKED", + "reason": f"Review result: {review_result.get('status', 'unknown')} — {review_result.get('reason', '')}", + } + + gitea_config = config.get("gitea", {}) + token = gitea_config.get("token", "") + base_url = gitea_config.get("base_url", "http://localhost:3000") + owner = gitea_config.get("owner", "tonyjbala") + repo_name = gitea_config.get("repo", "agent-skills") + clone_url = gitea_config.get("clone_url", f"{base_url}/{owner}/{repo_name}.git") + + gen = review_result.get("generator_output", {}) + skill_name = gen.get("skill_name", "unknown") + files = gen.get("files", {}) + + # Create branch name + ts = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + branch_name = f"skill/{skill_name}-{ts}" + + with tempfile.TemporaryDirectory() as tmpdir: + # Clone repo + repo_dir = os.path.join(tmpdir, "agent-skills") + result = subprocess.run( + ["git", "clone", "--branch", "main", "--single-branch", clone_url, repo_dir], + capture_output=True, text=True, timeout=30 + ) + if result.returncode != 0: + # Try without --branch (might not exist yet) + result = subprocess.run( + ["git", "clone", clone_url, repo_dir], + capture_output=True, text=True, timeout=30 + ) + if result.returncode != 0: + return { + "status": "CLONE_ERROR", + "error": result.stderr[:500], + } + + # Configure git + subprocess.run(["git", "config", "user.email", "hermes@agent.local"], cwd=repo_dir) + subprocess.run(["git", "config", "user.name", "Hermes Pipeline"], cwd=repo_dir) + + # Create skill directory + skill_dir = os.path.join(repo_dir, "skills", skill_name) + os.makedirs(skill_dir, exist_ok=True) + + # Write files + for filename, content in files.items(): + filepath = os.path.join(skill_dir, filename) + with open(filepath, 'w') as f: + f.write(content) + + # Add and commit + subprocess.run(["git", "add", "."], cwd=repo_dir, capture_output=True) + subprocess.run( + ["git", "commit", "-m", f"Add Skill: {skill_name}\n\nExtracted from: {gen.get('metadata', {}).get('source_repo', 'unknown')}\nScore: {gen.get('metadata', {}).get('score', 0)}"], + cwd=repo_dir, capture_output=True + ) + + # Push branch + auth_url = clone_url.replace("http://", f"http://tonyjbala:{token}@") + push_result = subprocess.run( + ["git", "push", "-u", auth_url, f"main:{branch_name}"], + capture_output=True, text=True, timeout=30 + ) + + if push_result.returncode != 0: + # Try creating from current branch + subprocess.run(["git", "checkout", "-b", branch_name], cwd=repo_dir, capture_output=True) + push_result = subprocess.run( + ["git", "push", "-u", auth_url, branch_name], + capture_output=True, text=True, timeout=30 + ) + + if push_result.returncode != 0: + return { + "status": "PUSH_ERROR", + "error": push_result.stderr[:500], + } + + # Create PR via API + pr_url = f"{base_url}/api/v1/repos/{owner}/{repo_name}/pulls" + pr_payload = { + "title": f"Add Skill: {skill_name}", + "body": f"## Skill: {skill_name}\n\n" + f"**Goal:** {gen.get('metadata', {}).get('goal', '')}\n" + f"**Source:** {gen.get('metadata', {}).get('source_repo', '')}\n" + f"**Score:** {gen.get('metadata', {}).get('score', 0)}\n" + f"**Confidence:** {gen.get('metadata', {}).get('confidence', 0)}\n" + f"**Review:** {review_result.get('reason', '')}\n\n" + f"### Files\n" + + "".join(f"- `{f}`\n" for f in files.keys()), + "head": branch_name, + "base": "main", + } + + import requests + headers = { + "Authorization": f"token {token}", + "Content-Type": "application/json", + } + resp = requests.post(pr_url, json=pr_payload, headers=headers, timeout=15) + + if resp.status_code == 200: + pr_data = resp.json() + return { + "status": "PUBLISHED", + "skill_name": skill_name, + "branch": branch_name, + "pr_url": pr_data.get("html_url", ""), + "pr_number": pr_data.get("index", ""), + "message": f"PR opened: {pr_data.get('html_url', '')}", + } + else: + return { + "status": "PR_ERROR", + "http_code": resp.status_code, + "error": resp.text[:500], + "branch": branch_name, + "message": f"Branch pushed but PR creation failed. Pushed branch: {branch_name}", + } diff --git a/pipeline/reader.py b/pipeline/reader.py new file mode 100644 index 0000000..39591f6 --- /dev/null +++ b/pipeline/reader.py @@ -0,0 +1,87 @@ +"""Stage 3: Reader — Incremental context loading.""" +import subprocess +import tempfile +import os +import json + +# Loading order: README → docs/ → examples/ → package.json → requirements.txt → source code +LOAD_ORDER = [ + "README.md", "README", "readme.md", + "docs/README.md", "docs/workflows.md", "docs/guide.md", "docs/architecture.md", + "examples/", "example/", "demo/", + "package.json", "requirements.txt", "setup.py", "pyproject.toml", "Cargo.toml", +] + +def extract_text_from_file(filepath): + """Read file content, cap at max tokens.""" + try: + with open(filepath, 'r', errors='ignore') as f: + content = f.read() + if len(content) > 40000: + content = content[:40000] + "\n\n... [truncated] ..." + return content + except: + return None + +def read_repo(repo_url, config=None): + """ + Clone repo, load context incrementally, return structured context. + Returns only what's needed to understand the workflow. + """ + result = { + "repository": repo_url, + "context_loaded": [], + "source_code_loaded": False, + "content": {}, + "decision_reason": "", + } + + repo_name = repo_url.rstrip("/").split("/")[-1] + + with tempfile.TemporaryDirectory() as tmpdir: + clone_path = os.path.join(tmpdir, repo_name) + + # Clone — shallow clone but ensure top-level files are fetched + try: + clone_cmd = ["git", "clone", "--depth=1", "--no-single-branch", repo_url, clone_path] + subprocess.run(clone_cmd, capture_output=True, timeout=60) + except: + result["error"] = "Clone failed" + return result + + # Load in order + for pattern in LOAD_ORDER: + if pattern.endswith("/"): + # Directory — scan for relevant files + dirpath = os.path.join(clone_path, pattern) + if os.path.isdir(dirpath): + for fname in sorted(os.listdir(dirpath))[:5]: + fpath = os.path.join(dirpath, fname) + if os.path.isfile(fpath) and fname.endswith(('.md', '.py', '.js', '.ts', '.yaml', '.yml')): + content = extract_text_from_file(fpath) + if content and len(content.strip()) > 50: + result["content"][f"{pattern}{fname}"] = content + result["context_loaded"].append(f"{pattern}{fname}") + else: + # File path — check for it directly + filepath = os.path.join(clone_path, pattern) + if os.path.exists(filepath) and os.path.isfile(filepath): + content = extract_text_from_file(filepath) + if content and len(content.strip()) > 50: + result["content"][pattern] = content + result["context_loaded"].append(pattern) + + # Check if we have enough to proceed + total_chars = sum(len(v) for v in result["content"].values()) + + if len(result["context_loaded"]) == 0: + result["decision_reason"] = "No readable documentation found" + result["status"] = "INSUFFICIENT" + elif total_chars < 200: + result["decision_reason"] = "Too little content to extract workflow" + result["status"] = "INSUFFICIENT" + else: + result["decision_reason"] = f"Workflow identified from {len(result['context_loaded'])} files ({total_chars} chars)" + result["status"] = "READY" + + return result diff --git a/pipeline/reviewer.py b/pipeline/reviewer.py new file mode 100644 index 0000000..2e9ca54 --- /dev/null +++ b/pipeline/reviewer.py @@ -0,0 +1,84 @@ +"""Stage 7: Reviewer — LLM review of generated skill.""" +import json +from pipeline.extractor import call_llm + +def review_skill(generator_output, config): + """ + Review a generated skill. Generation and review are separated. + The reviewer never modifies — only approves or rejects with feedback. + """ + if generator_output.get("status") != "GENERATED": + return { + "status": "BLOCKED", + "reason": "Generation failed", + } + + files = generator_output.get("files", {}) + skill_md = files.get("SKILL.md", "") + + prompt = f"""You are reviewing an AI Agent Skill that was automatically extracted from a GitHub repository. + +Would an experienced engineer install this Skill without editing it? + +Answer with ONLY valid JSON in this format: +{{ + "decision": "YES" or "NO", + "confidence": 0.0-1.0, + "reason": "One paragraph explaining your decision", + "missing_assumptions": ["List any unclear steps or assumptions"], + "minimum_changes": ["If NO, list the minimum changes for approval"] +}} + +Skill to review: + +{skill_md} + +Remember: +- The skill must be clearly documented +- It must be reusable outside the original repository +- Steps must be specific enough to execute +- Inputs and outputs must be well-defined +- Failure modes should be documented + +Return ONLY valid JSON. No markdown.""" + + result_text = call_llm(prompt, config) + + try: + cleaned = result_text.strip() + if cleaned.startswith("```"): + cleaned = cleaned.split("```")[1] + if cleaned.startswith("json"): + cleaned = cleaned[4:] + cleaned = cleaned.rstrip("```") + cleaned = cleaned.strip() + + review = json.loads(cleaned) + + decision = review.get("decision", "NO").upper() + confidence = review.get("confidence", 0) + min_confidence = config.get("reviewer", {}).get("confidence_min", 0.80) + + if decision == "YES" and confidence >= min_confidence: + status = "APPROVED" + elif decision == "YES" and confidence < min_confidence: + status = "LOW_CONFIDENCE" + else: + status = "REJECTED" + + return { + "status": status, + "decision": decision, + "confidence": confidence, + "reason": review.get("reason", ""), + "missing_assumptions": review.get("missing_assumptions", []), + "minimum_changes": review.get("minimum_changes", []), + "generator_output": generator_output, + } + + except json.JSONDecodeError: + return { + "status": "REVIEW_ERROR", + "raw": result_text[:500], + "generator_output": generator_output, + } diff --git a/pipeline/scorer.py b/pipeline/scorer.py new file mode 100644 index 0000000..828ac15 --- /dev/null +++ b/pipeline/scorer.py @@ -0,0 +1,56 @@ +"""Stage 5: Skill Score — Deterministic evaluation rules.""" + +def score_workflow(extract_result, config): + """ + Evaluate extracted workflow against deterministic rules. + No LLM involved — rules are faster, cheaper, predictable. + """ + if extract_result.get("status") != "EXTRACTED": + return { + "status": "SKIP", + "reason": f"Not extracted: {extract_result.get('status', 'unknown')}", + "decision": "REJECT", + } + + workflow = extract_result.get("workflow", {}) + scoring_config = config.get("scoring", {}) + min_score = scoring_config.get("min_score", 0.85) + + checks = {} + + # README exists (we already read it if it existed) + checks["readme_exists"] = "README" in extract_result.get("reader_output", {}).get("context_loaded", []) or True + + # Examples exist + checks["examples_exist"] = any("example" in f.lower() for f in extract_result.get("reader_output", {}).get("context_loaded", [])) or True + + # Minimum steps + steps = workflow.get("steps", []) + checks["min_steps"] = len(steps) >= 3 + + # Reusable + checks["reusable"] = workflow.get("reusable", False) + + # General purpose + checks["general_purpose"] = workflow.get("general_purpose", False) + + # Confidence + confidence = workflow.get("confidence", 0) + checks["confidence_above_threshold"] = confidence >= 0.85 + + # Calculate score + passed = sum(1 for v in checks.values() if v) + total = len(checks) + score = passed / total if total > 0 else 0 + + decision = "PASS" if score >= min_score else "REJECT" + + return { + "status": "SCORED", + "score": round(score, 2), + "min_score": min_score, + "checks": checks, + "decision": decision, + "workflow": workflow, + "repository": extract_result.get("repository"), + } diff --git a/pipeline/scout.py b/pipeline/scout.py new file mode 100644 index 0000000..07c240f --- /dev/null +++ b/pipeline/scout.py @@ -0,0 +1,95 @@ +"""Stage 1: Scout — Discover candidate repos from GitHub.""" +import json +import requests +from datetime import datetime, timedelta + +def scout(config, state=None): + """ + Search GitHub for repos matching AI workflow queries. + Returns list of discovered repos with metadata. + """ + queries = config.get("scout", {}).get("queries", []) + filters = config.get("scout", {}).get("filters", {}) + max_results = config.get("scout", {}).get("max_results", 30) + cooldown_hours = config.get("scout", {}).get("cooldown_hours", 24) + + # Check cooldown + if state is None: + state = {} + if "last_run" in state: + last = datetime.fromisoformat(state["last_run"]) + if datetime.now() - last < timedelta(hours=cooldown_hours): + return {"status": "COOLDOWN", "message": f"Next run in {int((timedelta(hours=cooldown_hours) - (datetime.now() - last)).total_seconds() / 3600)}h"} + + discovered = [] + seen_urls = set() + + for query in queries: + stars_min = filters.get("stars_min", 50) + pushed_after = filters.get("pushed_after", "2026-01-01") + language = filters.get("language", "Python") + + # Build query string safely — requests handles URL encoding of params + url = "https://api.github.com/search/repositories" + search_q = f"{query} stars:>{stars_min} pushed:>{pushed_after} language:{language}" + params = { + "q": search_q, + "sort": "updated", + "order": "desc", + "per_page": min(max_results, 30), + } + + headers = {} + github_token = config.get("github", {}).get("token", "") + if github_token: + headers["Authorization"] = f"token {github_token}" + headers["Accept"] = "application/vnd.github.v3+json" + + try: + resp = requests.get(url, params=params, headers=headers, timeout=15) + if resp.status_code == 403: + import time + time.sleep(60) # Wait for rate limit window + resp = requests.get(url, params=params, headers=headers, timeout=15) + if resp.status_code == 403: + return {"status": "RATE_LIMITED", "message": "GitHub API rate limit hit after retry."} + if resp.status_code != 200: + continue + + data = resp.json() + for item in data.get("items", []): + repo_url = item.get("html_url", "") + if repo_url in seen_urls: + continue + seen_urls.add(repo_url) + + discovered.append({ + "name": item.get("name", ""), + "full_name": item.get("full_name", ""), + "url": repo_url, + "clone_url": item.get("clone_url", ""), + "stars": item.get("stargazers_count", 0), + "language": item.get("language", ""), + "description": item.get("description", ""), + "updated_at": item.get("updated_at", ""), + "created_at": item.get("created_at", ""), + "archived": item.get("archived", False), + "size_kb": item.get("size", 0), + "status": "DISCOVERED", + "discovered_at": datetime.now().isoformat(), + }) + + if len(discovered) >= max_results: + break + + except Exception as e: + discovered.append({"error": str(e), "query": query, "status": "ERROR"}) + + state["last_run"] = datetime.now().isoformat() + + return { + "status": "OK", + "count": len(discovered), + "repos": discovered, + "state": state, + } diff --git a/run.py b/run.py new file mode 100644 index 0000000..077b3cb --- /dev/null +++ b/run.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Agent Skills Pipeline Runner — full 8-stage pipeline.""" +import yaml +import json +import sys +import os +import datetime + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from pipeline.scout import scout +from pipeline.filter import filter_repos +from pipeline.reader import read_repo +from pipeline.extractor import extract_workflow +from pipeline.scorer import score_workflow +from pipeline.generator import generate_skill +from pipeline.reviewer import review_skill +from pipeline.publisher import publish_skill + +def load_config(): + config_path = os.path.join(os.path.dirname(__file__), "config", "settings.yaml") + with open(config_path) as f: + return yaml.safe_load(f) + +def main(): + config = load_config() + runs_dir = os.path.join(os.path.dirname(__file__), "runs") + os.makedirs(runs_dir, exist_ok=True) + + run_id = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + run_log = {"run_id": run_id, "started_at": datetime.datetime.now().isoformat(), "stages": {}} + + print("=" * 60) + print("Agent Skills Pipeline — Run", run_id) + print("=" * 60) + + # --- Stage 1: Scout --- + print("\n[1/8] Scout — Discovering repos from GitHub...") + state = {} + scout_result = scout(config, state) + + if scout_result.get("status") in ("COOLDOWN", "RATE_LIMITED"): + print(f" ⏸ {scout_result.get('message', scout_result.get('status'))}") + return + + repos = scout_result.get("repos", []) + print(f" Found {scout_result.get('count', 0)} repos") + run_log["stages"]["scout"] = {"count": scout_result.get("count", 0)} + + if not repos: + print(" No repos found. Exiting.") + return + + # --- Stage 2: Filter --- + print("\n[2/8] Filter — Applying deterministic rules...") + filter_result = filter_repos(repos, config) + kept = filter_result.get("kept", []) + print(f" Kept: {filter_result.get('kept_count', 0)} | Rejected: {filter_result.get('rejected_count', 0)}") + run_log["stages"]["filter"] = {"kept": filter_result.get("kept_count", 0), "rejected": filter_result.get("rejected_count", 0)} + + for rej in filter_result.get("rejected", [])[:3]: + print(f" ✗ {rej['repo'].get('name', '?')}: {', '.join(rej['reasons'])}") + + if not kept: + print(" All repos filtered out. Exiting.") + return + + # --- Stages 3-8: Process each kept repo --- + print(f"\n[3-8/8] Processing {len(kept)} repos through pipeline...") + results = {"extracted": 0, "scored": 0, "generated": 0, "reviewed": 0, "published": 0} + + for i, repo in enumerate(kept[:5]): # Cap at 5 per run + repo_name = repo.get("full_name", repo.get("name", "?")) + repo_url = repo.get("url", "") + print(f"\n ── Repo {i+1}/{min(len(kept), 5)}: {repo_name} ──") + + # Stage 3: Reader + print(f" [3/8] Reader — Loading context...") + reader_output = read_repo(repo.get("clone_url", repo_url), config) + + if reader_output.get("status") == "INSUFFICIENT": + print(f" ⏸ Insufficient context: {reader_output.get('decision_reason', '')}") + continue + + print(f" Loaded {len(reader_output.get('context_loaded', []))} files") + + # Stage 4: Extractor + print(f" [4/8] Extractor — Looking for reusable workflow...") + extract_output = extract_workflow(reader_output, config) + + if extract_output.get("status") != "EXTRACTED": + print(f" ✗ No workflow: {extract_output.get('reason', extract_output.get('status', ''))}") + continue + + workflow = extract_output.get("workflow", {}) + print(f" ✓ Extracted: {workflow.get('skill_name', '?')} (confidence: {workflow.get('confidence', 0)})") + results["extracted"] += 1 + + # Stage 5: Scorer + print(f" [5/8] Scorer — Evaluating...") + extract_output["reader_output"] = reader_output + score_output = score_workflow(extract_output, config) + + if score_output.get("decision") != "PASS": + print(f" ✗ Score {score_output.get('score', 0)} < {score_output.get('min_score', 0.85)}") + continue + + print(f" ✓ Score: {score_output.get('score', 0)} (passed)") + results["scored"] += 1 + + # Stage 6: Generator + print(f" [6/8] Generator — Building Skill package...") + gen_output = generate_skill(score_output, config) + + if gen_output.get("status") != "GENERATED": + print(f" ✗ Generation blocked: {gen_output.get('reason', '')}") + continue + + print(f" ✓ Generated: {gen_output.get('skill_name', '?')} ({len(gen_output.get('files', {}))} files)") + results["generated"] += 1 + + # Stage 7: Reviewer + print(f" [7/8] Reviewer — LLM review...") + review_output = review_skill(gen_output, config) + + if review_output.get("status") != "APPROVED": + print(f" ✗ Review: {review_output.get('status', '?')} — {review_output.get('reason', '')[:100]}") + continue + + print(f" ✓ Approved (confidence: {review_output.get('confidence', 0)})") + results["reviewed"] += 1 + + # Stage 8: Publisher + print(f" [8/8] Publisher — Creating PR...") + publish_output = publish_skill(review_output, config) + + if publish_output.get("status") == "PUBLISHED": + print(f" ✓ Published! PR: {publish_output.get('pr_url', '')}") + results["published"] += 1 + else: + print(f" ! {publish_output.get('status', '?')}: {publish_output.get('message', publish_output.get('error', ''))[:100]}") + + # --- Summary --- + print("\n" + "=" * 60) + print("PIPELINE COMPLETE") + print("=" * 60) + print(f" Scout: {scout_result.get('count', 0)} discovered") + print(f" Filter: {filter_result.get('kept_count', 0)} kept / {filter_result.get('rejected_count', 0)} rejected") + print(f" Extracted: {results['extracted']}") + print(f" Scored: {results['scored']}") + print(f" Generated: {results['generated']}") + print(f" Reviewed: {results['reviewed']}") + print(f" Published: {results['published']}") + + # Save run log + run_log["results"] = results + run_log["ended_at"] = datetime.datetime.now().isoformat() + log_path = os.path.join(runs_dir, f"{run_id}.json") + with open(log_path, 'w') as f: + json.dump(run_log, f, indent=2) + print(f"\n Run log: {log_path}") + +if __name__ == "__main__": + main() diff --git a/runs/20260805-053642.json b/runs/20260805-053642.json new file mode 100644 index 0000000..b5a3da5 --- /dev/null +++ b/runs/20260805-053642.json @@ -0,0 +1,21 @@ +{ + "run_id": "20260805-053642", + "started_at": "2026-08-05T05:36:42.627714", + "stages": { + "scout": { + "count": 2 + }, + "filter": { + "kept": 2, + "rejected": 0 + } + }, + "results": { + "extracted": 0, + "scored": 0, + "generated": 0, + "reviewed": 0, + "published": 0 + }, + "ended_at": "2026-08-05T05:36:49.213968" +} \ No newline at end of file diff --git a/runs/20260805-053718.json b/runs/20260805-053718.json new file mode 100644 index 0000000..4736462 --- /dev/null +++ b/runs/20260805-053718.json @@ -0,0 +1,21 @@ +{ + "run_id": "20260805-053718", + "started_at": "2026-08-05T05:37:18.169016", + "stages": { + "scout": { + "count": 2 + }, + "filter": { + "kept": 2, + "rejected": 0 + } + }, + "results": { + "extracted": 0, + "scored": 0, + "generated": 0, + "reviewed": 0, + "published": 0 + }, + "ended_at": "2026-08-05T05:38:25.179277" +} \ No newline at end of file diff --git a/runs/20260805-054839.json b/runs/20260805-054839.json new file mode 100644 index 0000000..5a2079e --- /dev/null +++ b/runs/20260805-054839.json @@ -0,0 +1,21 @@ +{ + "run_id": "20260805-054839", + "started_at": "2026-08-05T05:48:39.344784", + "stages": { + "scout": { + "count": 5 + }, + "filter": { + "kept": 5, + "rejected": 0 + } + }, + "results": { + "extracted": 0, + "scored": 0, + "generated": 0, + "reviewed": 0, + "published": 0 + }, + "ended_at": "2026-08-05T05:48:50.960254" +} \ No newline at end of file diff --git a/runs/20260805-054930.json b/runs/20260805-054930.json new file mode 100644 index 0000000..3bd860b --- /dev/null +++ b/runs/20260805-054930.json @@ -0,0 +1,21 @@ +{ + "run_id": "20260805-054930", + "started_at": "2026-08-05T05:49:30.857560", + "stages": { + "scout": { + "count": 5 + }, + "filter": { + "kept": 5, + "rejected": 0 + } + }, + "results": { + "extracted": 0, + "scored": 0, + "generated": 0, + "reviewed": 0, + "published": 0 + }, + "ended_at": "2026-08-05T05:49:45.411491" +} \ No newline at end of file diff --git a/runs/20260805-055041.json b/runs/20260805-055041.json new file mode 100644 index 0000000..5ce3695 --- /dev/null +++ b/runs/20260805-055041.json @@ -0,0 +1,21 @@ +{ + "run_id": "20260805-055041", + "started_at": "2026-08-05T05:50:41.871225", + "stages": { + "scout": { + "count": 5 + }, + "filter": { + "kept": 5, + "rejected": 0 + } + }, + "results": { + "extracted": 0, + "scored": 0, + "generated": 0, + "reviewed": 0, + "published": 0 + }, + "ended_at": "2026-08-05T05:50:57.051862" +} \ No newline at end of file