Support Phi-3.5-MoE

2025-10-14 11:23:53 +05:30
parent 8b827ee386
commit 7ed0cb1ffb
1 changed files with 6 additions and 0 deletions
@@ -106,6 +106,12 @@ class Model:
                for expert in layer.mlp.experts:
                    try_add(expert.down_proj.weight)

+        # Phi-3.5-MoE (and possibly others).
+        if not matrices:
+            with suppress(Exception):
+                for expert in layer.block_sparse_moe.experts:
+                    try_add(expert.w2.weight)
+
        # We need at least one MLP down-projection.
        assert matrices