
    (HJj2                        d dl Z d dlmZ d dlmZmZmZmZmZ d dl	m
Z d dlmZ ddlmZ ddlmZmZmZ ddlmZmZ ddlmZ dd	lmZ e G d
 de             Z G d dej8                        Z G d dej8                        Z G d dej8                        Z G d dej8                        Z  G d dej8                        Z! G d dej8                        Z" G d dej8                        Z#y)    N)	dataclass)AnyDictListOptionalUnion   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)KVCacheRotatingKVCache)initialize_rope)	SwitchGLUc                      e Zd ZU eed<   ee   ed<   dZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeeeeeef   f      ed<   dZeed<   dZeed<   dZeed<   d Zeed!<   d"Zeed#<   d$Zeed%<   d&Zeed'<   d(Zeed)<   d Z eed*<   d Z!eed+<   dZ"eed,<   d$Z#eed-<   y).	ModelArgs
model_typelayer_typesi  
vocab_sizei   hidden_sizei   intermediate_sizei   moe_intermediate_size    num_hidden_layersnum_attention_heads   num_key_value_heads@   head_dimi   max_position_embeddingsgh㈵>rms_norm_epsi'  
rope_thetaNrope_scalingFtie_word_embeddings   num_experts   num_experts_per_tokr	   num_shared_experts   num_dense_layersT
route_normgS㥛@route_scalesigmoid
score_funcn_group
topk_groupsliding_windowmup_enabled)$__name__
__module____qualname__str__annotations__r   r   intr   r   r   r   r   r   r    r!   r"   floatr#   r$   r   r   r   r%   boolr'   r)   r*   r,   r-   r.   r0   r1   r2   r3   r4        ]/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/afmoe.pyr   r      s#   OcJK!s!!%3%s!!  Hc#)S)L%J;?L(4U5#:%6 678? %%K  cJKJGSJNCKr>   r   c            	            e Zd Zd	dedef fdZ	 	 d
dej                  deej                     dee	   dej                  fdZ
 xZS )	Attentionargsis_local_attentionc                    t         |           |j                  | _        |j                  | _        |j
                  | _        |j                  | _        || _        | j                  dz  | _	        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  | j                  z  d      | _        t        j                  | j                  | j                  z  | j                  d      | _        t        j                   | j                  |j"                        | _        t        j                   | j                  |j"                        | _        t        j                  | j                  | j                  | j                  z  d      | _        |r=t+        | j                  |j,                  d|j.                  |j0                        | _        y d | _        y )Ng      Fbiaseps)super__init__r   r   n_headsr   
n_kv_headsr    rC   scalennLinearq_projk_projv_projo_projRMSNormr"   q_normk_norm	gate_projr   r#   r$   r!   rope)selfrB   rC   	__class__s      r?   rJ   zAttention.__init__1   s   ++//22"4]]D(
iidllT]]:
 iidoo=E
 iidoo=E
 iiLL4==($*:*:
 jjD4E4EFjjD4E4EFdllT]]:
 '!!,,DI DIr>   xmaskcachereturnc                 8   |j                   \  }}}| j                  |      }| j                  |      }| j                  |      }	|j	                  ||| j
                  | j                        j                  dddd      }|j	                  ||| j                  | j                        j                  dddd      }|	j	                  ||| j                  | j                        j                  dddd      }	| j                  |      }| j                  |      }| j                  rk| j                  _|;| j                  ||j                        }| j                  ||j                        }n"| j                  |      }| j                  |      }||j                  ||	      \  }}	t        |||	|| j                   |      }
|
j                  dddd      j	                  ||d      }
t#        j$                  | j'                  |            }|
|z  }
| j)                  |
      S )Nr   r+   r	      )offset)r]   rM   r\   )shaperP   rQ   rR   reshaperK   r    	transposerL   rU   rV   rC   rX   ra   update_and_fetchr   rM   mxr/   rW   rS   )rY   r[   r\   r]   BLDquerieskeysvaluesoutputgates               r?   __call__zAttention.__call__[   s    ''1a++a.{{1~Q//!QdmmDNNq!Q
 ||Aq$//4==AKKAqRSUVW1doot}}EOOq!Q
 ++g&{{4 ""tyy'< ))GELL)Ayyelly;))G,yy 11$?LD&-T6djjt
 !!!Q1-55aB?zz$..+,${{6""r>   FNN)r5   r6   r7   r   r<   rJ   rg   arrayr   r   rp   __classcell__rZ   s   @r?   rA   rA   0   s_    (Y (D (Z $(#	+#88+# rxx +# }	+#
 
+#r>   rA   c                   P     e Zd Zddedee   f fdZdej                  fdZ	 xZ
S )MLPrB   r   c                    t         |           |j                  }||n|j                  }t	        j
                  ||d      | _        t	        j
                  ||d      | _        t	        j
                  ||d      | _        y NFrE   )	rI   rJ   r   r   rN   rO   rW   	down_projup_proj)rY   rB   r   dim
hidden_dimrZ   s        r?   rJ   zMLP.__init__   sr     !, '' 	 3
?:s?yyju=r>   r^   c                 t    | j                  t        | j                  |      | j                  |                  S N)rz   r
   rW   r{   rY   r[   s     r?   rp   zMLP.__call__   s)    ~~fT^^A%6QHIIr>   r   )r5   r6   r7   r   r   r:   rJ   rg   rs   rp   rt   ru   s   @r?   rw   rw      s,    >Y >8C= >JRXX Jr>   rw   c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )	MoERouterz;Router module that wraps the gate for proper weight naming.rB   c                     t         |           t        j                  |j                  |j
                  d      | _        y ry   )rI   rJ   rN   rO   r   r'   ro   rY   rB   rZ   s     r?   rJ   zMoERouter.__init__   s.    IId..0@0@uM	r>   r[   r^   c                 $    | j                  |      S r   )ro   r   s     r?   rp   zMoERouter.__call__   s    yy|r>   )
r5   r6   r7   __doc__r   rJ   rg   rs   rp   rt   ru   s   @r?   r   r      s/    ENY N"(( rxx r>   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )AfmoeMoErB   c                 ^   t         |           || _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        |      | _        t        j                  |j                  f      | _        t        |j                   |j"                  |j                        | _        |j&                  dkD  r,|j"                  |j&                  z  }t)        ||      | _        y y )Nr   )r   )rI   rJ   rB   r'   r)   r-   r.   r0   r1   r2   r   routerrg   zerosexpert_biasr   r   r   expertsr*   rw   shared_experts)rY   rB   shared_intermediate_sizerZ   s      r?   rJ   zAfmoeMoE.__init__   s    	++#'#;#; //++//||//o88T%5%5$78 &&
 ""Q&**T-D-DD % #&d>V"WD	 'r>   r[   r^   c                    | j                  |      }| j                  dk(  r3t        j                  |j	                  t        j
                              }n4t        j                  |j	                  t        j
                        d      }|| j                  z   }| j                  dkD  rt        j                  |d| j                  df      }t        j                  |dd      j                  dd      }| j                  | j                  z
  }t        j                  ||dz
  d	
      dd |d d f   }t        j                  |t        j                  |      t        j                   d      d	      }t        j"                  |d	d      }| j$                  }t        j                  | |dz
  d
      dd |f   }t        j&                  ||d      }	| j(                  r'| j$                  dkD  r|	j                  dd      }
|	|
z  }	|	| j*                  z  }	| j-                  ||      }||	d   z  j                  d	      j	                  |j.                        }| j0                  j2                  dkD  r|| j5                  |      z   }|S )Nr/   rb   )axisr	   )r   rc   r+   T)r   keepdims)kthr   .g        ).Nr   )r   r0   rg   r/   astypefloat32softmaxr   r1   	unflattentopksumr2   argpartitionput_along_axisstop_gradientrs   flattenr)   take_along_axisr-   r.   r   dtyperB   r*   r   )rY   r[   gatesscoresselection_scoresgroup_scoresk	group_idxindsselected_scoresdenominatorys               r?   rp   zAfmoeMoE.__call__   s0   A??i'ZZRZZ 89FZZRZZ 8rBF "D$4$44 <<!!|| r$,,1C  77#3QR@DD$ E L t.A!a%bI#rPQrST*UI!00 ""2"29"=rxx}SU   "zz*:BC $$ 00a!e"Ec2A2gN,,VTC??t77!;)--2-EK-;O)D,<,<<LLD!++00b09@@I99''!+D''**Ar>   	r5   r6   r7   r   rJ   rg   rs   rp   rt   ru   s   @r?   r   r      s,    XY X6,"(( ,rxx ,r>   r   c            	            e Zd Zd
dededef fdZ	 	 ddej                  de	ej                     de	e
   dej                  fd	Z xZS )DecoderLayerrB   	layer_idxuse_slidingc                 d   t         |           |j                  | _        || _        || _        t        ||      | _        ||j                  k  rt        |      | _	        nt        |      | _	        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y )N)rC   rG   )rI   rJ   r   r   r   rA   	self_attnr,   rw   mlpr   rN   rT   r"   input_layernormpost_attention_layernormpre_mlp_layernormpost_mlp_layernorm)rY   rB   r   r   rZ   s       r?   rJ   zDecoderLayer.__init__   s    ++&""4KHt,,,4yDH~DH!zz$*:*:@Q@QR(*

$"3"3)
% "$D,<,<$BSBS!T"$**T-=-=4CTCT"Ur>   r[   r\   r]   r^   c                     | j                  | j                  |      ||      }| j                  |      }||z   }| j                  | j	                  |            }| j                  |      }||z   S r   )r   r   r   r   r   r   )rY   r[   r\   r]   rhs         r?   rp   zDecoderLayer.__call__  sl     NN4//2D%@))!,EHHT++A./##A&1ur>   rq   rr   )r5   r6   r7   r   r:   r<   rJ   rg   rs   r   r   rp   rt   ru   s   @r?   r   r      si    VY V3 VT V. $(#	88 rxx  }	
 
r>   r   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )
AfmoeModelrB   c           
         t         |           || _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        t        j                  |j                  |j                        | _        t        | j
                        D cg c]  \  }}t        |||dk(         c}}| _        t        j                  |j                  |j                         | _        | j
                  j%                  d      | _        d | _        t        | j                        D ]  \  }}|j*                  s|| _         y  y c c}}w )Nsliding_attention)rB   r   r   rG   full_attention)rI   rJ   rB   r   r   r   r3   r4   r   rN   	Embeddingembed_tokens	enumerater   layersrT   r"   normindexfa_idxswa_idxr   )rY   rB   idx
layer_typelayerrZ   s        r?   rJ   zAfmoeModel.__init__  s8   	//!%!7!7++"11++++LL$:J:JK
 $-T-=-=#>	
 $?Z SjDW6W $?	
 JJt//T5F5FG	&&,,-=>#DKK0JC  " 1
s   E2inputsc                    | j                  |      }| j                  r"|t        j                  | j                        z  }|d gt        | j                        z  }t        ||| j                           }d }| j                  %t        ||| j                     | j                        }t        | j                  |      D ]   \  }}|j                  r|n|} ||||      }" | j                  |      S )N)window_size)r]   )r   r4   mathsqrtr   lenr   r   r   r   r3   zipr   r   )	rY   r   r]   r   fa_maskswa_maskr   cr\   s	            r?   rp   zAfmoeModel.__call__1  s    
 f%DIId..//A=FS--E'5+=><<#,5&D4G4GH DKK/HE1$008gDaQ'A 0 yy|r>   r   r   ru   s   @r?   r   r     s%    Y : r>   r   c                        e Zd Zdef fdZ	 d
dej                  fdZd Ze	d        Z
d Ze	d        Ze	d	        Z xZS )ModelrB   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y ry   )rI   rJ   rB   r   r   modelr%   rN   rO   r   r   lm_headr   s     r?   rJ   zModel.__init__M  sX    	//%
''99T%5%5tUSDL (r>   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r   )r   rB   r%   r   	as_linearr   )rY   r   r]   outs       r?   rp   zModel.__call__V  sW    
 jj'99((**))33C8C 
 ,,s#C
r>   c                 D   |j                         D ci c]  \  }}d|vs|| }}}| j                  j                  r|j                  dd        t	        | j                  j
                        D ]  }|| j                  j                  k  rd| }dD ]|  }dD ]u  }| d| d| |v st	        | j                  j                        D cg c]  }|j                  | d| d| d|         }}t        j                  |      || d| d| <   w ~  |S c c}}w c c}w )	Nzrotary_emb.inv_freqzlm_head.weightzmodel.layers.)r{   rz   rW   )weightscalesbiasesz.mlp.experts.0..z.mlp.experts.)
itemsrB   r%   popranger   r,   r'   rg   stack)	rY   weightsr   vlprefixneto_joins	            r?   sanitizezModel.sanitizeb  s=   $+MMOVODAq7LTU7U1a4OV99((KK($/ tyy223A499---$QC(F:7A 1QC8GC &+499+@+@%A#%A $KK6(-s!A3as(KL%A   # DF88GCT6(-s!A3 ?@ 8 ;	 4 ' W#s   DD
#D
c                 .    | j                   j                  S r   )r   r   )rY   s    r?   r   zModel.layersy  s    zz   r>   c                     | j                   D cg c]8  }|j                  r t        | j                  j                        n	t               : c}S c c}w )N)max_size)r   r   r   r   r3   r   )rY   r   s     r?   
make_cachezModel.make_cache}  sU     
 % $$  )B)BCY %
 	
 
s   =Ac                     d }|S )Nc                 
    d| vS )Nr   r=   )r   s    r?   	predicatez'Model.cast_predicate.<locals>.predicate  s     ))r>   r=   rY   r   s     r?   cast_predicatezModel.cast_predicate  s    	* r>   c                     d }|S )Nc                     d| v rdddS y)Nzrouter.gater   r(   )
group_sizebitsTr=   )path_s     r?   r   z(Model.quant_predicate.<locals>.predicate  s    $&(!44r>   r=   r   s     r?   quant_predicatezModel.quant_predicate  s    	
 r>   r   )r5   r6   r7   r   rJ   rg   rs   rp   r   propertyr   r   r   r   rt   ru   s   @r?   r   r   L  sm    TY T 

. ! !
    r>   r   )$r   dataclassesr   typingr   r   r   r   r   mlx.corecorerg   mlx.nnrN   activationsr
   baser   r   r   r]   r   r   
rope_utilsr   switch_layersr   r   ModulerA   rw   r   r   r   r   r   r=   r>   r?   <module>r     s     ! 3 3    T T + ' $   <V#		 V#rJ")) J&		 Hryy HV!299 !H3 3lIBII Ir>   