
    (HJj                        d dl mZ d dlmZmZmZmZ d dlmZ	 d dl
mZ ddlmZ ddlmZmZmZ ddlmZ e G d d	e             Z G d
 dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Z G d dej*                        Zy)    )	dataclass)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)	SwitchGLUc                       e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   eed<   dZee   ed<   dZ	eed<   dZ
eed<   dZeeeeeef   f      ed<   dZeed<   d Zy)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsnum_experts_per_toknum_expertsmoe_intermediate_sizeshared_expert_intermediate_sizerms_norm_eps
vocab_sizeNnum_key_value_headsi@B 
rope_thetaFrope_traditionalrope_scalingtie_word_embeddingsc                       j                    j                   _          j                  rDddh}t         fd|D              st	        d|        j                  d   dk7  rt	        d      y y )Nfactortypec              3   :   K   | ]  }|j                   v   y wN)r   ).0keyselfs     a/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/qwen2_moe.py	<genexpr>z*ModelArgs.__post_init__.<locals>.<genexpr>'   s     I=Csd///=s   zrope_scaling must contain keys linearz4rope_scaling 'type' currently only supports 'linear')r   r   r   all
ValueError)r'   required_keyss   ` r(   __post_init__zModelArgs.__post_init__!   sz    ##+'+'?'?D$%v.MI=II #B=/!RSS  (H4 !WXX 5     )__name__
__module____qualname__str__annotations__intfloatr   r   r   r   boolr   r   r   r   r.    r/   r(   r   r      s    O%((O)-#-J"d";?L(4U5#:%6 678? %%
Yr/   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )		Attentionargsc                 6   t         |           |j                  }|j                  x| _        }|j
                  J |j
                  x| _        }|j                  |z  }|dz  | _        t        j                  |||z  d      | _
        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  ||z  |d      | _        t        j                  ||j                  |j                         | _        y )Ng      TbiasF)traditionalbase)super__init__r   r   n_headsr   
n_kv_headsscalennLinearq_projk_projv_projo_projRoPEr   r   rope)r'   r;   dimrC   rD   head_dim	__class__s         r(   rB   zAttention.__init__/   s    !%!9!99w''333'+'?'??*##w.t^
iiWx%7dCiiZ(%:FiiZ(%:Fii( 2CeDGG--
	r/   xmaskcachereturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr      r      )offset)rS   rE   rR   )shaperH   rI   rJ   reshaperC   	transposerD   rM   rY   update_and_fetchr   rE   rK   )r'   rQ   rR   rS   BLDquerieskeysvaluesoutputs              r(   __call__zAttention.__call__E   si    ''1a $AAAv //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r/   NNr0   r1   r2   r   rB   mxarrayr   r   re   __classcell__rP   s   @r(   r:   r:   .   sW    
Y 
2 $(#	#88# rxx # }	#
 
#r/   r:   c                   >     e Zd Z fdZdej
                  fdZ xZS )MLPc                     t         |           t        j                  ||d      | _        t        j                  ||d      | _        t        j                  ||d      | _        y NFr=   )rA   rB   rF   rG   	gate_proj	down_projup_proj)r'   rN   
hidden_dimrP   s      r(   rB   zMLP.__init__d   sJ    3
?:s?yyju=r/   rT   c                 t    | j                  t        | j                  |      | j                  |                  S r$   )rq   r	   rp   rr   )r'   rQ   s     r(   re   zMLP.__call__j   s)    ~~fT^^A%6QHIIr/   )r0   r1   r2   rB   rh   ri   re   rj   rk   s   @r(   rm   rm   c   s    >JRXX Jr/   rm   c                   D     e Zd Zdef fdZdej                  fdZ xZS )Qwen2MoeSparseMoeBlockr;   c                 l   t         |           |j                  }|j                  }|j                  }|j
                  x| _        }|j                  | _        t        j                  ||d      | _
        t        |||      | _        t        ||      | _        t        j                  |dd      | _        y )NFr=   r   )rA   rB   r   r   r   r   r   top_krF   rG   gater   
switch_mlprm   shared_expertshared_expert_gate)r'   r;   rN   r   r   r   rP   s         r(   rB   zQwen2MoeSparseMoeBlock.__init__o   s     66*.*N*N')-)9)99;--
IIc;U;	#C):KH &EF"$))C"?r/   rQ   c                    | j                  |      }t        j                  |dd      }| j                  }t        j                  t        j
                  | |dz
  d      dd |f         }t        j                  ||d      }| j                  ||      }||d   z  j                  d	      }| j                  |      }t        j                  | j                  |            |z  }||z   S )
NrV   T)axispreciser   )kthr~   .)r~   ).N)ry   rh   softmaxrx   stop_gradientargpartitiontake_along_axisrz   sumr{   sigmoidr|   )r'   rQ   gateskindsscoresyshared_expert_outputs           r(   re   zQwen2MoeSparseMoeBlock.__call__~   s     		!

5r48JJAE KCQSRSQSG TU##E4b9OOAt$	""''R'0#11!4JJt..q125II 	 '''r/   	r0   r1   r2   r   rB   rh   ri   re   rj   rk   s   @r(   rv   rv   n   s"    @Y @(88(r/   rv   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	Qwen2MoeDecoderLayerr;   c                 R   t         |           |j                  | _        t        |      | _        t        |      | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        || _        y )Neps)rA   rB   r   r:   	self_attnrv   mlprF   RMSNormr   input_layernormpost_attention_layernormr;   r'   r;   rP   s     r(   rB   zQwen2MoeDecoderLayer.__init__   s}    ++"4)$/!zz$*:*:@Q@QR(*

$"3"3)
% 	r/   rQ   rR   rS   rT   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S r$   )r   r   r   r   )r'   rQ   rR   rS   rhouts          r(   re   zQwen2MoeDecoderLayer.__call__   sQ     NN4//2D%@EHHT22156!e
r/   rf   rg   rk   s   @r(   r   r      sW    
Y 
 $(#	
88
 rxx 
 }	

 

r/   r   c                   H     e Zd Zdef fdZ	 ddej                  fdZ xZS )Qwen2MoeModelr;   c                    t         |           || _        |j                  | _        |j                  | _        | j                  dkD  sJ t        j                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t        j                  |j                  |j                        | _        y c c}w )Nr   )r;   r   )rA   rB   r;   r   r   rF   	Embeddingr   embed_tokensranger   layersr   r   norm)r'   r;   _rP   s      r(   rB   zQwen2MoeModel.__init__   s    	//!%!7!7"""LL$:J:JK5:4;Q;Q5R
5R d+5R
 JJt//T5F5FG	
s   Cinputsc                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )r   lenr   r   zipr   )r'   r   rS   r   rR   layercs          r(   re   zQwen2MoeModel.__call__   su    
 f%=FS--E$Qa1DKK/HE1aq!A 0 yy|r/   r$   r   rk   s   @r(   r   r      s'    
HY 
H r/   r   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )Modelr;   c                     t         |           || _        |j                  | _        t	        |      | _        t        j                  |j                  |j                  d      | _
        y ro   )rA   rB   r;   r   r   modelrF   rG   r   r   lm_headr   s     r(   rB   zModel.__init__   sJ    	//"4(
yy!1!14??Or/   r   c                 H    | j                  ||      }| j                  |      S r$   )r   r   )r'   r   rS   r   s       r(   re   zModel.__call__   s#    
 jj'||C  r/   c                 p   d|vr|S t        | j                  j                        D ]  }d| }dD ]|  }dD ]u  }| d| d| |v st        | j                  j                        D cg c]  }|j	                  | d| d| d|         }}t        j                  |      || d| d| <   w ~  |S c c}w )	Nz+model.layers.0.mlp.experts.0.up_proj.weightzmodel.layers.)rr   rq   rp   )weightscalesbiasesz.mlp.experts.0..z.mlp.experts.z.mlp.switch_mlp.)r   r;   r   r   poprh   stack)r'   weightslprefixnr   eto_joins           r(   sanitizezModel.sanitize   s    8GNtyy223A$QC(F:7A 1QC8GC &+499+@+@%A#%A $KK6(-s!A3as(KL%A   # GIhhwFW6(*:1#Qqc BC 8 ; 4 #s   &#B3
c                 .    | j                   j                  S r$   )r   r   )r'   s    r(   r   zModel.layers   s    zz   r/   r$   )r0   r1   r2   r   rB   rh   ri   re   r   propertyr   rj   rk   s   @r(   r   r      s@    PY P !! ! !r/   r   )dataclassesr   typingr   r   r   r   mlx.corecorerh   mlx.nnrF   activationsr	   r@   r
   r   r   switch_layersr   r   Moduler:   rm   rv   r   r   r   r8   r/   r(   <module>r      s    " - -    T T $ Y Y Y>2#		 2#jJ")) J#(RYY #(L299 4BII >!!BII !!r/   