
    (HJj                     f   d dl Z d dlmZ d dlmZmZmZmZ d dlm	Z
 d dlmZ ddlmZmZmZ ddlmZ ddlmZ e G d d	e             Z G d
 dej,                        Z G d dej,                        Z G d dej,                        Z G d dej,                        Z G d dej,                        Zy)    N)	dataclass)DictListOptionalUnion   )BaseModelArgscreate_attention_maskscaled_dot_product_attention)SuScaledRoPE)	SwitchGLUc                       e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   d	Z
eed
<   d	Zeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeeeeee   f   f   ed<   dZeed<   dZeed<   dZeed<   y)	ModelArgsphimoe
model_typei@}  
vocab_sizei   hidden_sizei   intermediate_size    num_hidden_layersnum_attention_heads   num_key_value_headsi   max_position_embeddings original_max_position_embeddingsgư>rms_norm_epsNrope_scaling   num_local_experts   num_experts_per_tokg     @
rope_theta)__name__
__module____qualname__r   str__annotations__r   intr   r   r   r   r   r   r   r   floatr   r   r   r   r   r!   r"        ^/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/phimoe.pyr   r      s    JJK!s!s!!  #)S),0$c0L%9=L$sE%e"4556=s  Jr+   r   c                        e Zd Zdef fdZ	 	 ddej                  deej                     dej                  fdZ xZ	S )	Attentionargsc                    t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  |z  }|dz  | _        t        j                  |||z  d      | _
        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  ||z  |d      | _        t        ||j                  |j                   |j"                  |j$                  d   |j$                  d   |j$                  d   |j$                  d         | _        y )	Ng      Tbiasshort_factorlong_factorshort_mscalelong_mscale)baser   r   r3   r4   r5   r6   )super__init__r   r   n_headsr   
n_kv_headsscalennLinearq_projk_projv_projo_projr   r"   r   r   r   rope)selfr/   dimr:   r;   head_dim	__class__s         r,   r9   zAttention.__init__!   s   !%!9!99w'+'?'??*##w.t^
iiWx%7dCiiZ(%:FiiZ(%:Fii( 2CdC $($@$@-1-R-R**>:))-8**>:))-8	
	r+   xmaskreturnc                 2   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}|j	                  ||| j
                  d      j                  dddd      }|j	                  ||| j                  d      j                  dddd      }|	j	                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j	                  ||d      }
| j                  |
      S )Nr   r    r      )offset)cacher<   rI   )shaper?   r@   rA   reshaper:   	transposer;   rC   rN   update_and_fetchr   r<   rB   )rD   rH   rI   rO   BLDquerieskeysvaluesoutputs              r,   __call__zAttention.__call__;   si    ''1a $AAAv //!Qb9CCAq!QO||Aq$//26@@Aq!L1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r+   NN
r#   r$   r%   r   r9   mxarrayr   r[   __classcell__rG   s   @r,   r.   r.       sI    
Y 
: $(	#88# rxx #
 
#r+   r.   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )PhiMoESparseMoeBlockr/   c                 l   t         |           |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  | j                  | j                  d      | _        t        | j                  | j
                  | j                        | _        y )NFr1   )r8   r9   r   
hidden_dimr   ffn_dimr   num_expertsr!   top_kr=   r>   gater   
switch_mlprD   r/   rG   s     r,   r9   zPhiMoESparseMoeBlock.__init__Z   s    **--11--
IIdoot/?/?eL	#DOOT\\4CSCSTr+   rH   rJ   c                 `   | j                  |      }| j                  }t        j                  t        j                  | |dz
  d      dd |f         }t        j
                  ||d      }t        j                  |dd      }| j                  ||      }||d   z  j                  d	      }|S )
Nr   rL   )kthaxis.)rn   T)rn   precise).N)	ri   rh   r^   stop_gradientargpartitiontake_along_axissoftmaxrj   sum)rD   rH   gateskindsscoresys          r,   r[   zPhiMoESparseMoeBlock.__call__d   s    		!JJAE KCQSRSQSG TU##E4b9FT:OOAt$	""''R'0r+   	r#   r$   r%   r   r9   r^   r_   r[   r`   ra   s   @r,   rc   rc   Y   s,    UY U"(( rxx r+   rc   c                        e Zd Zdef fdZ	 	 ddej                  deej                     dej                  fdZ xZ	S )PhiMoEDecoderLayerr/   c                 D   t         |           |j                  | _        t        |      | _        t        |      | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y N)eps)r8   r9   r   r.   	self_attnrc   block_sparse_moer=   	LayerNormr   input_layernormpost_attention_layernormrk   s     r,   r9   zPhiMoEDecoderLayer.__init__s   su    ++"4 4T :!||D,<,<$BSBST(*$"3"3)
%r+   rH   rI   rJ   c                     |}| j                  |      }| j                  |||      }||z   }|}| j                  |      }| j                  |      }||z   }|S )N)rI   rO   )r   r   r   r   )rD   rH   rI   rO   residualhidden_statess         r,   r[   zPhiMoEDecoderLayer.__call__~   sl     ,,Q/}4uM =0 55mD--m< =0r+   r\   r]   ra   s   @r,   r}   r}   r   sI    	
Y 	
 $(	88 rxx 
 
r+   r}   c                   `     e Zd Zdef fdZ	 ddej                  dej                  fdZ xZS )PhiMoEModelr/   c                 |   t         |           || _        |j                  | _        t	        j
                  |j                  |j                        | _        t        |j                        D cg c]  }t        |       c}| _        t	        j                  |j                  |j                        | _        y c c}w r   )r8   r9   r/   r   r=   	Embeddingr   embed_tokensranger   r}   layersr   r   norm)rD   r/   _rG   s      r,   r9   zPhiMoEModel.__init__   s    	//LL$:J:JK9>t?U?U9VW9VA)$/9VWLL!1!1t7H7HI	 Xs   .B9inputsrJ   c                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )r   lenr   r
   zipr   )rD   r   rO   hrI   layercs          r,   r[   zPhiMoEModel.__call__   su    
 f%=FS--E$Qa1DKK/HE1aq!A 0 yy|r+   Nr{   ra   s   @r,   r   r      s5    JY J  
	r+   r   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )Modelr/   c                     t         |           |j                  | _        || _        t	        |      | _        t        j                  |j                  |j                  d      | _
        y )NTr1   )r8   r9   r   r/   r   modelr=   r>   r   r   lm_headrk   s     r,   r9   zModel.__init__   sJ    //	 &
yy!1!14??Nr+   r   c                 H    | j                  ||      }| j                  |      S r   )r   r   )rD   r   rO   outs       r,   r[   zModel.__call__   s#    
 jj'||C  r+   c                 v   d|vr|S t        | j                  j                        D ]  }d| }dD ]  \  }}dD ]u  }| d| d| |v st        | j                  j                        D cg c]  }|j	                  | d| d| d|         }}t        j                  |      || d| d| <   w   |S c c}w )	Nz3model.layers.0.block_sparse_moe.experts.0.w1.weightzmodel.layers.))w1	gate_proj)w2	down_proj)w3up_proj)weightscalesbiasesz.block_sparse_moe.experts.0..z.block_sparse_moe.experts.z.block_sparse_moe.switch_mlp.)r   r/   r   r   popr^   stack)	rD   weightslprefixnmrw   eto_joins	            r,   sanitizezModel.sanitize   s    @ONtyy223A$QC(FU17A !=aS!EP
 &+499+F+F%G	# &H $KK#)(*DQCq1QC P &H	   # HHW-  6(*Gs!A3 OP 8 V 4 #s   )#B6
c                 .    | j                   j                  S r   )r   r   )rD   s    r,   r   zModel.layers   s    zz   r+   r   )r#   r$   r%   r   r9   r^   r_   r[   r   propertyr   r`   ra   s   @r,   r   r      s@    OY O !!( ! !r+   r   )mathdataclassesr   typingr   r   r   r   mlx.corecorer^   mlx.nnr=   r7   r	   r
   r   
rope_utilsr   switch_layersr   r   Moduler.   rc   r}   r   r   r*   r+   r,   <module>r      s     ! . .   T T $ $      "6#		 6#r299 2 >")) 6&!BII &!r+   