
    (HJjE                     $   d dl Z d dlmZ d dlmZmZmZ d dlmZ	 d dl
mZ d dlmZmZmZ ddlmZ ddlmZmZmZ ddlmZ dd	lmZ e G d
 de             Z	 d dZ	 d dZd!dZd Z G d dej>                        Z  G d dej>                        Z! G d dej>                        Z" G d dej>                        Z# G d dej>                        Z$ G d dej>                        Z% G d deej>                        Z& G d dej>                        Z'y)"    N)	dataclass)AnyDictOptional)shard_inplaceshard_linearsum_gradients   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)PipelineMixin)	SwitchGLUc                      e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZee   ed<   dZee   ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZee   ed <   dZee   ed!<   dZee   ed"<   d#Zeed$<   d%Zeed&<   d'Zeed(<   d)Zeed*<   d+Z eed,<   dZ!e"ed-<   d.Z#e$ed/<   y)0	ModelArgsdeepseek_v2
model_typei  
vocab_size   hidden_sizei +  intermediate_sizei  moe_intermediate_size   num_hidden_layers    num_attention_headsnum_key_value_headsNn_shared_expertsn_routed_experts      ?routed_scaling_factori   kv_lora_ranki   q_lora_rank@   qk_rope_head_dim   
v_head_dimqk_nope_head_dimgreadytopk_methodn_group
topk_groupnum_experts_per_tokr
   moe_layer_freqr   first_k_dense_replace   max_position_embeddingsư>rms_norm_epsg     @
rope_thetarope_scalingFattention_bias)%__name__
__module____qualname__r   str__annotations__r   intr   r   r   r   r   r   r   r   r    r"   floatr#   r$   r&   r(   r)   r+   r,   r-   r.   r/   r0   r2   r4   r5   r6   r   r7   bool     c/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/deepseek_v2.pyr   r      s1   #J#JK"s"!%3%s!!!!&*hsm*&*hsm*#&5&L#KcJcK!GXc]! $J$)-#-NC!"3"#'S'L%JL$ ND rA   r   c                     |t        j                  || dz  t         j                  z  z        z  dt        j                  |      z  z  S )N   )mathlogpi)num_rotationsdimbaser2   s       rB   yarn_find_correction_dimrK   0   sC     $((2ma6G$''6QRSS	DHHTN rA   c                     t        j                  t        | |||            }t        j                  t        ||||            }t	        |d      t        ||dz
        fS )Nr   r
   )rE   floorrK   ceilmaxmin)low_rothigh_rotrI   rJ   r2   lowhighs          rB   yarn_find_correction_rangerU   8   s^     ** #t5LMC 99 36MND sA;D#'***rA   c                 J    | dk  ryd|z  t        j                  |       z  dz   S )Nr
   r!   g?)rE   rF   )scalemscales     rB   yarn_get_mscalerY   D   s(    z<$((5/)C//rA   c                     | |k(  r|dz  }t        j                  |t         j                        | z
  || z
  z  }t        j                  |dd      S )NgMbP?dtyper   r
   )mxarangefloat32clip)min_valmax_valrI   linear_funcs       rB   yarn_linear_ramp_maskrd   J   sJ    '599S

3g='GBSTK77;1%%rA   c                   8     e Zd Z	 	 	 	 	 	 	 	 d fd	ZddZ xZS )DeepseekV2YarnRotaryEmbeddingc
                    t         |           t        ||      t        ||	      z  | _        |t	        j
                  d|dt        j                        |z  z  }
||t	        j
                  d|dt        j                        |z  z  z  }t        |||||      \  }}dt        |||dz        z
  }||
z  ||z  |
d|z
  z  z   z  | _	        y )Nr   rD   r[   r!   r
   )
super__init__rY   rX   r]   r^   r_   rU   rd   _freqs)selfrI   r2   rJ   scaling_factor original_max_position_embeddings	beta_fast	beta_slowrX   mscale_all_dim
freq_extra
freq_interrS   rT   	freq_mask	__class__s                  rB   ri   z&DeepseekV2YarnRotaryEmbedding.__init__S   s     	%nf=NA
 
 bii3DsJK
#dIIaarzz2S8'
 

 /,
	T /T3!8DD	!J."Z1y=%AA
rA   c           	          | j                   dk7  r| j                   |z  }t        j                  j                  ||j                  d   dd d|| j
                        S )Nr!   T)traditionalrJ   rW   offsetfreqs)rX   r]   fastropeshaperj   )rk   xrx   s      rB   __call__z&DeepseekV2YarnRotaryEmbedding.__call__s   sV    ;;#aAww||GGBK++  
 	
rA   )r1   '  r!   r   r   r
   r
   r   )r   )r8   r9   r:   ri   r~   __classcell__rt   s   @rB   rf   rf   R   s(     !%)-
@
rA   rf   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	DeepseekV2Attentionconfigc                 l   t         |           || _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        |j                  | _
        |j                  | _        |j                  | _        |j                  |j                  z   | _        | j                  dz  | _        | j                  ?t        j                   | j                  | j
                  | j                  z  d      | _        nt        j                   | j                  | j                  |j$                        | _        t        j(                  | j                  d      | _        t        j                   | j                  | j
                  | j                  z  d      | _        t        j                   | j                  | j                  | j                  z   |j$                        | _        t        j(                  | j                  d      | _        t        j                   | j                  | j
                  | j                  | j                  z
  | j                  z   z  d      | _        t        j                   | j
                  | j                  z  | j                  |j$                        | _        | j                  j6                  j9                  dd      }| j                  j6                  d   }|r#t;        ||      }| j                  |z  |z  | _        d	D ci c]4  }|| j                  j6                  v r|| j                  j6                  |   6 }}t=        d| j                  | j                  || j                  d
|| _        y c c}w )Ng      Fbiasr3   epsrp   r   factor)rm   rn   ro   rX   rp   )rI   r2   rl   rJ   r@   ) rh   ri   r   r   r   	num_headsr2   r5   r$   r&   r#   r(   r)   
q_head_dimrW   nnLinearq_projr7   q_a_projRMSNormq_a_layernormq_b_projkv_a_proj_with_mqakv_a_layernorm	kv_b_projo_projr6   getrY   rf   r{   )rk   r   rp   rl   rX   keyrope_kwargsrt   s          rB   ri   zDeepseekV2Attention.__init__   s   !--33'-'E'E$ ++!-- & 7 7"// ++ & 7 7 11F4K4KK__d*
#))  $..4??"BDK II  $"2"29N9NDM "$D,<,<$!GDII  $..4??"BDM #%)) 5 55&&#

 !jj):):ENN!6!66HJ	
 iiNNT__,&&
 11556FJ11(;$^^DFf,v5DJ

 dkk... ))#.. 	 

 2 
%%$($@$@)	

 
	

s   9N1r}   maskcachereturnc                    |j                   \  }}}| j                  | j                  |      }n/| j                  | j	                  | j                  |                  }|j                  ||| j                  | j                        j                  dddd      }t        j                  || j                  gd      \  }}	| j                  |      }
t        j                  |
| j                  gd      \  }
}|j                  ||d| j                        j                  dddd      }| j!                  | j#                  |
            }|j                  ||| j                  d      j                  dddd      }t        j                  || j                  gd      \  }}|| j%                  |	|j&                        }	| j%                  ||j&                        }t        j(                  || j                  d      }|j+                  t        j,                  ||gd      |      \  }}n]| j%                  |	      }	| j%                  |      }t        j(                  || j                  d      }t        j,                  ||gd      }t        j,                  ||	gd      }t/        ||||| j0                  |      }|j                  dddd      j                  ||d      }| j3                  |      S )Nr   rD   r
      rv   axis)r   rW   r   )r|   r$   r   r   r   r   reshaper   r   	transposer]   splitr)   r   r#   r&   r   r   r{   rx   repeatupdate_and_fetchconcatenater   rW   r   )rk   r}   r   r   BLDqq_nopeq_pecompressed_kvk_pekvk_nopevalueskeysqueriesoutputs                     rB   r~   zDeepseekV2Attention.__call__   so    ''1a#AAd00q1ABCAIIaDNNDOO<FFq!QPQRxxD$9$9#:D//2 hh}t7H7H6IPRSt||Aq!T%:%:;EEaAqQ^^D//>?ZZ1dnnb1;;Aq!QG"t'<'<&=BG99T5<<0D99T5<<0D99T4>>:D 11~B7LD& 99T?D99T?D99T4>>:D>>64.r:D..&$b9-T6djjt
 !!!Q1-55aB?{{6""rA   NNr8   r9   r:   r   ri   r]   arrayr   r   r~   r   r   s   @rB   r   r      sZ    G
y G
X $(#	*#88*# rxx *# }	*#
 
*#rA   r   c                   6     e Zd Z	 ddededef fdZd Z xZS )DeepseekV2MLPr   r   r   c                    t         |           || _        ||j                  n|| _        ||j                  n|| _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _	        y NFr   )
rh   ri   r   r   r   r   r   	gate_projup_proj	down_proj)rk   r   r   r   rt   s       rB   ri   zDeepseekV2MLP.__init__   s     	1<1D6--+(9(AF$$GX 	 4#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXrA   c                 x    | j                  t        | j                  |      | j                  |                  }|S N)r   r   r   r   )rk   r}   r   s      rB   r~   zDeepseekV2MLP.__call__  s.    NN6$..*;T\\!_#MN	rA   r   )r8   r9   r:   r   r=   ri   r~   r   r   s   @rB   r   r      s,    SWYY.1YMPYrA   r   c                   *     e Zd Zdef fdZd Z xZS )MoEGater   c                 \   t         |           || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  | j
                  |j                  f      | _        y r   )rh   ri   r   r.   top_kr    r"   r+   r,   r-   r]   zerosr   weightrk   r   rt   s     rB   ri   zMoEGate.__init__  s    //
 & 7 7%+%A%A"!--~~ ++hh 5 5v7I7IJKrA   c                    || j                   j                  z  }t        j                  |dd      }| j                  dk(  r|j
                  d d \  }}|j                  ||| j                  d      }|j                  dd      }| j                  | j                  z
  }t        j                  ||dz
  d	      d
d |d d f   }t        j                  ||t        j                  d|j                        d      }|j                  ||d      }| j                  }t        j                  | |dz
  d	      d
d |f   }	t        j                  ||	d      }|| j                   z  }|	|fS )Nrv   T)r   precisegroup_limited_greedyrD   )r   keepdimsr
   )kthr   .g        r   )r   Tr]   softmaxr+   r|   r   r,   rO   r-   argpartitionput_along_axisr   r\   r   take_along_axisr"   )
rk   r}   gatesscoresbszseq_lengroup_scoresk	group_idxindss
             rB   r~   zMoEGate.__call__  sC   DKKMM!ED955772A;LC^^C$,,CF!::2:=Lt.A!a%bI#rPQrST*UI&&	288C#>RF ^^C"5FJJwAE;C!GD##FDr:$444V|rA   r8   r9   r:   r   ri   r~   r   r   s   @rB   r   r     s    	Ly 	LrA   r   c                   *     e Zd Zdef fdZd Z xZS )DeepseekV2MoEr   c                 N   t         |           || _        |j                  | _        t	        |j
                  |j                  |j                        | _        t        |      | _
        |j                  +|j                  |j                  z  }t        ||      | _        d | _        y )N)r   r   )rh   ri   r   r.   r   r   r   r    
switch_mlpr   gater   r   shared_expertssharding_group)rk   r   r   rt   s      rB   ri   zDeepseekV2MoE.__init__1  s    #)#=#= # < <f>U>U
 FO	"". & < <v?V?V V"/1B#D #rA   c                    | j                    t        | j                         |      }| j                  |      \  }}| j                  ||      }||d   z  j	                  d      }| j
                  j                  || j                  |      z   }| j                   +t        j                  j                  || j                         }|S )N).Nr   r   group)r   r	   r   r   sumr   r   r   r]   distributedall_sum)rk   r}   r   r   ys        rB   r~   zDeepseekV2MoE.__call__B  s    *2d11215Ayy|fOOAt$	""''R'0;;''3D''**A*&&q0C0C&DArA   r   r   s   @rB   r   r   0  s    #y #"rA   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
DeepseekV2DecoderLayerr   	layer_idxc                    t         |           t        |      | _        |j                  ,||j
                  k\  r||j                  z  dk(  rt        |      n
t        |      | _	        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y )Nr   r   )rh   ri   r   	self_attnr    r0   r/   r   r   mlpr   r   r   r4   input_layernormpost_attention_layernorm)rk   r   r   rt   s      rB   ri   zDeepseekV2DecoderLayer.__init__S  s    ,V4 ''3!=!== 5 55:	 &! v& 	  "zz&*<*<&BUBUV(*

F$7$7)
%rA   r}   r   r   r   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   }|S r   )r   r   r   r   )rk   r}   r   r   rhouts          rB   r~   zDeepseekV2DecoderLayer.__call__d  sQ     NN4//2D%@EHHT22156!e
rA   r   )r8   r9   r:   r   r=   ri   r]   r   r   r   r~   r   r   s   @rB   r   r   R  s^    
y 
S 
( $(#	
88
 rxx 
 }	

 

rA   r   c                   j     e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	 xZ
S )DeepseekV2Modelr   c                 p   t         |           |j                  | _        t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        ||       c}| _
        t        j                  |j
                  |j                        | _        y c c}w )Nr   )rh   ri   r   r   	Embeddingr   embed_tokensranger   r   layersr   r4   norm)rk   r   idxrt   s      rB   ri   zDeepseekV2Model.__init__r  s     ++LL):):F<N<NO V556
6 #63/6
 JJv11v7J7JK		
s   'B3r}   r   r   c                    | j                  |      }| j                  }| j                  }|d gt        | j                        z  }t        ||d         }||dz
  k  r#t        j                  j                  ||dz         }t        | j                  |      D ]  \  }} ||||      } |dk7  rVt        j                  j                  ||dz
  |z        }|d   +t        j                  |d   j                  |      |d   _        |dkD  r/t        j                  j                  |      d |j                  d    }| j                  |      S )Nr   r
   )r   rv   )r   pipeline_rankpipeline_sizelenpipeline_layersr   r]   r   	recv_likezipsenddependsr   
all_gatherr|   r   )	rk   r}   r   r   r  r  r   lcs	            rB   r~   zDeepseekV2Model.__call__|  s6   
 a ****=FS!5!566E$Qa1 =1,,((]Q->@A,,e4DAq!T#A 5 A##A(9]'JKARy$!#E"INNA!>b	 1))!,\qwwqz:Ayy|rA   r   r   r   s   @rB   r   r   q  sC    Ly L  $88 } 
	rA   r   c                        e Zd Zdef fdZ	 d
dej                  dee   fdZ	d Z
d
deej                  j                     fdZed	        Z xZS )Modelr   c                     t         |           || _        |j                  | _        t	        |      | _        t        j                  |j                  |j                  d      | _
        y r   )rh   ri   argsr   r   modelr   r   r   r   lm_headr   s     rB   ri   zModel.__init__  sN    	 ++$V,
yy!3!3V5F5FUSrA   inputsr   c                 H    | j                  ||      }| j                  |      S r   )r  r  )rk   r  r   r   s       rB   r~   zModel.__call__  s#    
 jj'||C  rA   c                 j   t        | j                  j                        D ]  }d| }dD ]  \  }}dD ]u  }| d| d| |v st        | j                  j                        D cg c]  }|j	                  | d| d| d|         }}t        j                  |      || d| d| <   w   |S c c}w )Nzmodel.layers.))w1r   )w2r   )w3r   )r   scalesbiasesz.mlp.experts.0..z.mlp.experts.z.mlp.switch_mlp.)r   r  r   r    popr]   stack)	rk   weightsr
  prefixnmr   eto_joins	            rB   sanitizezModel.sanitize  s    tyy223A$QC(FU17A 1QC8GC &+499+E+E%F#%F $KK6(-s!A3as(KL%F   # GIhhwFW6(*:1#Qqc BC 8 V 4 #s   ##B0
r   c                 &   |xs t         j                  j                         }|j                         }| j                  j
                  D ]  }|j                  j                  2t        |j                  j                  d|      |j                  _	        n1t        |j                  j                  d|      |j                  _
        t        |j                  j                  d|      |j                  _        t        |j                  j                  d|      |j                  _        |j                  xj                  |z  c_        t        |j                  t               rt        |j                  j"                  d|      |j                  _        t        |j                  j$                  d|      |j                  _        t        |j                  j&                  d|      |j                  _        ||j                  _        t+        |j                  j,                  j"                  d|       t+        |j                  j,                  j$                  d|       t+        |j                  j,                  j&                  d|       t+        |j                  j.                  j"                  d|       t+        |j                  j.                  j$                  d|       t+        |j                  j.                  j&                  d|        y )Nzall-to-shardedr   zsharded-to-all)r]   r   initsizer  r   r   r$   r   r   r   r   r   r   
isinstancer   r   r   r   r   r   r   r   r   )rk   r   Nlayers       rB   shardzModel.shard  sI   .,,.JJLZZ&&E**2)5OO**,<E*& ,8OO,,.>e,( )5))+;5)EOO% &2&&(8&EOO" OO%%!+% %))]3&2II'')9'		# '3II'')9'		# %1II%%'7u%		! ,1		(II,,668HPU II,,668HPU II,,446Fe II((224DE II((224DE II((002B%c 'rA   c                 .    | j                   j                  S r   )r  r  )rk   s    rB   r   zModel.layers  s    zz)))rA   r   )r8   r9   r:   r   ri   r]   r   r   r   r~   r#  r   Groupr*  propertyr   r   r   s   @rB   r  r    sj    Ty T  $!! }!68BNN$8$89 6p * *rA   r  )r   r1   )r
   r
   )(rE   dataclassesr   typingr   r   r   mlx.corecorer]   mlx.nnr   mlx.nn.layers.distributedr   r   r	   activationsr   rJ   r   r   r   pipeliner   switch_layersr   r   rK   rU   rY   rd   Modulerf   r   r   r   r   r   r   r  r@   rA   rB   <module>r8     s     ! & &   P P  T T # $ ! ! !> =A AE	+0&,
BII ,
^t#")) t#nBII (!bii !HBII DRYY >*mRYY *ZW*BII W*rA   