
    (HJj9O                         d dl Z d dlmZ d dlmZ d dlmZmZmZ d dl	m
Z d dlmZ d dlmZmZmZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZ e G d de             Z  G d dejB                        Z" G d dejB                        Z#ejH                  d        Z% G d dejB                        Z& G d dejB                        Z' G d dejB                        Z( G d deejB                        Z) G d dejB                        Z*y)    N)	dataclass)partial)AnyDictOptional)shard_inplaceshard_linearsum_gradients   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)MultiLinear)PipelineMixin)initialize_rope)	SwitchGLUc                      e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZee   ed<   dZee   ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   d Zeed!<   d"Zeed#<   d$Zeed%<   d$Zeed&<   d$Zeed'<   d$Zeed(<   d)Z eed*<   d+Z!eed,<   d-Z"eed.<   d/Z#eed0<   dZ$e%ed1<   d2Z&eed3<   y)4	ModelArgsdeepseek_v3
model_typei  
vocab_sizei   hidden_sizei +  intermediate_sizei  moe_intermediate_size   num_hidden_layers    num_attention_headsnum_key_value_headsNn_shared_expertsn_routed_experts      ?routed_scaling_factori   kv_lora_ranki   q_lora_rank@   qk_rope_head_dim   
v_head_dimqk_nope_head_dimnoaux_tctopk_methodsigmoidscoring_funcTnorm_topk_probr   n_group
topk_groupnum_experts_per_tokmoe_layer_freqr   first_k_dense_replacei   max_position_embeddingsư>rms_norm_epsg     @
rope_thetarope_scalingFattention_bias)'__name__
__module____qualname__r   str__annotations__r   intr   r   r   r   r   r    r!   r   r"   r$   floatr%   r&   r(   r*   r+   r-   r/   r0   boolr1   r2   r3   r4   r5   r6   r8   r9   r:   r   r;        c/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/deepseek_v3.pyr   r      s9   #J#JK"s"!%3%s!!!!&*hsm*&*hsm*#&5&L#KcJc!K!!L#!NDGSJ  NC!"3"#'S'L%JL$ ND rE   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	DeepseekV3Attentionconfigc                 v   t         |           || _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        |j                  | _
        |j                  | _        |j                  | _        |j                  |j                  z   | _        | j                  dz  | _        | j                  ?t        j                   | j                  | j
                  | j                  z  d      | _        nt        j                   | j                  | j                  |j$                        | _        t        j(                  | j                  d      | _        t        j                   | j                  | j
                  | j                  z  d      | _        t        j                   | j                  | j                  | j                  z   |j$                        | _        t        j(                  | j                  d      | _        t3        | j                  | j                  | j
                        | _        t3        | j                  | j                  | j
                        | _        t        j                   | j
                  | j                  z  | j                  |j$                        | _        | j                  j:                  {| j                  j:                  j=                  dd      }|rS| j                  j:                  d   }|d	kD  r5d
|z  t?        j@                  |      z  dz   }| j                  |z  |z  | _        tC        | j                  | j                  d| j                  | j                  j:                        | _"        y )Ng      Fbiasr7   epsmscale_all_dimr   factorr   g?r#   T)dimsbasetraditionalr6   scaling_config)#super__init__rI   r   r   	num_headsr6   r9   r&   r(   r%   r*   r+   
q_head_dimscalennLinearq_projr;   q_a_projRMSNormq_a_layernormq_b_projkv_a_proj_with_mqakv_a_layernormr   embed_qunembed_outo_projr:   getmathlogr   rope)selfrI   rO   scaling_factors	__class__s        rF   rV   zDeepseekV3Attention.__init__6   s   !--33'-'E'E$ ++!-- & 7 7"// ++ & 7 7 11F4K4KK__d*
#))  $..4??"BDK II  $"2"29N9NDM "$D,<,<$!GDII  $..4??"BDM #%)) 5 55&&#

 !jj):):E"!!4#4#4dnn
 't
 iiNNT__,&&
 ;;##/![[5599:JANN!%!9!9(!C!A%n,txx/GG#MA!%a!!3DJ#&&$($@$@;;33
	rE   xmaskcachereturnc           	      |   |j                   \  }}}| j                  | j                  |      }n/| j                  | j	                  | j                  |                  }|j                  ||| j                  | j                        j                  dddd      }t        j                  || j                  gd      \  }}	| j                  |      }
t        j                  |
| j                  gd      \  }
}|j                  ||d| j                        j                  dddd      }| j!                  |
      }||j"                  nd}| j%                  |	|      }	| j%                  ||      }t        j&                  |d      }||j)                  ||      \  }}|	| j*                  z  |j-                  dd      z  }|\t        j.                  ||t        j0                  t        j2                  |j4                        j6                  |j4                              }|dk(  r| j9                  |      }|x}}n$| j9                  |d	      }| j;                  |      }t=        ||||| j*                  |
      }|dk(  r| j;                  |      }|j                  dddd      j                  ||d      }| j?                  |      S )Nr      r      axisF)	transpose)rp   rY   ro   ) shaper&   r\   r`   r_   r]   reshaperW   rX   ry   mxsplitr+   ra   r%   r(   rb   offsetri   expand_dimsupdate_and_fetchrY   swapaxeswherearrayfinfodtypeminrc   rd   r   re   )rj   rn   ro   rp   BLDqq_nopeq_pecompressed_kvk_pe	kv_latentr~   	pe_scoreskvoutputs                     rF   __call__zDeepseekV3Attention.__call__v   sk    ''1a#AAd00q1ABCAIIaDNNDOO<FFq!QPQRxxD$9$9#:D//2 hh}t7H7H6IPRSt||Aq!T%:%:;EEaAqQ''6	!&!2yyv&yyv&NN915	#44YEOItDJJ&$--B*??	)//266	HI 6\\&)FAY%8A  +A-AqTZZi
 6%%f-F!!!Q1-55aB?{{6""rE   NNr<   r=   r>   r   rV   r|   r   r   r   r   __classcell__rm   s   @rF   rH   rH   5   sX    >
y >
F $(#	3#883# rxx 3# }	3#
 
3#rE   rH   c                   6     e Zd Z	 ddededef fdZd Z xZS )DeepseekV3MLPrI   r   r   c                    t         |           || _        ||j                  n|| _        ||j                  n|| _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _	        y NFrK   )
rU   rV   rI   r   r   rZ   r[   	gate_projup_proj	down_proj)rj   rI   r   r   rm   s       rF   rV   zDeepseekV3MLP.__init__   s     	1<1D6--+(9(AF$$GX 	 4#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXrE   c                 x    | j                  t        | j                  |      | j                  |                  }|S N)r   r   r   r   )rj   rn   r   s      rF   r   zDeepseekV3MLP.__call__   s.    NN6$..*;T\\!_#MN	rE   r   )r<   r=   r>   r   rA   rV   r   r   r   s   @rF   r   r      s,    SWYY.1YMPYrE   r   c                    t        j                  | j                  t         j                              }|}||z   }|dkD  rt        j                  |d|df      }t        j
                  |dd      j                  dd      }	||z
  }
t        j                  |	|
dz
  d	      d
d |
d d f   }t        j                  |t        j                  |      t        j                  d      d      }t        j                  |dd      }|}
t        j                  | |
dz
  d	      d
d |
f   }t        j                  ||d      }|dkD  r|r|j                  dd      }||z  }||z  }||fS )Nr   ru   )rw   rz   rs   rv   T)rw   keepdimsrx   )kthrw   .g        )r|   r.   astypefloat32	unflattentopksumargpartitionput_along_axisstop_gradientr   flattentake_along_axis)gatese_score_correction_biastop_kr1   r2   r$   r0   scoresorig_scoresgroup_scoresr   	group_idxindsdenominators                 rF   group_expert_selectr      sU    ZZRZZ01FK--F{f2gr]Cwwvqr266B6Nj OOLa!e"Ec2A2qjQ	""B$$Y/#R
 FB+A??F7AB7RaR@DT;Fqy^jjb4j8+%++F<rE   c                   *     e Zd Zdef fdZd Z xZS )MoEGaterI   c                    t         |           || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  | j                  |j                  f      | _        t        j                  | j                  f      | _        |j                  dk(  sJ d       y )Nr,   zUnsupported topk method.)rU   rV   rI   r3   r   r0   r"   r$   r1   r2   r|   zerosr   weightr   r-   rj   rI   rm   s     rF   rV   zMoEGate.__init__   s    //
$33 & 7 7%+%A%A"~~ ++hh 5 5v7I7IJK')xx1F1F0H'I$!!Z/K1KK/rE   c           	          t        || j                  j                  z  | j                  | j                  | j
                  | j                  | j                  | j                        S r   )	r   r   Tr   r   r1   r2   r$   r0   )rj   rn   s     rF   r   zMoEGate.__call__   sO    "((JJLLOO&&
 	
rE   r<   r=   r>   r   rV   r   r   r   s   @rF   r   r      s    Ly L	
rE   r   c                   *     e Zd Zdef fdZd Z xZS )DeepseekV3MoErI   c                 N   t         |           || _        |j                  | _        t	        |j
                  |j                  |j                        | _        t        |      | _
        |j                  +|j                  |j                  z  }t        ||      | _        d | _        y )N)rI   r   )rU   rV   rI   r3   r   r   r   r"   
switch_mlpr   gater!   r   shared_expertssharding_group)rj   rI   r   rm   s      rF   rV   zDeepseekV3MoE.__init__   s    #)#=#= #((##
 FO	"". & < <v?V?V V"/1B#D #rE   c                    | j                    t        | j                         |      }| j                  |      \  }}| j                  ||      }||d   z  j	                  d      j                  |j                        }| j                  j                  || j                  |      z   }| j                   +t        j                  j                  || j                         }|S )N).Nrx   rv   group)r   r
   r   r   r   r   r   rI   r!   r   r|   distributedall_sum)rj   rn   r   r   ys        rF   r   zDeepseekV3MoE.__call__  s    *2d11215Ayy|fOOAt$	""''R'077@;;''3D''**A*&&q0C0C&DArE   r   r   s   @rF   r   r      s    #y #&rE   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
DeepseekV3DecoderLayerrI   	layer_idxc                    t         |           t        |      | _        |j                  ,||j
                  k\  r||j                  z  dk(  rt        |      n
t        |      | _	        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y )Nr   rM   )rU   rV   rH   	self_attnr"   r5   r4   r   r   mlprZ   r^   r   r8   input_layernormpost_attention_layernorm)rj   rI   r   rm   s      rF   rV   zDeepseekV3DecoderLayer.__init__"  s    ,V4 ''3!=!== 5 55:	 &! v& 	  "zz&*<*<&BUBUV(*

F$7$7)
%rE   rn   ro   rp   rq   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   S r   )r   r   r   r   )rj   rn   ro   rp   rhs         rF   r   zDeepseekV3DecoderLayer.__call__3  sL     NN4//2D%@EHHT221561urE   r   )r<   r=   r>   r   rA   rV   r|   r   r   r   r   r   r   s   @rF   r   r   !  s^    
y 
S 
( $(#		88	 rxx 	 }		
 
	rE   r   c                   j     e Zd Zdef fdZ	 ddej                  dee   dej                  fdZ	 xZ
S )DeepseekV3ModelrI   c                 p   t         |           |j                  | _        t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        ||       c}| _
        t        j                  |j
                  |j                        | _        y c c}w )NrM   )rU   rV   r   rZ   	Embeddingr   embed_tokensranger   r   layersr^   r8   norm)rj   rI   idxrm   s      rF   rV   zDeepseekV3Model.__init__@  s     ++LL):):F<N<NO V556
6 #63/6
 JJv11v7J7JK		
s   'B3rn   rp   rq   c                    | j                  |      }| j                  }| j                  }|d gt        | j                        z  }t        ||d   d      }||dz
  k  r#t        j                  j                  ||dz         }t        | j                  |      D ]  \  }} ||||      } |dk7  rVt        j                  j                  ||dz
  |z        }|d   +t        j                  |d   j                  |      |d   _        |dkD  r/t        j                  j                  |      d |j                  d    }| j                  |      S )Nr   T)return_arrayr   )rp   ru   )r   pipeline_rankpipeline_sizelenpipeline_layersr   r|   r   	recv_likezipsenddependskeys
all_gatherrz   r   )	rj   rn   rp   r   r   r   ro   lcs	            rF   r   zDeepseekV3Model.__call__J  s8   
 a ****=FS!5!566E$QatD =1,,((]Q->@A,,e4DAq!T#A 5 A##A(9]'JKARy$!#E"INNA!>b	 1))!,\qwwqz:Ayy|rE   r   r   r   s   @rF   r   r   ?  sC    Ly L  $88 } 
	rE   r   c                        e Zd Zdef fdZ	 ddej                  dee   fdZ	d Z
ddeej                  j                     fdZed	        Zed
        Z xZS )ModelrI   c                     t         |           || _        |j                  | _        t	        |      | _        t        j                  |j                  |j                  d      | _
        y r   )rU   rV   argsr   r   modelrZ   r[   r   r   lm_headr   s     rF   rV   zModel.__init__m  sN    	 ++$V,
yy!3!3V5F5FUSrE   inputsrp   c                 H    | j                  ||      }| j                  |      S r   )r   r   )rj   r   rp   outs       rF   r   zModel.__call__t  s#    
 jj'||C  rE   c                    d }i }|j                         D ]  \  }}|j                  d      rY|j                  dd      }||dz      j                  t        j
                        ||dz   <   ||dz      }|||dz   <   d|z  ||d	z   <   p|j                  d      r|j                  d      r|||<    |}i }|j                         D ];  \  }}d
|v r(|}|j                  dd      }	||	   }
 ||
|      }
|
||	<   2||vs7|||<   = |}t        | j                  j                        D ]  }d| }dD ]  \  }}dD ]u  }| d| d| |v st        | j                  j                        D cg c]  }|j                  | d| d| d|         }}t	        j                  |      || d| d| <   w  d| d}| d|v s| j                  j                  |   j                  j                  }| d|v }|j                  | d      }| j                  j                   | j                  j"                  z   }|r| j                  j$                  }|j                  | d      }|j                  | d      }|j&                  d   dz  |z  }||j&                  d   z  }t	        j(                  |||||      }| j                  j*                  }|j-                  ||d      }t	        j.                  |d d d | j                  j                   d d f   j1                  dd            }	t	        j.                  |d d | j                  j                   d d d f         }|rXt	        j2                  |	      \  }	}}t	        j2                  |||      \  }}}||| d<   ||| d<   ||| d<   ||| d<   |	|| d<   ||| d <    |j                         D ci c]  \  }}|j5                  d!      sd"|vr|| c}}S c c}w c c}}w )#Nc                    t         j                  }t        j                  | t         j                        } d}| j                  \  }}| |z  }| |z  }t        j                  | d|fd|ff      } | j                  ||z   |z  |||z   |z  |f      } | |d d d d d d f   z  j                  ||z   ||z         } | d |d |f   j                  |      S )N)r   r)   r   )r|   bfloat16from_fp8rz   padr{   r   )r   	scale_invr   bsmn
pad_bottompad_sides           rF   dequantzModel.sanitize.<locals>.dequant}  s    KKE[[r{{;FB<<DAq"JbyHVVFa_q(m$DEF^^j.R'a(lr-A2FF yD!T)9::CCJHF "1"bqb&>((//rE   weight_shape weight_packedr   weight_scalescalesibiasesweight_scale_inv
_scale_invzmodel.layers.))w1r   )w2r   )w3r   )r   r
  r  z.mlp.experts.0..z.mlp.experts.z.mlp.switch_mlp.z
.self_attnz.kv_b_proj.weightz.kv_b_proj.scalesz.kv_b_proj.biasesru   r   )bits
group_sizerx   z.embed_q.scalesz.unembed_out.scalesz.embed_q.biasesz.unembed_out.biasesz.embed_q.weightz.unembed_out.weightzmodel.layers.61zrotary_emb.inv_freq)itemsendswithreplaceviewr|   uint32r   r   r   r"   popstackr   r   r   rc   r+   r*   r%   rz   
dequantizer   r{   
contiguousr   quantize
startswith)rj   weightsr  new_weightsr   r   rR   rl   r   wkr   r   prefixr  r  eto_joinlayer	quantizedhead_dimrQ   r
  r  r  r  rW   wv	wk_scales	wk_biases	wv_scales	wv_biasess                                  rF   sanitizezModel.sanitize|  s   	0" MMODAqzz.)yy4/6to7M/N/S/SII0D8O, D>12/0D8O,/1AvD8O,jj0AJJ4O!"A $  MMODAq!Q&	YY|R0  3"(B+%!"A $  tyy223A$QC(FU17A 1QC8GC &+499+E+E%F#%F $KK6(-s!A3as(KL%F   # GIhhwFW6(*:1#Qqc BC 8 V %QCz2F*+w6

))!,66>>%h&78GC	KK6(*; <=9955		8L8LL9911D$[[F83D)EFF$[[F83D)EFFGGBK",5D!%b)9!9J66A !II99	IIi26]]a5499555q89BB2rJ ]]1Q		(B(B(Da%G#HI/1{{*0,B	9 02{{*0,B	9 ;DGvho67>GGvh&9:;:CGvho67>GGvh&9:;686(/23:<6("567[ 4d  
'1<< 127LTU7U qD'
 	
W#V
s   (#O>
"Pr   c                    |xs t         j                  j                         }|j                         }|j	                         }| j
                  j                  D ]  }|j                  j                  2t        |j                  j                  d|      |j                  _
        n1t        |j                  j                  d|      |j                  _        |j                  xj                  |z  c_        |j                  j                  }||z  |z   fd}|j                  j                  j                  |       |j                  j                  j                  |       t        |j                  j                   d|      |j                  _        t#        |j$                  t&              rt        |j$                  j(                  d|      |j$                  _        t        |j$                  j*                  d|      |j$                  _        t        |j$                  j,                  d|      |j$                  _        ||j$                  _        t1        |j$                  j2                  j(                  d|       t1        |j$                  j2                  j*                  d|       t1        |j$                  j2                  j,                  d|       t1        |j$                  j4                  j(                  d|       t1        |j$                  j4                  j*                  d|       t1        |j$                  j4                  j,                  d|        y )Nzall-to-shardedr   c                     |  S r   rD   )wehshs    rF   shard_headsz Model.shard.<locals>.shard_heads  s    BxrE   zsharded-to-all)r|   r   initsizerankr   r   r   r&   r	   r\   r`   rW   rc   applyrd   re   
isinstancer   r   r   r   r   r   r   r   r   )	rj   r   Nr6  r%  rW   r3  r1  r2  s	          @@rF   shardzModel.shard  s   .,,.JJLzz|ZZ&&E**2)5OO**,<E*& ,8OO,,.>e,( OO%%!+%11I	!BiB  OO##))+6OO''--k:%1&&(8&EOO"
 %))]3&2II'')9'		# '3II'')9'		# %1II%%'7u%		! ,1		(II,,668HPU II,,668HPU II,,446Fe II((224DE II((224DE II((002B%q 'rE   c                 .    | j                   j                  S r   )r   r   )rj   s    rF   r   zModel.layers   s    zz)))rE   c                     d }|S )Nc                 
    d| vS )Nr   rD   )r   s    rF   	predicatez'Model.cast_predicate.<locals>.predicate&  s    ,A55rE   rD   )rj   r>  s     rF   cast_predicatezModel.cast_predicate$  s    	6 rE   r   )r<   r=   r>   r   rV   r|   r   r   r   r   r-  r   Groupr:  propertyr   r?  r   r   s   @rF   r   r   l  s    Ty T  $!! }!b
H>8BNN$8$89 >@ * *  rE   r   )+rg   dataclassesr   	functoolsr   typingr   r   r   mlx.corecorer|   mlx.nnrZ   mlx.nn.layers.distributedr   r	   r
   activationsr   rR   r   r   r   mlar   pipeliner   
rope_utilsr   switch_layersr   r   ModulerH   r   compiler   r   r   r   r   r   rD   rE   rF   <module>rP     s     !  & &   P P  T T  # ' $ ! ! !@t#")) t#nBII (  D
bii 
4!BII !HRYY <*mRYY *Z}BII }rE   