
    (HJjF^                     @   d dl Z d dlmZ d dlmZmZmZ d dlmZ	 d dl
mZ d dlmZmZmZ ddlmZ ddlmZmZmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ e G d de             Z G d dej@                        Z! G d dej@                        Z" G d dej@                        Z#e	jH                  d        Z% G d dej@                        Z& G d dej@                        Z' G d dej@                        Z( G d dej@                        Z) G d dej@                        Z*y)    N)	dataclass)AnyDictOptional)shard_inplaceshard_linearsum_gradients   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)	CacheListKVCache)MultiLinear)initialize_rope)	SwitchGLUc                      e Zd ZU dZeed<   dZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZee   ed<   dZee   ed<   dZeed<   dZeed<   dZeed<   d	Zeed<   dZeed <   dZeed!<   d"Zeed#<   d$Zeed%<   d&Zeed'<   d(Zeed)<   d(Z eed*<   d(Z!eed+<   d(Z"eed,<   d-Z#eed.<   dZ$eed/<   d0Z%eed1<   d2Z&eed3<   dZ'e(ed4<   d5Z)eed6<   y)7	ModelArgsdeepseek_v32
model_typei  
vocab_sizei   hidden_size   index_head_dim@   index_n_headsi   
index_topki +  intermediate_sizei  moe_intermediate_size   num_hidden_layers    num_attention_headsnum_key_value_headsNn_shared_expertsn_routed_experts      ?routed_scaling_factori   kv_lora_ranki   q_lora_rankqk_rope_head_dim
v_head_dimqk_nope_head_dimnoaux_tctopk_methodsigmoidscoring_funcTnorm_topk_probr
   n_group
topk_groupnum_experts_per_tokmoe_layer_freqr   first_k_dense_replacemax_position_embeddingsư>rms_norm_epsg     @
rope_thetarope_scalingFattention_bias)*__name__
__module____qualname__r   str__annotations__r   intr   r   r   r   r   r    r"   r$   r%   r&   r   r'   r)   floatr*   r+   r,   r-   r.   r0   r2   r3   boolr4   r5   r6   r7   r8   r9   r;   r<   r=   r   r>        d/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/deepseek_v32.pyr   r      sW   $J$JKNCM3J"s"!%3%s!!!!&*hsm*&*hsm*#&5&L#KcJc!K!!L#!NDGSJ  NC!"3"#'S'L%JL$ ND rH   r   c            
            e Zd Zdef fdZ	 ddej                  dej                  deej                     dee   fdZ	 xZ
S )	Indexerargsc                    t         |           |j                  | _        |j                  | _        |j                  | _        |j                  | _	        |j                  | _
        |j                  | _        t        j                  | j                  | j
                  | j                  z  d      | _        t        j                  | j                  | j                  d      | _        t        j                   | j                        | _        t        j                  | j                  | j
                  d      | _        | j                  dz  | _        t)        |j                  |j*                  d|j,                  |j.                        | _        y )NFbias      Tdimsbasetraditionalr9   scaling_config)super__init__r   dimr   n_headsr   head_dimr,   rope_head_dimr   r+   nnLinearwq_bwk	LayerNormk_normweights_projsoftmax_scaler   r<   r9   r=   rope)selfrL   	__class__s     rI   rW   zIndexer.__init__8   s   ##))++!22//++IIdllT]]:
	 ))DHHdmm%@ll4==1IIdhh5I!]]D0#&&$($@$@,,
	rH   xqrmaskcachec           	         |j                   \  }}}| j                  |      }|j                  ||| j                  | j                        j                  dd      }| j                  |      }	| j                  |	      }	t        j                  |	|d|| j                  f      }	||j                  nd}
| j                  ||
      }| j                  |	|
      }	|,|j                  |	t        j                  |d|dg            \  }	}|	j                   d   | j                  k  ry ||	j                  dd      z  }t        j                  |d      }| j                  |      | j                  dz  | j                   z  z  }|j                  dd      d   }||z  }|j#                  dd	
      }|!t        j$                  ||t'        d             }t        j(                  || j                   d      d| j                   d f   S )Nr
      r   )offsetrP   .NTaxiskeepdimsinfkthrr   .)shaper^   reshaperY   rZ   swapaxesr_   ra   mxrm   rd   update_and_fetchzerosr   maximumrb   rc   sumwhererE   argpartition)re   rg   rh   ri   rj   bs_qkrm   scoresweightss                rI   __call__zIndexer.__call__O   s    ''1aIIbMIIaDLL$--8AA!QGGGAJKKNJJq1aDMM23!&!2IIaI'IIaI'))!RXXq!Ql-CDDAq771:(QZZB''FA&##A&$,,*<t?Q?Q*QR""2r*95'!T2XXdFU5\M:FvDOO+;"E$//!##
 	
rH   Nr?   r@   rA   r   rW   rz   arrayr   r   r   __classcell__rf   s   @rI   rK   rK   7   sT    
Y 
8  $"
88"
 HH"
 rxx 	"

 }"
rH   rK   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	DeepseekV32Attentionconfigc                     t         |           || _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        |j                  | _
        |j                  | _        |j                  | _        |j                  |j                  z   | _        | j                  dz  | _        t        j                   | j                  | j                  |j"                        | _        t        j&                  | j                  d      | _        t        j                   | j                  | j
                  | j                  z  d      | _        t        j                   | j                  | j                  | j                  z   |j"                        | _        t        j&                  | j                  d      | _        t1        | j                  | j                  | j
                        | _        t1        | j                  | j                  | j
                        | _        t        j                   | j
                  | j                  z  | j                  |j"                        | _        | j                  j8                  {| j                  j8                  j;                  dd      }|rS| j                  j8                  d   }|d	kD  r5d
|z  t=        j>                  |      z  dz   }| j                  |z  |z  | _        tA        |      | _!        tE        | j                  | j                  d| j                  | j                  j8                        | _#        y )NrP   rN   r:   epsFmscale_all_dimr   factorr
   g?r(   TrQ   )$rV   rW   r   r   r$   	num_headsr9   r<   r+   r,   r*   r-   r.   
q_head_dimscaler\   r]   r>   q_a_projRMSNormq_a_layernormq_b_projkv_a_proj_with_mqakv_a_layernormr   embed_qunembed_outo_projr=   getmathlogrK   indexerr   rd   )re   r   r   scaling_factorr   rf   s        rI   rW   zDeepseekV32Attention.__init__u   s   !--33'-'E'E$ ++!-- & 7 7"// ++ & 7 7 11F4K4KK__d*
		d..V5J5J
  ZZ(8(8dC		dnnt>U
 #%)) 5 55&&#

 !jj):):E"!!4#4#4dnn
 't
 iiNNT__,&&
 ;;##/![[5599:JANN!%!9!9(!C!A%n,txx/GG#MA!%a!!3DJv#&&$($@$@;;33
	rH   rg   ri   rj   returnc           	         |j                   \  }}}| j                  | j                  |            }| j                  |      }|j	                  ||| j
                  | j                        j                  dddd      }t        j                  || j                  gd      \  }	}
| j                  |      }t        j                  || j                  gd      \  }}|j	                  ||d| j                        j                  dddd      }| j                  |      }||d   j                  nd}| j!                  |
|      }
| j!                  ||      }t        j"                  |d      }||d   j%                  ||      \  }}nd gdz  }| j'                  ||||d         }|E|dk(  r|d d d d dd d d f   }t        j(                  |t        j*                  ||j                   d d |j                   d   fz         d      }t        j(                  |t        j*                  ||j                   d d |j                   d   fz         d      }|t        j(                  ||d      }nt-        |j                         }|j                   d   |d<   t        j.                  |t        j0                        }t        j2                  ||t        j4                  d	      d      }|||z  }|}|L|d   Gt        j6                  |d   j8                  |d   j8                  |d   j:                  f      |d   _        |
| j<                  z  |j?                  dd
      z  }|\t        j@                  ||t        j4                  t        jB                  |jD                        jF                  |jD                              }|dk(  r| jI                  |	      }	|x}}n$| jI                  |d      }| jK                  |      }tM        |	|||| j<                  |      }|dk(  r| jK                  |      }|j                  dddd      j	                  ||d      }| jO                  |      S )Nr   rl   r
      rn   rr   )rj   dtypeTro   F)	transpose)rj   r   ri   )(rw   r   r   r   rx   r   r   r   rz   splitr.   r   r*   r,   r   rm   rd   expand_dimsr{   r   take_along_axisbroadcast_tolistr|   bool_put_along_axisr   dependskeysvaluesr   ry   r   finfor   minr   r   r   r   )re   rg   ri   rj   BLDrh   r   q_nopeq_pecompressed_kvk_pe	kv_latentrm   topk_indicesidxrw   sparse_mask	pe_scoresr   voutputs                          rI   r   zDeepseekV32Attention.__call__   s    ''1aa 01MM"IIaDNNDOO<FFq!QPQRxxD$9$9#:D//2 hh}t7H7H6IPRSt||Aq!T%:%:;EEaAqQ''6	$)$5q1yyv&yyv&NN915	#Ah77	4HOItFQJE||Ar4uQx|@#Av"1aAt#34..OOC3B9??2;N:P)PQ	
 ))OOC3B4::b>:K)KL
 #--dLrJD\//0%OOA.b	 hhuBHH= //rxx~B #"-"4K" q!5JJuQx}}uQx}}eAhoo6VWE!HMDJJ&$--B*??	)//266	HI 6\\&)FAY%8A  +A-AqTZZi
 6%%f-F!!!Q1-55aB?{{6""rH   NNr   r   s   @rI   r   r   t   s]    :
y :
~ $(#	R#88R# rxx R# }	R#
 
R#rH   r   c                   6     e Zd Z	 ddededef fdZd Z xZS )DeepseekV32MLPr   r   r   c                    t         |           || _        ||j                  n|| _        ||j                  n|| _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _	        y NFrN   )
rV   rW   r   r   r   r\   r]   	gate_projup_proj	down_proj)re   r   r   r   rf   s       rI   rW   zDeepseekV32MLP.__init__  s     	1<1D6--+(9(AF$$GX 	 4#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXrH   c                 x    | j                  t        | j                  |      | j                  |                  }|S r   )r   r   r   r   )re   rg   r   s      rI   r   zDeepseekV32MLP.__call__  s.    NN6$..*;T\\!_#MN	rH   r   )r?   r@   rA   r   rD   rW   r   r   r   s   @rI   r   r     s,    SWYY.1YMPYrH   r   c                    t        j                  | j                  t         j                              }|}||z   }|dkD  rt        j                  |d|df      }t        j
                  |dd      j                  dd      }	||z
  }
t        j                  |	|
dz
  d	      d
d |
d d f   }t        j                  |t        j                  |      t        j                  d      d      }t        j                  |dd      }|}
t        j                  | |
dz
  d	      d
d |
f   }t        j                  ||d      }|dkD  r|r|j                  dd      }||z  }||z  }||fS )Nr
   rn   )rr   rw   rl   r   Trq   ro   ru   .g        )rz   r1   astypefloat32	unflattentopkr~   r   r   stop_gradientr   flattenr   )gatese_score_correction_biastop_kr4   r5   r)   r3   r   orig_scoresgroup_scoresr   	group_idxindsdenominators                 rI   group_expert_selectr     sU    ZZRZZ01FK--F{f2gr]Cwwvqr266B6Nj OOLa!e"Ec2A2qjQ	""B$$Y/#R
 FB+A??F7AB7RaR@DT;Fqy^jjb4j8+%++F<rH   c                   *     e Zd Zdef fdZd Z xZS )MoEGater   c                    t         |           || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  | j                  |j                  f      | _        t        j                  | j                  f      | _        |j                  dk(  sJ d       y )Nr/   zUnsupported topk method.)rV   rW   r   r6   r   r3   r'   r)   r4   r5   rz   r|   r   weightr   r0   re   r   rf   s     rI   rW   zMoEGate.__init__>  s    //
$33 & 7 7%+%A%A"~~ ++hh 5 5v7I7IJK')xx1F1F0H'I$!!Z/K1KK/rH   c           	          t        || j                  j                  z  | j                  | j                  | j
                  | j                  | j                  | j                        S r   )	r   r   Tr   r   r4   r5   r)   r3   )re   rg   s     rI   r   zMoEGate.__call__K  sO    "((JJLLOO&&
 	
rH   r?   r@   rA   r   rW   r   r   r   s   @rI   r   r   =  s    Ly L	
rH   r   c                   *     e Zd Zdef fdZd Z xZS )DeepseekV32MoEr   c                 N   t         |           || _        |j                  | _        t	        |j
                  |j                  |j                        | _        t        |      | _
        |j                  +|j                  |j                  z  }t        ||      | _        d | _        y )N)r   r   )rV   rW   r   r6   r   r   r    r'   
switch_mlpr   gater&   r   shared_expertssharding_group)re   r   r   rf   s      rI   rW   zDeepseekV32MoE.__init__X  s    #)#=#= #((##
 FO	"". & < <v?V?V V"01B#D #rH   c                    | j                    t        | j                         |      }| j                  |      \  }}| j                  ||      }||d   z  j	                  d      j                  |j                        }| j                  j                  || j                  |      z   }| j                   +t        j                  j                  || j                         }|S )Nrp   ro   r   group)r   r	   r   r   r~   r   r   r   r&   r   rz   distributedall_sum)re   rg   r   r   ys        rI   r   zDeepseekV32MoE.__call__k  s    *2d11215Ayy|fOOAt$	""''R'077@;;''3D''**A*&&q0C0C&DArH   r   r   s   @rI   r   r   W  s    #y #&rH   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
DeepseekV32DecoderLayerr   	layer_idxc                    t         |           t        |      | _        |j                  ,||j
                  k\  r||j                  z  dk(  rt        |      n
t        |      | _	        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y )Nr   r   )rV   rW   r   	self_attnr'   r8   r7   r   r   mlpr\   r   r   r;   input_layernormpost_attention_layernorm)re   r   r   rf   s      rI   rW   z DeepseekV32DecoderLayer.__init__|  s    -f5 ''3!=!== 5 55:	 6"  ' 	  "zz&*<*<&BUBUV(*

F$7$7)
%rH   rg   ri   rj   r   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   S r   )r   r   r   r   )re   rg   ri   rj   rhs         rI   r   z DeepseekV32DecoderLayer.__call__  sL     NN4//2D%@EHHT221561urH   r   )r?   r@   rA   r   rD   rW   rz   r   r   r   r   r   r   s   @rI   r   r   {  s^    
y 
S 
( $(#		88	 rxx 	 }		
 
	rH   r   c                   p     e Zd Zdef fdZd Z	 ddej                  dee	   dej                  fdZ
 xZS )	DeepseekV32Modelr   c                    t         |           |j                  | _        t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        ||       c}| _
        d| _        t        | j                        | _        | j                  | _        t        j                  |j
                  |j                         | _        d| _        d| _        y c c}w )Nr   r   r
   )rV   rW   r   r\   	Embeddingr   embed_tokensranger"   r   layers	start_idxlenend_idx
num_layersr   r;   normpipeline_rankpipeline_size)re   r   r   rf   s      rI   rW   zDeepseekV32Model.__init__  s     ++LL):):F<N<NO V556
6 $FC06
 4;;',,JJv11v7J7JK	
s   'C3c                 V   |j                         | _        |j                         | _        t	        | j
                        | j                  z  }t	        | j
                        || j                  z  z
  }| j                  |k  r|dz  }| j                  | j                  z
  dz
  |z  | _        | j                  |z   | _        | j
                  d | j                   | _        d g| j                  z  | j
                  d | j                   t	        | j
                        | j                  z
  | _        y )Nr
   )	rankr  sizer  r	  r  r  r
  r  )re   r   layers_per_rankextras       rI   pipelinezDeepseekV32Model.pipeline  s     #ZZ\"ZZ\dkk*d.@.@@DKK ?T5G5G#GG%q O,,t/A/AAAEX~~7kk.DLL1)-(?$dnn%dkk*T^^;rH   rg   rj   r   c                    | j                  |      }| j                  }| j                  }|d g| j                  z  }t	        ||d   r|d   d   nd d      }||dz
  k  r#t
        j                  j                  ||dz         }t        | j                        D ])  } | j                  | j                  |z      ||||         }+ |dk7  r\t
        j                  j                  ||dz
  |z        }|d   1t        j                  |d   d   j                  |      |d   d   _        |dkD  r/t
        j                  j                  |      d |j                  d    }| j!                  |      S )Nr   T)return_arrayr
   rn   )r  r  r  r  r   rz   r   	recv_liker  r  r  sendr   r   
all_gatherrw   r  )re   rg   rj   r   r  r  ri   is           rI   r   zDeepseekV32Model.__call__  s[   
 a ****=FT__,E$eAhuQx{Dt
 =1,,((]Q->@At'A/DNNQ./4qBA ( A##A(9]'JKARy$$&JJuRy|/@/@!$Db	!! 1))!,\qwwqz:Ayy|rH   r   )r?   r@   rA   r   rW   r  rz   r   r   r   r   r   r   s   @rI   r  r    sF    y  <$  $"88" }" 
	"rH   r  c                        e Zd Zdef fdZ	 ddej                  dee   fdZ	d Z
ddeej                  j                     fdZed	        Zed
        Zd Z xZS )Modelr   c                     t         |           || _        |j                  | _        t	        |      | _        t        j                  |j                  |j                  d      | _
        y r   )rV   rW   rL   r   r  modelr\   r]   r   r   lm_headr   s     rI   rW   zModel.__init__  sN    	 ++%f-
yy!3!3V5F5FUSrH   inputsrj   c                 H    | j                  ||      }| j                  |      S r   )r  r  )re   r   rj   outs       rI   r   zModel.__call__  s#    
 jj'||C  rH   c                    | j                   j                  }i }|j                         D ]C  \  }}|j                  d      }t	        |      dk\  r|d   dk(  rt        |d         |k\  r?|||<   E |}d }i }|j                         D ];  \  }}d|v r(|}|j                  dd	      }	||	   }
 ||
|      }
|
||	<   2||vs7|||<   = |}t        | j                   j                        D ]  }d
| }dD ]  \  }}dD ]u  }| d| d| |v st        | j                   j                        D cg c]  }|j                  | d| d| d|         }}t        j                  |      || d| d| <   w  d
| d}| d|v s| j                  j                  |   j                  j                  }| d|v }|j                  | d      }| j                   j                   | j                   j"                  z   }|r| j                   j$                  }|j                  | d      }|j                  | d      }|j&                  d   dz  |z  }||j&                  d   z  }t        j(                  |||||      }| j                   j*                  }|j-                  ||d      }t        j.                  |d d d | j                   j                   d d f   j1                  dd            }	t        j.                  |d d | j                   j                   d d d f         }|rXt        j2                  |	      \  }	}}t        j2                  |||      \  }}}||| d<   ||| d<   ||| d<   ||| d<   |	|| d<   ||| d<    |S c c}w )N.r   r
   r  rl   c                    t         j                  }t        j                  | t         j                        } d}| j                  \  }}| |z  }| |z  }t        j                  | d|fd|ff      } | j                  ||z   |z  |||z   |z  |f      } | |d d d d d d f   z  j                  ||z   ||z         } | d |d |f   j                  |      S )Nr   r   r   )rz   bfloat16from_fp8rw   padrx   r   )r   	scale_invr   bsmn
pad_bottompad_sides           rI   dequantzModel.sanitize.<locals>.dequant  s    KKE[[r{{;FB<<DAq"JbyHVVFa_q(m$DEF^^j.R'a(lr-A2FF yD!T)9::CCJHF "1"bqb&>((//rH   weight_scale_inv
_scale_inv zmodel.layers.))w1r   )w2r   )w3r   )r   scalesbiasesz.mlp.experts.0.z.mlp.experts.z.mlp.switch_mlp.z
.self_attnz.kv_b_proj.weightz.kv_b_proj.scalesz.kv_b_proj.biasesrn   r#   )bits
group_sizero   z.embed_q.scalesz.unembed_out.scalesz.embed_q.biasesz.unembed_out.biasesz.embed_q.weightz.unembed_out.weight)rL   r"   itemsr   r	  rD   replacer  r'   poprz   stackr  r  r   r   r.   r-   r*   rw   
dequantizer$   rx   
contiguousry   quantize)re   r   	mpt_layernew_weightsr   r   partsr/  r)  r_   r   lprefixr,  r+  eto_joinlayer	quantizedrZ   rR   r6  r7  r8  r9  r   wv	wk_scales	wk_biases	wv_scales	wv_biasess                                  rI   sanitizezModel.sanitize  s   II//	MMODAqGGCLE5zQ58x#7CaMY<VKN	 $
 	0" MMODAq!Q&	YY|R0  3"(B+%!"A $  tyy223A$QC(FU17A 1QC8GC &+499+E+E%F#%F $KK6(-s!A3as(KL%F   # GIhhwFW6(*:1#Qqc BC 8 V %QCz2F*+w6

))!,66>>%h&78GC	KK6(*; <=9955		8L8LL9911D$[[F83D)EFF$[[F83D)EFFGGBK",5D!%b)9!9J66A !II99	IIi26]]a5499555q89BB2rJ ]]1Q		(B(B(Da%G#HI/1{{*0,B	9 02{{*0,B	9 ;DGvho67>GGvh&9:;:CGvho67>GGvh&9:;686(/23:<6("567[ 4^ U#s   )#N
r   c                 <   |xs t         j                  j                         }|j                         }|j	                         }| j
                  j                  D ]  }t        |j                  j                  d|      |j                  _	        t        |j                  j                  d|      |j                  _
        |j                  xj                  |z  c_        |j                  j                  }||z  |z   fd}|j                  j                  j                  |       |j                  j                  j                  |       t        |j                   t"              rt        |j                   j$                  d|      |j                   _        t        |j                   j&                  d|      |j                   _        t        |j                   j(                  d|      |j                   _        |x|j                   _        }t-        |j                   j.                  j$                  d|       t-        |j                   j.                  j&                  d|       t-        |j                   j.                  j(                  d|       t-        |j                   j0                  j$                  d|       t-        |j                   j0                  j&                  d|       t-        |j                   j0                  j(                  d|        y )Nzall-to-shardedr   zsharded-to-allc                     |  S r   rG   )wehshs    rI   shard_headsz Model.shard.<locals>.shard_headsY  s    BxrH   )rz   r   initr  r  r  r  r   r   r   r   r   r   applyr   
isinstancer   r   r   r   r   r   r   r   r   )	re   r   Nr  rH  r   rU  rS  rT  s	          @@rI   shardzModel.shardH  sY   .,,.JJLzz|ZZ&&E'3((*:%(EOO$ &2&&(8&EOO" OO%%!+%11I	!BiB  OO##))+6OO''--k: %))^4&2II'')9'		# '3II'')9'		# %1II%%'7u%		! 498		(5II,,668HPU II,,668HPU II,,446Fe II((224DE II((224DE II((002B%e 'rH   c                     | j                   j                  | j                   j                  | j                   j                   S r   )r  r  r  r
  )re   s    rI   r  zModel.layers  s-    zz  !5!5

8J8JKKrH   c                     d }|S )Nc                 
    d| vS )Nr   rG   )r   s    rI   	predicatez'Model.cast_predicate.<locals>.predicate  s    ,A55rH   rG   )re   r^  s     rI   cast_predicatezModel.cast_predicate  s    	6 rH   c                 t    | j                   D cg c]  }t        t               t                       c}S c c}w r   )r  r   r   )re   r   s     rI   
make_cachezModel.make_cache  s)    9=EA	')WY/EEEs   #5r   )r?   r@   rA   r   rW   rz   r   r   r   r   rO  r   GrouprZ  propertyr  r_  ra  r   r   s   @rI   r  r    s    Ty T  $!! }!Xt88BNN$8$89 8t L L  FrH   r  )+r   dataclassesr   typingr   r   r   mlx.corecorerz   mlx.nnr\   mlx.nn.layers.distributedr   r   r	   activationsr   rS   r   r   r   rj   r   r   mlar   
rope_utilsr   switch_layersr   r   ModulerK   r   r   compiler   r   r   r   r  r  rG   rH   rI   <module>rp     s     ! & &   P P  T T %  ' $  !  !  !F:
bii :
zO#299 O#dRYY (  D
bii 
4!RYY !Hbii <Bryy BJpFBII pFrH   