
    (HJj'                        d dl mZ d dlmZ d dlmZmZmZmZ d dl	m
Z d dlmZ ddlmZ ddlmZmZmZ ddlmZ dd	lmZ e G d
 de             Z G d dej2                        Zej6                  d        Z G d dej2                        Z G d dej2                        Z G d dej2                        Z G d dej2                        Z  G d dej2                        Z! G d dej2                        Z"y)    )	dataclass)partial)AnyDictOptionalUnionN   )swiglu)BaseModelArgscreate_attention_maskscaled_dot_product_attention)initialize_rope)	SwitchGLUc                   Z   e Zd ZU eed<   eed<   eed<   eed<   eed<   eed<   eed<   ee   ed<   eed	<   eed
<   eed<   eed<   eed<   eed<   eed<   eed<   eed<   dZ	ee   ed<   dZ
eed<   dZee   ed<   dZee   ed<   dZeed<   dZeed<   dZeeeeeef   f      ed<   dZeed<   y)	ModelArgs
model_typehidden_sizenum_hidden_layersintermediate_sizenum_attention_headsrms_norm_eps
vocab_sizemax_position_embeddingsnum_key_value_headsfirst_k_dense_replacemoe_intermediate_sizen_routed_expertsn_shared_expertsnorm_topk_probnum_experts_per_tok
rope_thetarouted_scaling_factorNhead_dim)noaux_tcscoring_funcr	   n_group
topk_groupFattention_biasmlp_biasrope_scalingtie_word_embeddings)__name__
__module____qualname__str__annotations__intfloatr   boolr#   r%   r&   r'   r(   r)   r*   r   r   r+        ]/Users/ahmed/devFolder/claude-voice/.venv/lib/python3.12/site-packages/mlx_lm/models/dots1.pyr   r      s    OO%c]*  "Hhsm"%L#%GXc] !J! ND Hd;?L(4U5#:%6 678? %%r5   r   c            	            e Zd Zdef fdZ	 	 ddej                  deej                     dee   dej                  fdZ	 xZ
S )	Dots1Attentionargsc                    t         |           |j                  }|j                  x| _        }|j
                  x| _        }|j                  xs |j                  |z  }|dz  | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  |||z  d      | _        t        j                  ||z  |d      | _        t        j                  ||j                         | _        t        j                  ||j                         | _        t'        ||j(                  d|j*                  |j,                        | _        y )Ng      Fbiaseps)basetraditionalscaling_configr   )super__init__r   r   n_headsr   
n_kv_headsr#   scalennLinearq_projk_projv_projo_projRMSNormr   q_normk_normr   r!   r*   r   rope)selfr9   dimrD   rE   r#   	__class__s         r6   rC   zDots1Attention.__init__.   s    !%!9!99w'+'?'??*==?D$4$4$?t^
iiWx%7eDiiZ(%:GiiZ(%:Gii( 2CeDjjt/@/@Ajjt/@/@A#,,$($@$@
	r5   xmaskcachereturnc                 n   |j                   \  }}}| j                  |      | j                  |      | j                  |      }	}}| j	                  |j                  ||| j                  d            j                  dddd      }| j                  |j                  ||| j                  d            j                  dddd      }|	j                  ||| j                  d      j                  dddd      }	|P| j                  ||j                        }| j                  ||j                        }|j                  ||	      \  }}	n"| j                  |      }| j                  |      }t        |||	|| j                  |      }
|
j                  dddd      j                  ||d      }
| j                  |
      S )Nr      r	      )offset)rV   rF   rU   )shaperI   rJ   rK   rN   reshaperD   	transposerO   rE   rP   r\   update_and_fetchr   rF   rL   )rQ   rT   rU   rV   BLDquerieskeysvaluesoutputs              r6   __call__zDots1Attention.__call__G   s    ''1a $AAAv++gooaDLL"EFPPq!Q
 {{4<<1doorBCMMq!Q
 1door:DDQ1aPiii=G99T%,,97D 11$?LD&ii(G99T?D-T6djjt
 !!!Q1-55aB?{{6""r5   NN)r,   r-   r.   r   rC   mxarrayr   r   rh   __classcell__rS   s   @r6   r8   r8   -   sW    
Y 
8 $(#	#88# rxx # }	#
 
#r5   r8   c                    |}t        j                  | j                  t         j                              }|}	||z   }||z
  }|dk7  rt        j                  |d|df      }t        j
                  |dd      j                  dd      }
t        j                  |
|dz
  d	
      dd |d d f   }t        j                  ||t        j                  d      d	      }t        j                  |d	d      }|}t        j                  | |dz
  d
      dd |f   }t        j                  |	|d      }|dkD  r|r|j                  dd      }||z  }||z  }||fS )Nr   rY   )axisr]   rZ   ro   T)ro   keepdimsr	   )kthro   .g        )rj   sigmoidastypefloat32	unflattentopksumargpartitionput_along_axisrk   flattentake_along_axis)gatese_score_correction_biastop_kr&   r'   r"   r   kscoresorig_scoresgroup_scores	group_idxindsdenominators                 r6   group_expert_selectr   h   sJ    	AZZRZZ01FK--F*AAvf2gr]Cwwvqr266B6NOOLa!e"Ec2A2qjQ	""69bhhsm"MFB+A??F7AB7RaR@DT;Fqy^jjb4j8+%++F<r5   c                   *     e Zd Zdef fdZd Z xZS )Dots1TopkRouterr9   c                    t         |           |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        t        j                  | j
                  |j                  f      | _        t        j                  | j
                  f      | _        y N)rB   rC   r    r   r   r   r"   r&   r'   rj   zerosr   weightr   rQ   r9   rS   s     r6   rC   zDots1TopkRouter.__init__   s    --
"11 $ 5 5%)%?%?"||//hh 5 5t7G7GHI')xx1F1F0H'I$r5   c           	          t        || j                  j                  z  | j                  | j                  | j
                  | j                  | j                  | j                        S r   )	r   r   Tr   r   r&   r'   r"   r   rQ   rT   s     r6   rh   zDots1TopkRouter.__call__   sO    "((JJLLOO&&
 	
r5   r,   r-   r.   r   rC   rh   rl   rm   s   @r6   r   r      s    	JY 	J	
r5   r   c                   P     e Zd Z	 ddededef fdZdej                  fdZ xZ	S )Dots1MLPr9   r   r   c                    t         |           ||j                  n|| _        ||j                  n|| _        t	        j
                  | j                  | j                  |j                        | _        t	        j
                  | j                  | j                  |j                        | _        t	        j
                  | j                  | j                  |j                        | _	        y )Nr;   )
rB   rC   r   r   rG   rH   r)   	gate_projup_proj	down_proj)rQ   r9   r   r   rS   s       r6   rC   zDots1MLP.__init__   s     	/:/B4++&7&?D""EV 	 d444==
 yyd444==
 ""D$4$44==
r5   rW   c                 t    | j                  t        | j                  |      | j                  |                  S r   )r   r
   r   r   r   s     r6   rh   zDots1MLP.__call__   s)    ~~fT^^A%6QHIIr5   ri   )
r,   r-   r.   r   r1   rC   rj   rk   rh   rl   rm   s   @r6   r   r      s5    QU

,/
KN
(JRXX Jr5   r   c                   *     e Zd Zdef fdZd Z xZS )Dots1MoEr9   c                 8   t         |           |j                  | _        |j                  | _        t	        |j
                  |j                  |j                        | _        t        |      | _
        t        ||j                  |j                  z        | _        y )N)r9   r   )rB   rC   r    r   r   r   r   r   expertsr   gater   shared_expertsr   s     r6   rC   zDots1MoE.__init__   s    #'#;#;  $ 5 5 &&!!
 $D)	&"884;P;PP
r5   c                     | j                  |      \  }}| j                  ||      }||d   z  j                  d      j                  |j                        }| j
                  || j                  |      z   }|S )N).Nrr   rp   )r   r   ry   ru   dtyper   r   )rQ   rT   r   r   ys        r6   rh   zDots1MoE.__call__   su    yy|fLLD!	""''R'077@  ,D''**Ar5   r   rm   s   @r6   r   r      s    
Y 
$r5   r   c            	            e Zd Zdedef fdZ	 	 d	dej                  deej                     dee	   dej                  fdZ
 xZS )
Dots1DecoderLayerr9   	layer_idxc                 b   t         |           t        |      | _        ||j                  k\  rt        |      | _        nt        |      | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        y Nr=   )rB   rC   r8   	self_attnr   r   mlpr   rG   rM   r   r   input_layernormpost_attention_layernormrQ   r9   r   rS   s      r6   rC   zDots1DecoderLayer.__init__   s    '-222~DH~DH!zz$*:*:@Q@QR(*

$"3"3)
%r5   rT   rU   rV   rW   c                     | j                  | j                  |      ||      }||z   }| j                  | j                  |            }||z   S r   )r   r   r   r   )rQ   rT   rU   rV   rhs         r6   rh   zDots1DecoderLayer.__call__   sL     NN4//2D%@EHHT221561ur5   ri   )r,   r-   r.   r   r1   rC   rj   rk   r   r   rh   rl   rm   s   @r6   r   r      s^    
Y 
3 
" $(#		88	 rxx 	 }		
 
	r5   r   c                   `     e Zd Zdef fdZ	 ddej                  dej                  fdZ xZS )
Dots1Modelr9   c                 N   t         |           t        j                  |j                  |j
                        | _        t        |j                        D cg c]  }t        ||       c}| _
        t        j                  |j
                  |j                        | _        y c c}w r   )rB   rC   rG   	Embeddingr   r   embed_tokensranger   r   layersrM   r   normr   s      r6   rC   zDots1Model.__init__   s    LL$:J:JK #4#9#9:
:	 dI.:
 JJt//T5F5FG		
s   B"inputsrW   c                     | j                  |      }|d gt        | j                        z  }t        ||d         }t	        | j                  |      D ]  \  }} ||||      } | j                  |      S )Nr   )r   lenr   r   zipr   )rQ   r   rV   r   rU   layercs          r6   rh   zDots1Model.__call__   su    
 f%=FS--E$Qa1DKK/HE1aq!A 0 yy|r5   r   )	r,   r-   r.   r   rC   rj   rk   rh   rl   rm   s   @r6   r   r      s5    HY H  
	r5   r   c                   ^     e Zd Zdef fdZ	 ddej                  fdZd Ze	d        Z
 xZS )Modelr9   c                     t         |           || _        |j                  | _        t	        |      | _        |j                  s2t        j                  |j                  |j                  d      | _        y y )NFr;   )rB   rC   r9   r   r   modelr+   rG   rH   r   r   lm_headr   s     r6   rC   zModel.__init__  sX    	//%
''99T%5%5tUSDL (r5   r   c                     | j                  ||      }| j                  j                  r'| j                   j                  j	                  |      }|S | j                  |      }|S r   )r   r9   r+   r   	as_linearr   )rQ   r   rV   outs       r6   rh   zModel.__call__  sW    
 jj'99((**))33C8C 
 ,,s#C
r5   c                 H   | j                   j                  r|j                  dd        t        | j                   j                        D ]  }d| }|| j                   j
                  k\  s"dD ]  \  }}dD ]u  }| d| d| |v st        | j                   j                        D cg c]  }|j                  | d| d| d|         }}t        j                  |      || d| d| <   w   |j                         D 	ci c]  \  }}	d|vs||	 c}	}S c c}w c c}	}w )	Nzlm_head.weightzmodel.layers.))w1r   )w2r   )w3r   )r   scalesbiasesz.mlp.experts.0..z.mlp.experts.zrotary_emb.inv_freq)
r9   r+   popr   r   r   r   rj   stackitems)
rQ   weightslprefixnmr   eto_joinvs
             r6   sanitizezModel.sanitize$  s=   99((KK($/tyy223A$QC(FDII333DAq
 <$X_QCq<G */tyy/I/I)J')JA !(vhmA3as!A3,O P)J $ ' HJxxPWGXGvhmA3as$CD < 4  ")SA4IQR4R1SS' Ts   %#D
DDc                 .    | j                   j                  S r   )r   r   )rQ   s    r6   r   zModel.layers:  s    zz   r5   r   )r,   r-   r.   r   rC   rj   rk   rh   r   propertyr   rl   rm   s   @r6   r   r     sA    TY T 

T, ! !r5   r   )#dataclassesr   	functoolsr   typingr   r   r   r   mlx.corecorerj   mlx.nnrG   activationsr
   r?   r   r   r   
rope_utilsr   switch_layersr   r   Moduler8   compiler   r   r   r   r   r   r   r4   r5   r6   <module>r      s    "  - -    T T ' $ & & &88#RYY 8#v  B
bii 
0Jryy J2ryy :		 6 8-!BII -!r5   